From 900a5c4f1ff40ff091fc1648027ee56b9a1a9592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 8 Oct 2026 08:41:25 +0200 Subject: [PATCH] perf: stop keeping folded samples and re-reading duplicates The nightly folds soft-deleted every raw Apple Health sample they folded into an hourly or daily stats row and kept it for 75 days. Nothing ever read those rows as deletions; they only held a re-upload of the sample off, and they were most of the measurements table and its indexes. Ingest now refuses such a re-upload directly: a raw sample of a dense or mean type, old enough to have been folded, whose hour or day a live stats row covers, is a duplicate with reason folded_window. The batch route and the Apple Health export import apply the same rule. With that in place the dense retention and the daily-mean consolidation delete the raw rows outright, and a new compaction-tombstone-purge job removes the backlog: 5,000 rows a batch, one account at a time under the restore lock, a short pause between batches, resumable because the predicate is idempotent, queued at every boot and continuing itself while it makes progress. A person's own deletions are never in its predicate. The sync feed leaves compaction tombstones out and a restore does not write them back; all three classify through the same function. The batch route answers an exact re-sent sample without a transaction: one read over the batch's sample ids, live or deleted, decides what the reconciler would have decided, at the same point in the loop, so every status, reason, counter and the cross-source merge stay as they were. A batch of duplicates only no longer queues PR detection, a reminder sweep or the detection audit row. Caches are evicted only for a foreground or manual sync or a batch that wrote a hand-entered reading, and marked stale otherwise; the targets cache follows the same split. Also: - POST /api/admin/maintenance/measurements queues VACUUM (ANALYZE) and a REINDEX INDEX CONCURRENTLY per index, largest first, on a connection of its own; it refuses while the purge runs, drops invalid leftover copies and stops at the first failed rebuild. Runbook in docs/ops. - The pure volume queues keep their finished jobs for 96 hours instead of seven days, never less than the failure window plus a day. - work_mem is set per app connection from DATABASE_WORK_MEM (16MB), also when the statement timeout is disabled. - The PR-detection fallback tick re-scans only accounts whose readings or workouts changed since the previous tick. - Type discovery for the status refill walks the live-rows index as a loose index scan instead of grouping every live row. - The host metrics chart polls every five minutes, like its sampler. - The batch route records per-type HealthKit arrivals. --- docs/ops/measurement-maintenance.md | 111 +++ knip.jsonc | 3 - .../dense-measurement-read-guard.test.ts | 2 + .../measurement-write-tail-guard.test.ts | 2 + .../admin/maintenance/measurements/route.ts | 90 ++- .../__tests__/aggregated-hr-wire.test.ts | 20 +- .../__tests__/duplicate-prefilter.test.ts | 616 ++++++++++++++++ .../__tests__/hr-bucket-observability.test.ts | 2 + .../batch/__tests__/sync-trigger.test.ts | 6 +- src/app/api/measurements/batch/route.ts | 248 ++++++- src/app/api/sync/changes/route.ts | 35 + src/components/admin/host-metrics-chart.tsx | 11 +- src/lib/__tests__/db.test.ts | 29 +- src/lib/cache/__tests__/invalidate.test.ts | 18 + src/lib/cache/invalidate.ts | 20 +- src/lib/db.ts | 49 +- src/lib/export/restore-backup.ts | 65 +- src/lib/export/streamed-backup.ts | 8 +- .../invalidate-status-insights.test.ts | 18 +- src/lib/insights/status-invalidation.ts | 17 +- .../measurement-tombstone-cleanup.test.ts | 21 +- src/lib/jobs/compaction-tombstone-purge.ts | 341 ++++++++- src/lib/jobs/dense-intraday-retention.ts | 7 +- src/lib/jobs/host-metric-sampler.ts | 8 +- src/lib/jobs/mean-consolidation.ts | 8 +- src/lib/jobs/measurement-maintenance.ts | 363 ++++++++- src/lib/jobs/measurement-tombstone-cleanup.ts | 24 +- src/lib/jobs/pr-detection.ts | 73 +- .../__tests__/queue-retention.test.ts | 87 +++ src/lib/jobs/reminder/ops-handlers.ts | 17 +- src/lib/jobs/reminder/register-maintenance.ts | 16 + src/lib/jobs/reminder/registrar-shared.ts | 88 ++- .../__tests__/consolidate-daily-mean.test.ts | 33 +- .../dense-intraday-pulse-summary.test.ts | 12 +- .../dense-intraday-resting-mint.test.ts | 2 + .../dense-intraday-retention.test.ts | 26 +- .../dense-intraday-spo2-summary.test.ts | 32 +- .../__tests__/folded-window.test.ts | 228 ++++++ .../measurements/consolidate-daily-mean.ts | 20 +- .../measurements/dense-intraday-retention.ts | 37 +- src/lib/measurements/folded-window.ts | 191 ++++- .../import-apple-health-export.ts | 44 +- src/lib/measurements/live-types.ts | 69 +- .../integration/compaction-tombstones.test.ts | 693 ++++++++++++++++++ .../consolidate-daily-mean.test.ts | 14 +- .../dense-intraday-retention.test.ts | 8 +- tests/integration/sync-changes.test.ts | 7 +- .../tombstone-cleanup-restore-lock.test.ts | 15 +- 48 files changed, 3619 insertions(+), 235 deletions(-) create mode 100644 docs/ops/measurement-maintenance.md create mode 100644 src/app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts create mode 100644 src/lib/jobs/reminder/__tests__/queue-retention.test.ts create mode 100644 src/lib/measurements/__tests__/folded-window.test.ts create mode 100644 tests/integration/compaction-tombstones.test.ts diff --git a/docs/ops/measurement-maintenance.md b/docs/ops/measurement-maintenance.md new file mode 100644 index 000000000..070b7b0a6 --- /dev/null +++ b/docs/ops/measurement-maintenance.md @@ -0,0 +1,111 @@ +# Measurement table maintenance + +Since v1.42 the nightly folds of Apple Health data delete the raw samples they +fold into hourly or daily rows, instead of keeping them as deleted rows for 75 +days. On an instance that has synced Apple Health for a while, those leftovers +("compaction tombstones") were most of the `measurements` table and of its +indexes. Two pieces clean them up: + +1. **The backlog purge** runs by itself. Every worker boot queues one + `compaction-tombstone-purge` job. It deletes the leftovers 5,000 rows at a + time, one account at a time, pausing briefly between batches, and stops + after 40 batches; a run that stopped with work left queues the next one a + minute later. Rows a person deleted are never touched by it: they keep the + 75-day retention of the nightly tombstone cleanup. +2. **The table maintenance** is yours to start, after the purge. It runs + `VACUUM (ANALYZE)` on `measurements` and then rebuilds each of its indexes + with `REINDEX INDEX CONCURRENTLY`, largest first. The purge leaves the + indexes full of empty pages, and only a rebuild gives that space back. + +## Is the purge finished? + +The purge reports every run on the wide event `job.compaction_tombstone_purge` +(`compaction_purge_deleted`, `compaction_purge_drained`, +`compaction_purge_deferred_accounts`). It is finished when a run reports +`drained: true` and deletes nothing. From `psql`: + +```sql +SELECT state, count(*) +FROM pgboss.job +WHERE name = 'compaction-tombstone-purge' +GROUP BY state; +``` + +No `created`, `retry` or `active` rows means nothing is queued or running. The +maintenance refuses to start while one is. + +## Before you start + +- **Pick a quiet window.** Reads and writes keep working during the rebuild, + but each rebuild reads the whole table twice and writes a full new copy of + the index. +- **Free disk.** A concurrent rebuild needs room for one more copy of the + index it is rebuilding, plus the WAL it writes. Check the largest index: + + ```sql + SELECT c.relname, pg_size_pretty(pg_relation_size(c.oid)) + FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid + WHERE i.indrelid = 'measurements'::regclass + ORDER BY pg_relation_size(c.oid) DESC + LIMIT 3; + ``` + + Have at least twice the largest size free on the database volume. + +- **Take a backup** (Settings, Data, Backups), as before any operation on the + whole table. + +## Starting it + +In the admin area, or with a signed-in admin session: + +```sh +curl -X POST https:///api/admin/maintenance/measurements \ + -H 'content-type: application/json' \ + --cookie 'healthlog_session=' \ + -d '{"vacuum": true, "reindex": true}' +``` + +The answer is `202` with `{ "enqueued": true }`, or `enqueued: false` when a +run is already queued or running. A Bearer token is refused: the route is +cookie-only, like every admin route. Without a background worker the route +answers `503`. + +Either step can be left out (`"reindex": false` runs only the vacuum). + +## What it does, and what stops it + +The run uses a connection of its own: no statement timeout (a rebuild of a +large index takes longer than the app's 60 seconds), `maintenance_work_mem` +of 128 MB for this session only, and a lock timeout of five minutes, so a +rebuild that cannot get its lock gives up instead of holding writers behind +it. + +- It refuses to start while the purge is queued or running. +- Before the first rebuild it drops any invalid `_ccnew` copy an + interrupted earlier run left behind. +- If a rebuild fails, most likely for lack of disk, it drops that rebuild's + half-built copy at once and stops. The remaining indexes keep working as + they are. Free some space and start it again; finished indexes are simply + rebuilt once more. +- One run at a time. It is never retried automatically. + +## Reading the result + +The run's wide event `job.measurement_maintenance` carries +`maintenance_outcome` (`completed`, `refused_purge_running`, `failed`), the +table and index sizes before and after, and `maintenance_steps`, one entry +per step with its target, size before and after, and duration. The same +sizes from `psql`: + +```sql +SELECT pg_size_pretty(pg_total_relation_size('measurements')) AS total, + pg_size_pretty(pg_indexes_size('measurements')) AS indexes; +``` + +## Not part of this + +`VACUUM FULL` would shrink the table file as well, but it locks the table for +reads and writes while it runs. After the purge the remaining table is small +and the space inside it is reused by new rows, so it is not needed. If you +want it anyway, run it yourself in a maintenance window with the app stopped. diff --git a/knip.jsonc b/knip.jsonc index 1eefe5575..6a01a4198 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -126,9 +126,6 @@ // a line the moment its first consumer lands; the release integration // checklist removes whatever is left before the tag. "src/lib/environment/air-quality-contract.ts", - "src/lib/integrations/healthkit-type-sync.ts", - "src/lib/measurements/folded-window.ts", - "src/lib/measurements/live-types.ts", ], "ignoreDependencies": [ "@types/node", diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index 224bfc15b..8c54282fd 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -123,6 +123,8 @@ const ALLOWED_SQL: Record = { "Canonical-source blood pressure and weight over 90 days: sparse types, a few rows a day.", "lib/measurements/reconcile-external-measurement.ts::1": "Row locks (`FOR UPDATE`) on an id list the caller already read; returns those ids only.", + "lib/jobs/pr-detection.ts::1": + "Selects account ids; `measurements` is read inside `EXISTS`, one row per account at most.", "lib/rollups/measurement-coverage.ts::1": "Selects from `unnest` of the caller's type list; `measurements` is read inside `EXISTS`, one row per type at most.", }; diff --git a/src/__tests__/measurement-write-tail-guard.test.ts b/src/__tests__/measurement-write-tail-guard.test.ts index dc56b62f8..231a9cd24 100644 --- a/src/__tests__/measurement-write-tail-guard.test.ts +++ b/src/__tests__/measurement-write-tail-guard.test.ts @@ -98,6 +98,8 @@ const WRITE_SITES: Record = { "encrypts the note column in place; no value, instant or type changes", "lib/jobs/measurement-tombstone-cleanup.ts": "hard-deletes rows that were already soft-deleted and read nowhere", + "lib/jobs/compaction-tombstone-purge.ts": + "hard-deletes compaction tombstones, rows already soft-deleted and read nowhere", }; function sourceFiles(): string[] { diff --git a/src/app/api/admin/maintenance/measurements/route.ts b/src/app/api/admin/maintenance/measurements/route.ts index 624a37271..618f02fbc 100644 --- a/src/app/api/admin/maintenance/measurements/route.ts +++ b/src/app/api/admin/maintenance/measurements/route.ts @@ -3,12 +3,90 @@ * `VACUUM (ANALYZE)` / `REINDEX INDEX CONCURRENTLY` pass over `measurements` * (v1.42). Cookie-only admin: `requireAdmin()` refuses every Bearer. * - * Contract stub: authenticates as the finished route will, then answers 501. + * Nothing runs in the request: the pass runs in the background worker + * (`measurement-maintenance.ts`), one statement at a time, and refuses to + * start while the compaction-tombstone purge is still working. At most one + * run is queued or running; a second press answers `enqueued: false`. + * Runbook: `docs/ops/measurement-maintenance.md`. */ -import { apiHandler, requireAdmin } from "@/lib/api-handler"; -import { notImplemented } from "@/lib/http/not-implemented"; +import { NextRequest } from "next/server"; +import { z } from "zod/v4"; -export const POST = apiHandler(async () => { - await requireAdmin(); - return notImplemented(); +import { apiHandler, requireAdmin } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + getClientIp, + returnAllZodIssues, +} from "@/lib/api-response"; +import { auditLog } from "@/lib/auth/audit"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { + MEASUREMENT_MAINTENANCE_QUEUE, + MEASUREMENT_MAINTENANCE_SEND_OPTIONS, + type MeasurementMaintenancePayload, +} from "@/lib/jobs/measurement-maintenance"; +import { annotate } from "@/lib/logging/context"; + +/** Two booleans; anything longer is not a request for this route. */ +const MAX_BODY_BYTES = 1024; + +const maintenanceRequestSchema = z + .object({ + vacuum: z.boolean().default(true), + reindex: z.boolean().default(true), + }) + .strict(); + +export const POST = apiHandler(async (request: NextRequest) => { + const { user } = await requireAdmin(); + + // The body is optional: an empty one runs both steps. + const text = await request.text(); + if (text.length > MAX_BODY_BYTES) { + return apiError("Request body too large", 413); + } + let body: unknown = {}; + if (text.trim().length > 0) { + try { + body = JSON.parse(text); + } catch { + return apiError("Invalid JSON", 400); + } + } + const parsed = maintenanceRequestSchema.safeParse(body); + if (!parsed.success) return returnAllZodIssues(parsed.error); + const payload: MeasurementMaintenancePayload = { + vacuum: parsed.data.vacuum, + reindex: parsed.data.reindex, + }; + if (!payload.vacuum && !payload.reindex) { + return apiError("Choose at least one step", 422); + } + + annotate({ + action: { name: "admin.maintenance.measurements.requested" }, + meta: { vacuum: payload.vacuum, reindex: payload.reindex }, + }); + + const boss = getGlobalBoss(); + if (!boss) { + return apiError("Background worker is not available", 503); + } + + const jobId = await boss.send( + MEASUREMENT_MAINTENANCE_QUEUE, + payload, + MEASUREMENT_MAINTENANCE_SEND_OPTIONS, + ); + const enqueued = jobId !== null; + + await auditLog("admin.maintenance.measurements.requested", { + userId: user.id, + ipAddress: getClientIp(request), + details: { ...payload, enqueued }, + }); + annotate({ meta: { maintenance_enqueued: enqueued } }); + + return apiSuccess({ enqueued }, 202); }); diff --git a/src/app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts b/src/app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts index 3f766d9fb..085fe93d4 100644 --- a/src/app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts +++ b/src/app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts @@ -19,6 +19,8 @@ vi.mock("@/lib/db", () => ({ prisma: { user: { update: vi.fn(), + // The `folded_window` guard reads the account's zone for old samples. + findUnique: vi.fn().mockResolvedValue({ timezone: null }), }, measurement: { findMany: vi.fn(), @@ -366,7 +368,14 @@ describe("POST /api/measurements/batch — aggregated HR wire contract (iOS #34) it("keeps the per-sample uuid HR path immutable (duplicate, not overwrite)", async () => { const SAMPLE_ID = "B3A1C0DE-0000-4000-8000-000000000000"; vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { type: "PULSE", source: "APPLE_HEALTH", externalId: SAMPLE_ID }, + { + type: "PULSE", + source: "APPLE_HEALTH", + externalId: SAMPLE_ID, + // Not the posted instant, so the exact-duplicate prefilter leaves + // the row to the reconciler this suite stubs. + measuredAt: new Date(0), + }, ] as never); const res = await POST( @@ -461,7 +470,14 @@ describe("POST /api/measurements/batch — stats tombstone resurrection", () => // sample stays a checkpointing `duplicate` — no update runs at all. const SAMPLE_ID = "D00DAD00-0000-4000-8000-000000000000"; vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { type: "PULSE", source: "APPLE_HEALTH", externalId: SAMPLE_ID }, + { + type: "PULSE", + source: "APPLE_HEALTH", + externalId: SAMPLE_ID, + // Not the posted instant, so the exact-duplicate prefilter leaves + // the row to the reconciler this suite stubs. + measuredAt: new Date(0), + }, ] as never); const res = await POST( diff --git a/src/app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts b/src/app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts new file mode 100644 index 000000000..63d2c32bf --- /dev/null +++ b/src/app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts @@ -0,0 +1,616 @@ +/** + * v1.42 — the exact-duplicate prefilter, the `folded_window` guard, and what + * a batch of duplicates only no longer sets off. + * + * The route runs against an in-memory `measurements` table and a reconciler + * stand-in that answers from it the way `reconcileExternalMeasurement` does + * for samples (external and natural identity, live or tombstoned). The golden + * comparison runs every batch twice: once as shipped, once with the + * prefilter's read answering nothing, which is the pre-v1.42 path where every + * row went through the reconciler. Statuses, reasons and counters must match + * exactly; only the transaction count may differ. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +interface StoredRow { + id: string; + userId: string; + type: string; + source: string; + externalId: string | null; + measuredAt: Date; + sleepStage: string | null; + value: number; + deletedAt: Date | null; +} + +const store = vi.hoisted(() => ({ + rows: [] as StoredRow[], + /** Golden mode: the prefilter's read answers nothing. */ + prefilterBlind: false, + nextId: 1, +})); + +function sameInstant(a: Date, b: Date) { + return a.getTime() === b.getTime(); +} + +vi.mock("@/lib/db", () => { + const prisma = { + user: { + update: vi.fn().mockResolvedValue({}), + findUnique: vi.fn().mockResolvedValue({ timezone: "Europe/Berlin" }), + }, + measurement: { + findMany: vi.fn( + async (args: { + where: { + userId: string; + OR?: unknown; + deletedAt?: null; + type?: { in: string[] }; + source?: string; + externalId?: { in: string[] }; + }; + }) => { + const where = args.where; + // Cross-source merge probe: these suites post no MANUAL twins. + if (where.OR) return []; + const ids = where.externalId?.in ?? []; + // The `folded_window` lookup: live `stats:` rows by id. + if (where.deletedAt === null) { + return store.rows.filter( + (r) => + r.userId === where.userId && + r.deletedAt === null && + r.source === where.source && + (where.type?.in ?? []).includes(r.type) && + r.externalId !== null && + ids.includes(r.externalId), + ); + } + // The prefilter: every row, live or not, carrying a posted id. + if (store.prefilterBlind) return []; + return store.rows.filter( + (r) => + r.userId === where.userId && + r.externalId !== null && + ids.includes(r.externalId), + ); + }, + ), + }, + $transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn({})), + }; + return { prisma }; +}); + +vi.mock("@/lib/measurements/reconcile-external-measurement", () => ({ + reconcileExternalMeasurement: async ( + _tx: unknown, + desired: { + userId: string; + type: string; + source: string; + externalId: string; + measuredAt: Date; + sleepStage?: string | null; + value: number; + }, + options: { exactExternalMatch?: "update" | "duplicate" } = {}, + ) => { + const scope = store.rows.filter( + (r) => + r.userId === desired.userId && + r.type === desired.type && + r.source === desired.source, + ); + const externalHit = scope.find((r) => r.externalId === desired.externalId); + const naturalHit = scope.find( + (r) => + sameInstant(r.measuredAt, desired.measuredAt) && + r.sleepStage === (desired.sleepStage ?? null), + ); + if (!externalHit && !naturalHit) { + const row: StoredRow = { + id: `m${store.nextId++}`, + userId: desired.userId, + type: desired.type, + source: desired.source, + externalId: desired.externalId, + measuredAt: desired.measuredAt, + sleepStage: desired.sleepStage ?? null, + value: desired.value, + deletedAt: null, + }; + store.rows.push(row); + return { status: "inserted", row }; + } + const canonical = (externalHit ?? naturalHit)!; + const redundant = + externalHit && naturalHit && externalHit !== naturalHit + ? naturalHit + : undefined; + if ( + options.exactExternalMatch === "duplicate" && + externalHit === naturalHit + ) { + return { status: "duplicate", row: canonical }; + } + if (redundant) { + redundant.deletedAt = new Date(); + redundant.measuredAt = new Date(0); + } + const wasDeleted = canonical.deletedAt !== null; + canonical.externalId = desired.externalId; + canonical.measuredAt = desired.measuredAt; + canonical.sleepStage = desired.sleepStage ?? null; + canonical.value = desired.value; + canonical.deletedAt = null; + return { + status: wasDeleted ? "resurrected" : "updated", + row: canonical, + dirtyIdentities: [], + }; + }, +})); + +vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); +vi.mock("@/lib/auth/audit", () => ({ + auditLog: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn() })); +vi.mock("@/lib/jobs/pr-detection", () => ({ + enqueuePrDetection: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/reminder-satisfy", () => ({ + enqueueReminderSatisfy: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserMeasurements: vi.fn(), +})); +vi.mock("@/lib/rollups/after-measurement-mutation", () => ({ + afterMeasurementMutation: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/arrivals/emit-shared", () => ({ + emitDataArrival: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/daily/morning-refresh-trigger", () => ({ + maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("next/headers", () => ({ + headers: vi.fn(async () => ({ get: () => null })), + cookies: vi.fn(async () => ({ + get: () => undefined, + set: () => {}, + delete: () => {}, + })), +})); + +import { POST } from "../route"; +import { prisma } from "@/lib/db"; +import { getSession } from "@/lib/auth/session"; +import { auditLog } from "@/lib/auth/audit"; +import { checkRateLimit } from "@/lib/rate-limit"; +import { enqueuePrDetection } from "@/lib/jobs/pr-detection"; +import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; +import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; + +const USER = "user-1"; +const NOW = Date.now(); +const HOUR = 3_600_000; +const DAY = 24 * HOUR; + +interface Entry { + hkIdentifier: string; + value: number; + unit: string; + startDate: string; + endDate: string; + externalId: string; + sleepStage?: number; + source?: "APPLE_HEALTH" | "MANUAL"; +} + +function weight(externalId: string, at: Date, value = 80): Entry { + return { + hkIdentifier: "HKQuantityTypeIdentifierBodyMass", + value, + unit: "kg", + startDate: at.toISOString(), + endDate: at.toISOString(), + externalId, + }; +} + +function pulse(externalId: string, at: Date, value = 60): Entry { + return { + hkIdentifier: "HKQuantityTypeIdentifierHeartRate", + value, + unit: "count/min", + startDate: at.toISOString(), + endDate: at.toISOString(), + externalId, + }; +} + +function seed(row: Partial & { externalId: string; type: string }) { + store.rows.push({ + id: `seed${store.nextId++}`, + userId: USER, + source: "APPLE_HEALTH", + measuredAt: new Date(NOW - DAY), + sleepStage: null, + value: 80, + deletedAt: null, + ...row, + }); +} + +function request(entries: Entry[], syncTrigger?: string) { + return new NextRequest("http://localhost/api/measurements/batch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ entries, ...(syncTrigger ? { syncTrigger } : {}) }), + }); +} + +type Body = { + data: { + processed: number; + inserted: number; + updated: number; + duplicates: number; + failed: number; + entries: Array<{ index: number; status: string; reason?: string }>; + }; +}; + +async function post(entries: Entry[], syncTrigger?: string) { + const res = await POST(request(entries, syncTrigger)); + expect(res.status).toBe(200); + return ((await res.json()) as Body).data; +} + +/** Run a batch on a copy of the store as shipped and on a copy without the prefilter. */ +async function golden(setup: () => void, entries: Entry[]) { + store.rows = []; + store.nextId = 1; + setup(); + store.prefilterBlind = true; + vi.mocked(prisma.$transaction).mockClear(); + const old = await post(entries); + const oldTransactions = vi.mocked(prisma.$transaction).mock.calls.length; + const oldRows = JSON.stringify(store.rows); + + store.rows = []; + store.nextId = 1; + setup(); + store.prefilterBlind = false; + vi.mocked(prisma.$transaction).mockClear(); + const next = await post(entries); + const transactions = vi.mocked(prisma.$transaction).mock.calls.length; + expect(next).toEqual(old); + expect(JSON.stringify(store.rows)).toBe(oldRows); + return { old, next, oldTransactions, transactions }; +} + +beforeEach(() => { + vi.clearAllMocks(); + store.rows = []; + store.prefilterBlind = false; + vi.mocked(getSession).mockResolvedValue({ + session: { id: "s", expiresAt: new Date(NOW + HOUR) }, + user: { id: USER, username: "u", role: "USER" as const }, + } as never); + vi.mocked(checkRateLimit).mockResolvedValue({ + allowed: true, + limit: 60, + remaining: 60, + resetAt: NOW + 60_000, + }); +}); + +describe("exact-duplicate prefilter (v1.42)", () => { + it("1: 500 exact live duplicates open no transaction and answer as before", async () => { + const entries = Array.from({ length: 500 }, (_, i) => + weight(`uuid-${i}`, new Date(NOW - DAY - i * 60_000)), + ); + const setup = () => + entries.forEach((e) => + seed({ + type: "WEIGHT", + externalId: e.externalId, + measuredAt: new Date(e.endDate), + }), + ); + const { next, oldTransactions, transactions } = await golden( + setup, + entries, + ); + expect(next.duplicates).toBe(500); + expect(oldTransactions).toBe(500); + expect(transactions).toBe(0); + }); + + it("2: an exact hit on a tombstone stays a duplicate and the row stays deleted", async () => { + const at = new Date(NOW - DAY); + const setup = () => + seed({ + type: "WEIGHT", + externalId: "uuid-gone", + measuredAt: at, + deletedAt: new Date(NOW - HOUR), + }); + const { next, transactions } = await golden(setup, [ + weight("uuid-gone", at), + ]); + expect(next.entries[0]).toEqual({ index: 0, status: "duplicate" }); + expect(transactions).toBe(0); + expect(store.rows[0].deletedAt).not.toBeNull(); + }); + + it("3: the same id at another instant still reconciles to `updated`", async () => { + const setup = () => + seed({ + type: "WEIGHT", + externalId: "uuid-move", + measuredAt: new Date(NOW - 2 * DAY), + }); + const { next, transactions } = await golden(setup, [ + weight("uuid-move", new Date(NOW - DAY)), + ]); + expect(next.entries[0].status).toBe("updated"); + expect(transactions).toBe(1); + }); + + it("4: the same id with another sleep stage still reconciles", async () => { + const at = new Date(NOW - DAY); + const setup = () => + seed({ + type: "SLEEP_DURATION", + externalId: "uuid-sleep", + measuredAt: at, + sleepStage: "DEEP", + value: 30, + }); + const entry: Entry = { + hkIdentifier: "HKCategoryTypeIdentifierSleepAnalysis", + value: 30, + unit: "min", + startDate: new Date(at.getTime() - 30 * 60_000).toISOString(), + endDate: at.toISOString(), + externalId: "uuid-sleep", + sleepStage: 3, + }; + const { transactions } = await golden(setup, [entry]); + expect(transactions).toBe(1); + }); + + it("5: `stats:` rows are never prefiltered; a second snapshot is superseded", async () => { + const day = new Date(NOW - DAY).toISOString().slice(0, 10); + const id = `stats:HKQuantityTypeIdentifierStepCount:${day}`; + const steps = (value: number): Entry => ({ + hkIdentifier: "HKQuantityTypeIdentifierStepCount", + value, + unit: "count", + startDate: `${day}T00:00:00.000Z`, + endDate: `${day}T18:00:00.000Z`, + externalId: id, + }); + const setup = () => + seed({ + type: "ACTIVITY_STEPS", + externalId: id, + measuredAt: new Date(`${day}T18:00:00.000Z`), + value: 1000, + }); + const { next, transactions } = await golden(setup, [ + steps(2000), + steps(3000), + ]); + expect(next.entries).toEqual([ + { index: 0, status: "duplicate", reason: "superseded_in_batch" }, + { index: 1, status: "updated" }, + ]); + expect(transactions).toBe(1); + }); + + it("6: an id twice in the batch and not stored inserts once, then duplicates", async () => { + const at = new Date(NOW - DAY); + const { next } = await golden(() => {}, [ + weight("uuid-twice", at), + weight("uuid-twice", at), + ]); + expect(next.entries.map((e) => e.status)).toEqual([ + "inserted", + "duplicate", + ]); + }); + + it("6b: a stored id posted twice is left to the reconciler", async () => { + const at = new Date(NOW - DAY); + const setup = () => + seed({ type: "WEIGHT", externalId: "uuid-twice", measuredAt: at }); + const { transactions } = await golden(setup, [ + weight("uuid-twice", at), + weight("uuid-twice", at), + ]); + expect(transactions).toBe(2); + }); + + it("9: the same id held by another account is not a duplicate", async () => { + const at = new Date(NOW - DAY); + const setup = () => + seed({ + type: "WEIGHT", + externalId: "uuid-shared", + measuredAt: at, + userId: "user-2", + }); + const { next } = await golden(setup, [weight("uuid-shared", at)]); + expect(next.entries[0].status).toBe("inserted"); + }); + + it("10: an all-duplicate batch still sets the sync checkpoint", async () => { + const at = new Date(NOW - DAY); + seed({ type: "WEIGHT", externalId: "uuid-dup", measuredAt: at }); + await post([weight("uuid-dup", at)], "background"); + expect(prisma.user.update).toHaveBeenCalledWith({ + where: { id: USER }, + data: { + lastSyncedAt: expect.any(Date), + healthKitLastSyncedAt: expect.any(Date), + healthKitLastSyncTrigger: "background", + healthKitLastBackgroundSyncAt: expect.any(Date), + }, + }); + }); +}); + +describe("a batch of duplicates only (v1.42)", () => { + it("enqueues no PR detection, no reminder sweep and writes no detection audit row", async () => { + const at = new Date(NOW - DAY); + seed({ type: "WEIGHT", externalId: "uuid-dup", measuredAt: at }); + await post([weight("uuid-dup", at)]); + expect(enqueuePrDetection).not.toHaveBeenCalled(); + expect(enqueueReminderSatisfy).not.toHaveBeenCalled(); + expect( + vi + .mocked(auditLog) + .mock.calls.some( + ([action]) => action === "personal_records.detection_enqueued", + ), + ).toBe(false); + }); + + it("still enqueues all three when a row was written", async () => { + await post([weight("uuid-new", new Date(NOW - DAY))]); + expect(enqueuePrDetection).toHaveBeenCalledTimes(1); + expect(enqueueReminderSatisfy).toHaveBeenCalledTimes(1); + expect( + vi + .mocked(auditLog) + .mock.calls.some( + ([action]) => action === "personal_records.detection_enqueued", + ), + ).toBe(true); + }); +}); + +describe("cache invalidation by origin (v1.42)", () => { + it.each([ + ["foreground", { evict: true }], + ["manual", { evict: true }], + ["background", undefined], + ["push", undefined], + [undefined, undefined], + ])("syncTrigger %s → %o", async (trigger, expected) => { + await post([weight(`uuid-${trigger}`, new Date(NOW - DAY))], trigger); + expect(invalidateUserMeasurements).toHaveBeenCalledWith(USER, expected); + }); + + it("evicts for a background batch that wrote a hand-entered reading", async () => { + await post( + [ + { + ...weight("manual-1", new Date(NOW - DAY)), + source: "MANUAL", + }, + ], + "background", + ); + expect(invalidateUserMeasurements).toHaveBeenCalledWith(USER, { + evict: true, + }); + }); +}); + +describe("folded_window guard (v1.42)", () => { + // Europe/Berlin is UTC+1 in winter and UTC+2 in summer; the store's + // account zone is Berlin. 120 days back is past the 90-day dense window. + const old = new Date(NOW - 120 * DAY); + const localHourId = () => { + const parts = new Intl.DateTimeFormat("en-CA", { + timeZone: "Europe/Berlin", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + hour12: false, + }).formatToParts(old); + const get = (t: string) => parts.find((p) => p.type === t)!.value; + const hour = get("hour") === "24" ? "00" : get("hour"); + return `stats:HKQuantityTypeIdentifierHeartRate:${get("year")}-${get("month")}-${get("day")}T${hour}`; + }; + + it("12a: an old raw sample under a live hourly `stats:` row is a folded duplicate", async () => { + seed({ + type: "PULSE", + externalId: localHourId(), + measuredAt: new Date(old.getTime() + 1), + }); + const data = await post([pulse("uuid-old-hr", old)]); + expect(data.entries[0]).toEqual({ + index: 0, + status: "duplicate", + reason: "folded_window", + }); + expect(store.rows.some((r) => r.externalId === "uuid-old-hr")).toBe(false); + expect(prisma.$transaction).not.toHaveBeenCalled(); + }); + + it("12b: without a covering `stats:` row it is inserted", async () => { + const data = await post([pulse("uuid-old-hr", old)]); + expect(data.entries[0].status).toBe("inserted"); + }); + + it("12b': a tombstoned `stats:` row does not cover", async () => { + seed({ + type: "PULSE", + externalId: localHourId(), + measuredAt: new Date(old.getTime() + 1), + deletedAt: new Date(), + }); + const data = await post([pulse("uuid-old-hr", old)]); + expect(data.entries[0].status).toBe("inserted"); + }); + + it("12c: a mean-type sample younger than 36 h is inserted even under a daily row", async () => { + const recent = new Date(NOW - 2 * HOUR); + const day = new Intl.DateTimeFormat("en-CA", { + timeZone: "Europe/Berlin", + }).format(recent); + seed({ + type: "RESPIRATORY_RATE", + externalId: `stats:HKQuantityTypeIdentifierRespiratoryRate:${day}`, + measuredAt: new Date(recent.getTime() + 1), + }); + const data = await post([ + { + hkIdentifier: "HKQuantityTypeIdentifierRespiratoryRate", + value: 14, + unit: "count/min", + startDate: recent.toISOString(), + endDate: recent.toISOString(), + externalId: "uuid-rr", + }, + ]); + expect(data.entries[0].status).toBe("inserted"); + }); + + it("does not touch a MANUAL reading of the same instant", async () => { + seed({ + type: "PULSE", + externalId: localHourId(), + measuredAt: new Date(old.getTime() + 1), + }); + const data = await post([{ ...pulse("manual-hr", old), source: "MANUAL" }]); + expect(data.entries[0].status).toBe("inserted"); + }); +}); diff --git a/src/app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts b/src/app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts index 7597199f4..188687af5 100644 --- a/src/app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts +++ b/src/app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts @@ -21,6 +21,8 @@ vi.mock("@/lib/db", () => ({ prisma: { user: { update: vi.fn(), + // The `folded_window` guard reads the account's zone for old samples. + findUnique: vi.fn().mockResolvedValue({ timezone: null }), }, measurement: { findMany: vi.fn(), diff --git a/src/app/api/measurements/batch/__tests__/sync-trigger.test.ts b/src/app/api/measurements/batch/__tests__/sync-trigger.test.ts index 83a7416d5..38f2318fb 100644 --- a/src/app/api/measurements/batch/__tests__/sync-trigger.test.ts +++ b/src/app/api/measurements/batch/__tests__/sync-trigger.test.ts @@ -25,7 +25,11 @@ const mocks = vi.hoisted(() => ({ annotate: vi.fn() })); vi.mock("@/lib/db", () => ({ prisma: { - user: { update: vi.fn() }, + user: { + update: vi.fn(), + // The `folded_window` guard reads the account's zone for old samples. + findUnique: vi.fn().mockResolvedValue({ timezone: null }), + }, measurement: { findMany: vi.fn(), createManyAndReturn: vi.fn(), diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index 8e23b2148..fa0c55400 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -65,6 +65,15 @@ import { type MergeCandidate, } from "@/lib/measurements/cross-source-merge"; import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; +import { classifyRangeRejection } from "@/lib/measurements/plausibility-gate"; +import { + FOLDED_WINDOW_REASON, + findFoldedWindowDuplicates, +} from "@/lib/measurements/folded-window"; +import { + recordHealthKitTypeArrivals, + type HealthKitTypeArrival, +} from "@/lib/integrations/healthkit-type-sync"; import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant"; import { classifyExternalId, @@ -86,7 +95,11 @@ import { emitDataArrival } from "@/lib/arrivals/emit-shared"; import { groupRowsByArrivalKind } from "@/lib/arrivals/measurement-kind"; import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; -import { Prisma, type MeasurementType } from "@/generated/prisma/client"; +import { + Prisma, + type MeasurementSource, + type MeasurementType, +} from "@/generated/prisma/client"; // v1.4.25 W16c — historical-backfill threshold for PR push // suppression. A batch larger than this fires the detection job with @@ -530,6 +543,8 @@ async function postBatch(request: NextRequest): Promise { // composite-key duplicate). Surfaced as a dedicated wide-event count so // an operator can see how often the standalone-pair mirror collapses. let crossSourceMergedCount = 0; + // v1.42 — subset of `duplicateCount` refused by the `folded_window` guard. + let foldedWindowCount = 0; // Only rows the reconciler actually inserted drive arrival side effects. const insertedPrepared: Prepared[] = []; const writtenIdentities: Array<{ @@ -621,6 +636,128 @@ async function postBatch(request: NextRequest): Promise { ); } + // v1.42 — exact-duplicate prefilter. A re-sent sample used to cost one + // interactive transaction of about ten round-trips (savepoint, three + // advisory locks, two candidate reads, a row lock) only for the + // reconciler to answer `duplicate` without writing. A backfill after an + // anchor reset re-sends whole 500-entry batches of exactly that. + // + // The reconciler answers `duplicate` for a non-`stats:` row exactly when + // an existing row, live OR tombstoned, has the same + // `(type, source, externalId)` AND the same `measuredAt` and + // `sleepStage`: the natural-identity index is unique NULLS NOT DISTINCT, + // so that row is both its external and its natural match, and with no + // aggregation provenance on a sample nothing else can win. One read over + // the batch's sample ids, without a `deletedAt` filter, answers that for + // every row at once. + // + // The verdict is taken at the very point the reconciler would have been + // called, after `superseded_in_batch` and the cross-source merge, so + // statuses, reasons, counters and the in-batch merge candidates are the + // same as before. A row is only prefiltered when neither its external + // key nor its natural identity occurs a second time in the batch: an + // earlier entry sharing either could move or retire the matched row + // before this one is reached, and then the reconciler has to decide. + // `stats:` rows are never prefiltered (overwrite semantics), and neither + // is a value the reconciler's own plausibility gate would refuse. + const externalKeyCounts = new Map(); + const naturalKeyCounts = new Map(); + const naturalKeyOf = ( + type: unknown, + source: unknown, + measuredAt: Date, + sleepStage: unknown, + ) => `${type}::${source}::${measuredAt.getTime()}::${sleepStage ?? "null"}`; + for (const p of prepared) { + const externalKey = `${p.row.type}::${p.row.source}::${p.row.externalId}`; + externalKeyCounts.set( + externalKey, + (externalKeyCounts.get(externalKey) ?? 0) + 1, + ); + const naturalKey = naturalKeyOf( + p.row.type, + p.row.source, + p.row.measuredAt as Date, + p.row.sleepStage, + ); + naturalKeyCounts.set( + naturalKey, + (naturalKeyCounts.get(naturalKey) ?? 0) + 1, + ); + } + const sampleExternalIds = [ + ...new Set( + prepared + .map((p) => p.row.externalId as string) + .filter((externalId) => !isStatsExternalId(externalId)), + ), + ]; + const existingByExternalKey = new Map< + string, + { measuredAt: Date; sleepStage: unknown } + >(); + if (sampleExternalIds.length > 0) { + const existing = await prisma.measurement.findMany({ + where: { userId: user.id, externalId: { in: sampleExternalIds } }, + select: { + type: true, + source: true, + externalId: true, + measuredAt: true, + sleepStage: true, + }, + }); + for (const row of existing) { + existingByExternalKey.set( + `${row.type}::${row.source}::${row.externalId}`, + { measuredAt: row.measuredAt, sleepStage: row.sleepStage }, + ); + } + } + function isExactStoredDuplicate(p: Prepared): boolean { + const externalKey = `${p.row.type}::${p.row.source}::${p.row.externalId}`; + const stored = existingByExternalKey.get(externalKey); + if (!stored) return false; + const measuredAt = p.row.measuredAt as Date; + const naturalKey = naturalKeyOf( + p.row.type, + p.row.source, + measuredAt, + p.row.sleepStage, + ); + return ( + stored.measuredAt.getTime() === measuredAt.getTime() && + (stored.sleepStage ?? null) === (p.row.sleepStage ?? null) && + externalKeyCounts.get(externalKey) === 1 && + naturalKeyCounts.get(naturalKey) === 1 && + classifyRangeRejection( + p.row.type as MeasurementType, + p.row.value as number, + ) === null + ); + } + + // v1.42 — the `folded_window` guard (`folded-window.ts`). A raw Apple + // Health sample of a dense or mean type, old enough to have been folded, + // whose hour or day a live `stats:` row already covers, is a duplicate: + // the fold no longer leaves the raw rows behind as tombstones, so this is + // what keeps a re-upload after an anchor reset from standing beside the + // mean that already accounts for it. One read for the batch, and none at + // all when no row is old enough. + const foldedIndexes = await findFoldedWindowDuplicates( + prisma, + user.id, + prepared.map((p) => ({ + type: p.row.type as MeasurementType, + source: p.row.source as MeasurementSource, + externalId: p.row.externalId as string, + measuredAt: p.row.measuredAt as Date, + })), + ); + const foldedPrepared = new Set( + [...foldedIndexes].map((index) => prepared[index]), + ); + for (const p of prepared) { const externalId = p.row.externalId as string; const statsRow = isStatsExternalId(externalId); @@ -646,6 +783,32 @@ async function postBatch(request: NextRequest): Promise { continue; } + // Both verdicts below are the reconciler's `duplicate`, reached without + // its transaction; like every duplicate they fall through to the + // in-batch merge candidates. + const prefiltered = !statsRow && isExactStoredDuplicate(p); + const folded = !prefiltered && !statsRow && foldedPrepared.has(p); + if (prefiltered || folded) { + results[p.index] = folded + ? { + index: p.index, + status: "duplicate", + reason: FOLDED_WINDOW_REASON, + } + : { index: p.index, status: "duplicate" }; + duplicateCount++; + if (folded) foldedWindowCount++; + if (isMergeableSource(p.row.source as string)) { + inBatchCandidates.push({ + type: p.row.type as MeasurementType, + source: p.row.source as string, + value: p.row.value as number, + measuredAt: p.row.measuredAt as Date, + }); + } + continue; + } + const verdict = await prisma.$transaction((tx) => reconcileExternalMeasurement( tx, @@ -776,6 +939,34 @@ async function postBatch(request: NextRequest): Promise { const syncedAt = new Date(); const deliveredWithoutTheApp = syncTrigger === "background" || syncTrigger === "push"; + // v1.42 (#1173) — what this sync brought per type, for the per-type + // freshness list on the Apple Health card. Apple Health rows only, and + // under the same conditions as the checkpoint beside it. + const arrivals = new Map(); + for (const p of prepared) { + if (p.row.source !== "APPLE_HEALTH") continue; + const status = results[p.index]?.status; + if ( + status !== "inserted" && + status !== "updated" && + status !== "duplicate" + ) { + continue; + } + const type = p.row.type as MeasurementType; + const entry = arrivals.get(type) ?? { accepted: 0, newSamples: 0 }; + entry.accepted += 1; + if (status !== "duplicate") entry.newSamples += 1; + arrivals.set(type, entry); + } + if (arrivals.size > 0) { + await recordHealthKitTypeArrivals(prisma, { + userId: user.id, + counts: arrivals, + trigger: syncTrigger ?? null, + at: syncedAt, + }); + } await prisma.user.update({ where: { id: user.id }, data: { @@ -920,15 +1111,18 @@ async function postBatch(request: NextRequest): Promise { }); } - // v1.4.25 W16c — kick off PR detection for this user. We always - // enqueue when at least one row was written (or the batch had any - // measurements to consider) so a single off-day reading still gets - // evaluated; the warm-up gate inside the detector decides whether - // it's a record. Suppress push notifications for historical - // backfills above the silent threshold. Updated `stats:*` rows also - // count — a per-day-total overwrite can flip the day's value past a - // personal record. - if (insertedCount > 0 || updatedCount > 0 || duplicateCount > 0) { + // v1.4.25 W16c — kick off PR detection for this user when at least one + // row was written, so a single off-day reading still gets evaluated; the + // warm-up gate inside the detector decides whether it's a record. Suppress + // push notifications for historical backfills above the silent threshold. + // Updated `stats:*` rows also count — a per-day-total overwrite can flip + // the day's value past a personal record. + // + // v1.42 — a batch of duplicates only no longer enqueues. It changed + // nothing a record or a reminder could read, and an anchor-reset backfill + // sends hundreds of them, each of which used to queue a detection pass, a + // reminder sweep and an audit row for no new reading. + if (insertedCount > 0 || updatedCount > 0) { const silent = entries.length > PR_DETECTION_SILENT_THRESHOLD; // v1.18.1 — eventful Vorsorge satisfaction. A matching reading just // landed; resolve the user's reminders now rather than waiting on the @@ -970,9 +1164,13 @@ async function postBatch(request: NextRequest): Promise { skipped_by_reason: skippedByReason, failed: failedCount, // v1.32.8 (iOS #66) — diagnostic sync-trigger tag (foreground / - // background / push), or null when the client did not send one. - // Observability only; it never influenced any count above. + // background / push / manual), or null when the client did not send + // one. It never influenced any count above; since v1.42 it decides + // how hard the caches below are invalidated. syncTrigger: syncTrigger ?? null, + // v1.42 — duplicates refused because a live `stats:` row already + // covers the sample's hour or day (`folded-window.ts`). + folded_window: foldedWindowCount, }, }); @@ -984,12 +1182,26 @@ async function postBatch(request: NextRequest): Promise { // and must invalidate every consumer that reads through this user's // measurements. if (insertedCount > 0 || updatedCount > 0) { - // v1.18.9 (#38) — hard-evict so a focus-refetch after a background - // iOS / Apple-Health batch sync returns post-sync data. A mark-stale - // would let the `cachedSwr` snapshot serve the pre-batch body on the - // very next read, leaving the dashboard up to ~180 s stale; the manual - // measurement routes already pass `{ evict: true }` for the same reason. - invalidateUserMeasurements(user.id, { evict: true }); + // v1.42 — evict only when someone is looking. v1.18.9 (#38) evicted on + // every batch so a focus-refetch after a sync showed post-sync data, but + // a phone syncing in the background posts a batch every few minutes all + // day, and each eviction sent the next dashboard read into a cold + // rebuild. Every other background sync marks stale, which `cachedSwr` + // serves once while one rebuild runs; this route now does the same + // unless the app is open (`foreground`), the person pressed sync + // (`manual`), or the batch wrote one of their own hand-entered readings + // (`MANUAL`), whose entry has to show on the very next read. + const wroteManualRow = prepared.some( + (p) => + p.row.source === "MANUAL" && + (results[p.index]?.status === "inserted" || + results[p.index]?.status === "updated"), + ); + const evict = + syncTrigger === "foreground" || + syncTrigger === "manual" || + wroteManualRow; + invalidateUserMeasurements(user.id, evict ? { evict: true } : undefined); // v1.37.19 (C2-F1) — shared post-mutation tail for every distinct // (type, day) the batch touched. Updates participate as well as diff --git a/src/app/api/sync/changes/route.ts b/src/app/api/sync/changes/route.ts index 810cf3460..43c379a9d 100644 --- a/src/app/api/sync/changes/route.ts +++ b/src/app/api/sync/changes/route.ts @@ -51,6 +51,11 @@ import type { NextRequest } from "next/server"; import { z } from "zod/v4"; import { prisma } from "@/lib/db"; +import { + findCompactionTombstones, + isFoldedWindowCandidate, + loadFoldTimezone, +} from "@/lib/measurements/folded-window"; import { apiHandler, requireAuth } from "@/lib/api-handler"; import { apiError, @@ -416,7 +421,37 @@ export const GET = apiHandler(async (request: NextRequest) => { const mPage = mHasMore ? measurementRows.slice(0, limit) : measurementRows; const measurementUpserts: MeasurementUpsert[] = []; const measurementTombstones: MeasurementTombstone[] = []; + // v1.42 — compaction tombstones are not deletions anyone made. The folds + // used to soft-delete the raw Apple Health samples they folded into an + // hourly or daily `stats:` row, and those rows rode this feed as + // tombstones for 75 days. They are left out now, by the same classifier the + // backlog purge uses (`folded-window.ts`); the cursor still advances past + // them, so the keyset walk is unchanged. A person's own deletions are + // reported exactly as before. + const compaction = new Set(); + const tombstonesOnPage = mPage.filter((row) => row.deletedAt !== null); + if (tombstonesOnPage.length > 0) { + const candidates = tombstonesOnPage.map((row) => ({ + type: row.type, + source: row.source, + externalId: row.externalId, + measuredAt: row.measuredAt, + deletedAt: row.deletedAt, + })); + if ( + candidates.some((row) => isFoldedWindowCandidate(row, row.deletedAt!)) + ) { + const classA = await findCompactionTombstones( + prisma, + user.id, + await loadFoldTimezone(prisma, user.id), + candidates, + ); + for (const index of classA) compaction.add(tombstonesOnPage[index].id); + } + } for (const row of mPage) { + if (compaction.has(row.id)) continue; if (row.deletedAt) { measurementTombstones.push({ id: row.id, diff --git a/src/components/admin/host-metrics-chart.tsx b/src/components/admin/host-metrics-chart.tsx index 8c7f5a742..041d6c762 100644 --- a/src/components/admin/host-metrics-chart.tsx +++ b/src/components/admin/host-metrics-chart.tsx @@ -20,8 +20,9 @@ * `src/components/charts/scatter-correlation-chart.tsx` for the same * pattern + rationale. * - * Polling: TanStack Query with `refetchInterval: 60_000` so the chart - * surfaces a freshly-sampled minute as soon as the worker writes one. + * Polling: TanStack Query with `refetchInterval: 300_000` so the chart + * surfaces a fresh sample as soon as the worker writes one (every five + * minutes since v1.42). */ import { useQuery } from "@tanstack/react-query"; @@ -148,9 +149,9 @@ export function HostMetricsChart() { queryFn: async () => { return apiGet("/api/admin/host-metrics?since=2h"); }, - // 60s matches the sampler cadence — anything faster would just - // re-render the same data. - refetchInterval: 60_000, + // Five minutes matches the sampler cadence (`host-metric-sample`, v1.42) + // — anything faster would just re-render the same data. + refetchInterval: 300_000, refetchOnWindowFocus: false, }); diff --git a/src/lib/__tests__/db.test.ts b/src/lib/__tests__/db.test.ts index 8ed282b60..95f704c5b 100644 --- a/src/lib/__tests__/db.test.ts +++ b/src/lib/__tests__/db.test.ts @@ -108,6 +108,7 @@ describe("getStatementTimeoutMs", () => { describe("buildSessionOptions", () => { const originalEnv = process.env.DATABASE_STATEMENT_TIMEOUT_MS; + const originalWorkMem = process.env.DATABASE_WORK_MEM; afterEach(() => { if (originalEnv === undefined) { @@ -115,6 +116,11 @@ describe("buildSessionOptions", () => { } else { process.env.DATABASE_STATEMENT_TIMEOUT_MS = originalEnv; } + if (originalWorkMem === undefined) { + delete process.env.DATABASE_WORK_MEM; + } else { + process.env.DATABASE_WORK_MEM = originalWorkMem; + } }); it("emits both session timeouts as libpq startup options by default", () => { @@ -124,9 +130,28 @@ describe("buildSessionOptions", () => { expect(opts).toContain("-c idle_in_transaction_session_timeout=60000"); }); - it("returns undefined (no options) when the timeout is disabled", () => { + // v1.42 — work_mem rides the same startup options, and must survive a + // disabled timeout: up to v1.41 the whole string was undefined at 0. + it("keeps work_mem when the timeout is disabled", () => { process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0"; - expect(buildSessionOptions()).toBeUndefined(); + delete process.env.DATABASE_WORK_MEM; + expect(buildSessionOptions()).toBe("-c work_mem=16MB"); + }); + + it("sets the default work_mem beside the timeouts", () => { + delete process.env.DATABASE_STATEMENT_TIMEOUT_MS; + delete process.env.DATABASE_WORK_MEM; + expect(buildSessionOptions()).toBe( + "-c statement_timeout=60000 -c idle_in_transaction_session_timeout=60000 -c work_mem=16MB", + ); + }); + + it("takes DATABASE_WORK_MEM and refuses a value that would inject a flag", () => { + process.env.DATABASE_WORK_MEM = "64MB"; + expect(buildSessionOptions()).toContain("-c work_mem=64MB"); + process.env.DATABASE_WORK_MEM = "8MB -c statement_timeout=0"; + expect(buildSessionOptions()).toContain("-c work_mem=16MB"); + expect(buildSessionOptions()).toContain("statement_timeout=60000"); }); }); diff --git a/src/lib/cache/__tests__/invalidate.test.ts b/src/lib/cache/__tests__/invalidate.test.ts index fbe65fe56..476145139 100644 --- a/src/lib/cache/__tests__/invalidate.test.ts +++ b/src/lib/cache/__tests__/invalidate.test.ts @@ -197,6 +197,24 @@ describe("invalidateUserMeasurements", () => { expect(caches.achievements.get(USER_A)).toBeNull(); expect(caches.workouts.get(`${USER_A}|3|0||`)).toBeNull(); }); + + // v1.42 — the targets bucket follows the same split. Its route reads + // through `cachedSwr`, so a background sync marks it stale instead of + // throwing it away on every batch. + it("marks the targets bucket stale by default and evicts it with { evict: true }", async () => { + await cached(caches.insightsTargets, USER_A, async () => ({ t: 1 })); + await cached(caches.insightsTargets, USER_B, async () => ({ t: 2 })); + invalidateUserMeasurements(USER_A); + expect(caches.insightsTargets.getAllowStale(USER_A)).toEqual({ + value: { t: 1 }, + stale: true, + }); + expect(caches.insightsTargets.get(USER_B)).not.toBeNull(); + + invalidateUserMeasurements(USER_A, { evict: true }); + expect(caches.insightsTargets.getAllowStale(USER_A)).toBeNull(); + expect(caches.insightsTargets.get(USER_B)).not.toBeNull(); + }); }); describe("invalidateUserMood", () => { diff --git a/src/lib/cache/invalidate.ts b/src/lib/cache/invalidate.ts index 40157bb1b..bb16d8dd3 100644 --- a/src/lib/cache/invalidate.ts +++ b/src/lib/cache/invalidate.ts @@ -65,6 +65,12 @@ export function invalidateUserDashboardSnapshot(userId: string): void { * posture. Background batch syncs (Apple Health, Withings, workouts) * keep the default mark-stale so a high-frequency sync never busts the * bucket into inline cold-rebuild storms. + * + * v1.42 — the same split now covers `insightsTargets`, and the Apple Health + * batch route follows it: it evicts only for a `foreground` or `manual` + * sync or a batch that wrote a hand-entered (`MANUAL`) reading, and marks + * stale otherwise. The achievements and workouts buckets still evict on + * every call. */ export function invalidateUserMeasurements( userId: string, @@ -94,9 +100,17 @@ export function invalidateUserMeasurements( caches.workouts.deleteByPrefix(`${userId}|`); // v1.4.36 W1 — measurement writes change the per-target consistency // strips, the in-range rates and the streak counters that - // `/api/insights/targets` computes. Evict the user's bucket so the - // next mount paints fresh data. - caches.insightsTargets.deleteByPrefix(userId); + // `/api/insights/targets` computes. + // + // v1.42 — same split as the analytics bucket. The route reads through + // `cachedSwr`, so a background sync marks the bucket stale (the next read + // serves the prior strips once while one rebuild runs) instead of + // throwing it away on every batch; an interactive write still evicts. + if (opts?.evict) { + caches.insightsTargets.deleteByPrefix(userId); + } else { + caches.insightsTargets.markStaleByPrefix(userId); + } // v1.16.8 — the batched derived-wellness payload computes over the // same measurement rollups. Same split as the analytics bucket: // interactive writes evict (the user's own entry must reflect on the diff --git a/src/lib/db.ts b/src/lib/db.ts index 86b45c035..5d9cfbf48 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -83,25 +83,56 @@ export function getStatementTimeoutMs(): number { return 60_000; } +/** `work_mem` when `DATABASE_WORK_MEM` is unset. */ +export const DEFAULT_WORK_MEM = "16MB"; + /** - * Build the libpq `options` startup string applying the session timeouts, or - * `undefined` when disabled (timeout 0). Passed straight through `PrismaPg` to - * the underlying `pg.Pool`, which forwards it as the connection's `options` - * startup parameter so every session is timeout-bounded from the first query. + * A Postgres memory size: digits and an optional unit. The value is spliced + * into the libpq `options` string, where a space would start another `-c` + * flag, so anything else is refused and the default used. */ -export function buildSessionOptions(): string | undefined { +const WORK_MEM_SHAPE = /^[1-9][0-9]{0,6}(kB|MB|GB)?$/; + +/** + * Per-session `work_mem` (v1.42), from `DATABASE_WORK_MEM`. + * + * The server default of 4 MB made the larger sorts and hash aggregates of the + * analytics reads spill to disk. `work_mem` is a USERSET setting, so it rides + * the same startup `options` as the timeouts and applies to this app's + * connections only, without a database restart. Each sort or hash step of + * each connection may use this much, so raise it with the pool size in mind. + * An unparsable value falls back to the default rather than reaching libpq. + */ +export function getWorkMem(): string { + const raw = process.env.DATABASE_WORK_MEM?.trim(); + return raw && WORK_MEM_SHAPE.test(raw) ? raw : DEFAULT_WORK_MEM; +} + +/** + * Build the libpq `options` startup string applying the session settings. + * Passed straight through `PrismaPg` to the underlying `pg.Pool`, which + * forwards it as the connection's `options` startup parameter so every + * session carries them from the first query. + * + * The timeouts are left out when disabled (timeout 0); `work_mem` is always + * set. Up to v1.41 the whole string was `undefined` at timeout 0, which would + * have taken `work_mem` with it. + */ +export function buildSessionOptions(): string { const timeoutMs = getStatementTimeoutMs(); - if (timeoutMs <= 0) return undefined; - return `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs}`; + const timeouts = + timeoutMs > 0 + ? `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs} ` + : ""; + return `${timeouts}-c work_mem=${getWorkMem()}`; } function createPrismaClient() { - const sessionOptions = buildSessionOptions(); const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL!, max: getPrismaPoolMax(), connectionTimeoutMillis: getPoolConnectionTimeoutMs(), - ...(sessionOptions ? { options: sessionOptions } : {}), + options: buildSessionOptions(), }); return new PrismaClient({ adapter }); } diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index dbf305380..9dc633e95 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -122,6 +122,12 @@ import { } from "@/lib/medication-category"; import { invalidateUserData } from "@/lib/cache/invalidate"; import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; +import { + FOLDED_TYPES, + coveringStatsExternalId, + isFoldedWindowCandidate, +} from "@/lib/measurements/folded-window"; +import { resolveUserTimezone } from "@/lib/measurements/consolidation-base"; import { stampSyncReset } from "@/lib/sync/reset"; import { classifyRestoreFailure, @@ -418,14 +424,30 @@ export async function restoreBackup( // The measurements are the exception: `readStreamedBackup` checks each one // against the element schema as it goes and keeps only their count, and // the transaction reads them a second time, in batches, as it writes them. + // v1.42 — the live `stats:` ids of the folded types, gathered on the first + // read, so the measurement pass can tell a compaction tombstone from a + // person's deletion without a third read of the file. + const liveFoldStatsIds = new Set(); + const foldedTypes = new Set(FOLDED_TYPES); + const noteLiveFoldRow = (row: BackupMeasurement) => { + if ( + row.deletedAt == null && + row.source === "APPLE_HEALTH" && + foldedTypes.has(row.type) && + row.externalId?.startsWith("stats:") + ) { + liveFoldStatsIds.add(`${row.type}|${row.externalId}`); + } + }; let raw: unknown; let payload; let streamed: StreamedBackup; try { streamed = await readStreamedBackup(source, { - onMeasurementChecked: (checked) => { + onMeasurementChecked: (checked, row) => { progress.measurementsChecked = checked; report("validating"); + noteLiveFoldRow(row); }, }); progress.measurementsTotal = streamed.measurementCount; @@ -986,9 +1008,48 @@ export async function restoreBackup( // row. const tombstoneHorizonMs = Date.now() - TOMBSTONE_RETENTION_DAYS * 86_400_000; + // + // v1.42 — nor is a compaction tombstone, whatever its age: a raw + // Apple Health sample the folds soft-deleted before v1.42, whose hour + // or day a live `stats:` row in the same file covers. The folds + // delete those outright now and the backlog purge removes the old + // ones, so writing one back would only hand the purge the same row + // again; the `folded_window` ingest guard is what keeps the sample + // from being uploaded again. Classified by the rule the purge and + // the sync feed use (`folded-window.ts`), against the `stats:` ids + // gathered on the file's first read and the zone the account will + // have after the restore. Counted with the expired ones. + const foldTz = resolveUserTimezone( + payload.accountSettings?.timezone ?? + ( + await tx.user.findUnique({ + where: { id: ownerId }, + select: { timezone: true }, + }) + )?.timezone ?? + null, + ); + const isCompactionTombstone = (measurement: BackupMeasurement) => { + if (measurement.deletedAt == null) return false; + const row = { + type: measurement.type, + source: measurement.source ?? "MANUAL", + externalId: measurement.externalId ?? null, + measuredAt: new Date(measurement.measuredAt), + }; + if (!isFoldedWindowCandidate(row, new Date(measurement.deletedAt))) { + return false; + } + const covering = coveringStatsExternalId(row, foldTz); + return ( + covering !== null && + liveFoldStatsIds.has(`${measurement.type}|${covering}`) + ); + }; const isExpiredTombstone = (measurement: BackupMeasurement) => measurement.deletedAt != null && - new Date(measurement.deletedAt).getTime() < tombstoneHorizonMs; + (new Date(measurement.deletedAt).getTime() < tombstoneHorizonMs || + isCompactionTombstone(measurement)); let expiredTombstonesSkipped = 0; report("measurements"); await streamed.forEachMeasurementBatch( diff --git a/src/lib/export/streamed-backup.ts b/src/lib/export/streamed-backup.ts index 15bece054..bdcd6dccf 100644 --- a/src/lib/export/streamed-backup.ts +++ b/src/lib/export/streamed-backup.ts @@ -104,9 +104,11 @@ function parseMeasurement( export interface ReadStreamedBackupOptions { /** * Called after each measurement passes the element schema, with how many - * have so far. For a caller that reports progress; keep it cheap. + * have so far and the parsed row. For a caller that reports progress or + * gathers a small index over the rows (the restore collects the live + * `stats:` ids it classifies compaction tombstones against); keep it cheap. */ - onMeasurementChecked?: (checked: number) => void; + onMeasurementChecked?: (checked: number, row: BackupMeasurement) => void; } export async function readStreamedBackup( @@ -129,7 +131,7 @@ export async function readStreamedBackup( const row = parseMeasurement(schema, element, index); if (!row.id && firstWithoutId === null) firstWithoutId = index; keys.visit(element, MEASUREMENTS); - options.onMeasurementChecked?.(index + 1); + options.onMeasurementChecked?.(index + 1, row); }, }); diff --git a/src/lib/insights/__tests__/invalidate-status-insights.test.ts b/src/lib/insights/__tests__/invalidate-status-insights.test.ts index 3a9e39463..e47f34ee0 100644 --- a/src/lib/insights/__tests__/invalidate-status-insights.test.ts +++ b/src/lib/insights/__tests__/invalidate-status-insights.test.ts @@ -20,7 +20,18 @@ const aiCapabilityForRecord = vi.fn(); const enqueueStatusGeneration = vi.fn(); const userFindUnique = vi.fn(); const measurementFindMany = vi.fn(); -const measurementGroupBy = vi.fn(); +// v1.42 — type discovery goes through the loose-index-scan helper; the +// rows it answers with are the types themselves. +const listLiveTypes = vi.fn(); +const measurementGroupBy = { + mockResolvedValue: (rows: Array<{ type: string }>) => + listLiveTypes.mockResolvedValue(rows.map((row) => row.type)), + mockRejectedValue: (err: Error) => listLiveTypes.mockRejectedValue(err), +}; + +vi.mock("@/lib/measurements/live-types", () => ({ + listLiveMeasurementTypes: (...a: unknown[]) => listLiveTypes(...a), +})); vi.mock("@/lib/db", () => ({ prisma: { @@ -31,9 +42,6 @@ vi.mock("@/lib/db", () => ({ user: { findUnique: (...a: unknown[]) => userFindUnique(...a) }, measurement: { findMany: (...a: unknown[]) => measurementFindMany(...a), - // v1.28.25 — type discovery runs as a server-side GROUP BY, not a - // client-deduped `distinct` findMany. - groupBy: (...a: unknown[]) => measurementGroupBy(...a), }, }, })); @@ -361,7 +369,7 @@ describe("enqueueStatusRefillForUser", () => { const count = await enqueueStatusRefillForUser("u1", "de"); expect(count).toBe(0); expect(enqueueStatusGeneration).not.toHaveBeenCalled(); - expect(measurementGroupBy).not.toHaveBeenCalled(); + expect(listLiveTypes).not.toHaveBeenCalled(); }); it("still refills the specialised scopes when the generic-scope discovery read fails", async () => { diff --git a/src/lib/insights/status-invalidation.ts b/src/lib/insights/status-invalidation.ts index 373e50df7..b12d59023 100644 --- a/src/lib/insights/status-invalidation.ts +++ b/src/lib/insights/status-invalidation.ts @@ -29,6 +29,7 @@ import { metricStatusScope, } from "@/lib/insights/metric-status-registry"; import { annotate } from "@/lib/logging/context"; +import { listLiveMeasurementTypes } from "@/lib/measurements/live-types"; import { aiCapabilityForRecord } from "@/lib/ai/capabilities/gate"; import type { MeasurementType } from "@/generated/prisma/client"; @@ -248,16 +249,12 @@ export async function enqueueStatusRefillForUser( } const scopes = new Set(PER_STATUS_SCOPES); try { - // `groupBy` compiles to a server-side `GROUP BY` — Prisma's - // `distinct` dedups in the client AFTER pulling every live row, which - // on a dense multi-year account walks a six-figure row set to answer - // "which types exist?" on the request path. - const rows = await prisma.measurement.groupBy({ - by: ["type"], - where: { userId, deletedAt: null }, - }); - for (const row of rows) { - const metricId = metricIdForMeasurementType(row.type); + // v1.42 — a loose index scan over the live-rows index (`live-types.ts`) + // rather than a `GROUP BY` that reads every live row of a dense + // multi-year account to answer "which types exist?". + const types = await listLiveMeasurementTypes(userId); + for (const type of types) { + const metricId = metricIdForMeasurementType(type); if (metricId) scopes.add(metricStatusScope(metricId)); } } catch { diff --git a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts index 84de68b99..f27b28739 100644 --- a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts +++ b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts @@ -57,7 +57,12 @@ describe("cleanupExpiredMeasurementTombstones", () => { const { prisma } = mockClient({ findMany, deleteMany }); const outcome = await cleanupExpiredMeasurementTombstones(prisma, now); - expect(outcome).toEqual({ deleted: 3, drained: true, deferredAccounts: 0 }); + expect(outcome).toEqual({ + deleted: 3, + drained: true, + deferredAccounts: 0, + deferredUserIds: [], + }); expect(findMany).toHaveBeenCalledTimes(1); const where = findMany.mock.calls[0][0].where as { @@ -118,7 +123,12 @@ describe("cleanupExpiredMeasurementTombstones — accounts under restore", () => const outcome = await cleanupExpiredMeasurementTombstones(prisma); - expect(outcome).toEqual({ deleted: 3, drained: true, deferredAccounts: 0 }); + expect(outcome).toEqual({ + deleted: 3, + drained: true, + deferredAccounts: 0, + deferredUserIds: [], + }); expect(lockCalls).toEqual(["a", "b"]); expect(deleteMany.mock.calls.map((call) => call[0].where)).toEqual([ { id: { in: ["a1", "a2"] } }, @@ -146,7 +156,12 @@ describe("cleanupExpiredMeasurementTombstones — accounts under restore", () => const outcome = await cleanupExpiredMeasurementTombstones(prisma); - expect(outcome).toEqual({ deleted: 1, drained: true, deferredAccounts: 1 }); + expect(outcome).toEqual({ + deleted: 1, + drained: true, + deferredAccounts: 1, + deferredUserIds: ["restoring"], + }); expect(findMany.mock.calls[0][0].where).not.toHaveProperty("userId"); expect(findMany.mock.calls[1][0].where.userId).toEqual({ notIn: ["restoring"], diff --git a/src/lib/jobs/compaction-tombstone-purge.ts b/src/lib/jobs/compaction-tombstone-purge.ts index 764a0637e..3c9d17fa7 100644 --- a/src/lib/jobs/compaction-tombstone-purge.ts +++ b/src/lib/jobs/compaction-tombstone-purge.ts @@ -1,38 +1,337 @@ /** * Backlog purge of compaction tombstones (v1.42). * - * The dense-intraday consolidation used to soft-delete the raw rows it folded - * into an hourly or daily `stats:` row. Those tombstones ("class A") never - * reach a client as a meaningful deletion and only weigh on the - * `measurements` table and its indexes. This queue removes the backlog in - * batches of 5 000 rows, each batch under `holdAccountAgainstRestore`; the - * delete predicate is idempotent, so a run that stops early simply resumes on - * the next one. User deletions ("class B") are never touched here and keep - * their 75-day retention. + * The dense-intraday retention and the daily-mean consolidation used to + * soft-delete the raw rows they folded into an hourly or daily `stats:` row. + * Those tombstones ("class A") never reached a client as a meaningful + * deletion; they held one thing, a re-upload of a folded sample, which the + * `folded_window` ingest guard now does from the live `stats:` row. They made + * up most of the `measurements` table and its indexes. Since v1.42 the folds + * delete outright; this queue removes what they left behind before. * - * Contract stub: the queue is registered and bound so the registrar, the - * runtime table and the subsystem surface are settled. The handler refuses to - * claim success until the purge is implemented, and the boot enqueue sends - * nothing yet. + * Which tombstone is class A is decided by `findCompactionTombstones` + * (`folded-window.ts`), the same classifier the sync feed and the restore use: + * a soft-deleted raw Apple Health sample of a folded type, already past the + * fold threshold when it was deleted, whose hour or day a live `stats:` row + * covers. Everything else ("class B": a person's deletions, collision + * retirements, a dense day still at the pre-hourly daily grain that the + * one-shot rebuild reads) keeps the 75-day retention of + * `measurement-tombstone-cleanup.ts` and is never touched here. + * + * Shape of a run: + * - The tombstones are walked in `(deleted_at, id)` order over the partial + * `measurements_tombstone_purge_idx`, narrowed to Apple Health rows of + * the folded types, and classified a page at a time with one lookup per + * account and page. + * - The class-A rows are deleted 5 000 at a time through + * `purgeTombstonesByAccount`: one transaction per account per batch, + * each starting with `holdAccountAgainstRestore`, so an account under + * restore is skipped for the rest of the run and picked up by the next. + * - A short pause separates two batches, and at most forty batches run per + * job, inside three quarters of the job's expiry. + * - The delete predicate is idempotent, so nothing records progress: a run + * that stopped early sends a follow-up a minute later, at most + * {@link MAX_CONTINUATIONS} in a row and only when it deleted something, + * and every boot queues one run. + * - Every run reports its outcome per account (`reportJobRun`) and its + * totals as job facts. */ -import { jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { createHash } from "node:crypto"; +import type { Job } from "pg-boss"; + +import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { reportJobRun, type JobRunCandidate } from "@/lib/jobs/job-run-report"; +import { purgeTombstonesByAccount } from "@/lib/jobs/measurement-tombstone-cleanup"; +import { PURGE_BATCH_SIZE, PURGE_MAX_BATCHES } from "@/lib/jobs/purge-batch"; +import { withBackgroundEvent } from "@/lib/logging/background"; +import { resolveUserTimezone } from "@/lib/measurements/consolidation-base"; +import { + FOLDED_TYPES, + findCompactionTombstones, +} from "@/lib/measurements/folded-window"; +import { prisma as defaultPrisma } from "@/lib/db"; export const COMPACTION_TOMBSTONE_PURGE_QUEUE = "compaction-tombstone-purge"; -/** Empty payload: one run walks every account. */ -export type CompactionTombstonePurgePayload = Record; +/** One key for every send, so a queued run absorbs a second boot's send. */ +export const COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY = + "compaction-tombstone-purge"; -export async function handleCompactionTombstonePurge(): Promise { - return jobFailed("compaction tombstone purge not implemented"); +/** Follow-up runs one chain may send: forty runs of up to 200 000 rows. */ +export const MAX_CONTINUATIONS = 40; + +/** Delay before a follow-up run, in seconds. */ +export const CONTINUATION_DELAY_SECONDS = 60; + +/** Pause between two delete batches, in milliseconds. */ +export const BATCH_PAUSE_MS = 150; + +/** `continuation` is 0 (or absent) for the boot's run. */ +export interface CompactionTombstonePurgePayload { + continuation?: number; +} + +export interface CompactionTombstonePurgeOutcome { + deleted: number; + /** False when the run stopped at the batch cap or its time budget. */ + drained: boolean; + /** Accounts skipped for this run because a restore of them was running. */ + deferredAccounts: number; + /** Tombstones of the folded types the run looked at. */ + scanned: number; + /** The deferred accounts' ids. Never leaves the process unhashed. */ + deferredUserIds: string[]; +} + +export interface PurgeCompactionTombstonesOptions { + batchSize?: number; + maxBatches?: number; + /** Tombstones read per classification page. */ + scanPageSize?: number; + pauseMs?: number; + shouldStop?: () => boolean; +} + +type ScannedRow = { + id: string; + userId: string; + type: MeasurementType; + externalId: string | null; + measuredAt: Date; + deletedAt: Date | null; +}; + +/** + * Delete the class-A tombstones, in batches, one account at a time under the + * restore lock. See the module comment for the shape of a run. + */ +export async function purgeCompactionTombstones( + prisma: PrismaClient, + options: PurgeCompactionTombstonesOptions = {}, +): Promise { + const scanPageSize = options.scanPageSize ?? PURGE_BATCH_SIZE; + const pauseMs = options.pauseMs ?? BATCH_PAUSE_MS; + const timezoneOf = new Map(); + let scanned = 0; + + // The walk's position. Rows behind it are classified; the class-A ones wait + // in `ready` until a batch takes them. + let cursor: { deletedAt: Date; id: string } | null = null; + let exhausted = false; + let ready: Array<{ id: string; userId: string }> = []; + + async function timezoneFor(userIds: string[]): Promise { + const missing = userIds.filter((id) => !timezoneOf.has(id)); + if (missing.length === 0) return; + const users = await prisma.user.findMany({ + where: { id: { in: missing } }, + select: { id: true, timezone: true }, + }); + for (const user of users) { + timezoneOf.set(user.id, resolveUserTimezone(user.timezone)); + } + } + + async function classifyNextPage(skipUserIds: ReadonlySet) { + const page: ScannedRow[] = await prisma.measurement.findMany({ + where: { + deletedAt: { not: null }, + source: "APPLE_HEALTH", + type: { in: [...FOLDED_TYPES] }, + NOT: { externalId: { startsWith: "stats:" } }, + ...(cursor + ? { + OR: [ + { deletedAt: { gt: cursor.deletedAt } }, + { deletedAt: cursor.deletedAt, id: { gt: cursor.id } }, + ], + } + : {}), + }, + orderBy: [{ deletedAt: "asc" }, { id: "asc" }], + take: scanPageSize, + select: { + id: true, + userId: true, + type: true, + externalId: true, + measuredAt: true, + deletedAt: true, + }, + }); + scanned += page.length; + if (page.length < scanPageSize) exhausted = true; + const last = page[page.length - 1]; + if (last?.deletedAt) cursor = { deletedAt: last.deletedAt, id: last.id }; + + const byAccount = new Map(); + for (const row of page) { + if (skipUserIds.has(row.userId)) continue; + const rows = byAccount.get(row.userId); + if (rows) rows.push(row); + else byAccount.set(row.userId, [row]); + } + await timezoneFor([...byAccount.keys()]); + for (const [userId, rows] of byAccount) { + const tz = timezoneOf.get(userId); + // An account deleted since the page was read has nothing left to purge. + if (!tz) continue; + const classA = await findCompactionTombstones( + prisma, + userId, + tz, + rows.map((row) => ({ + type: row.type, + source: "APPLE_HEALTH" as const, + externalId: row.externalId, + measuredAt: row.measuredAt, + deletedAt: row.deletedAt, + })), + ); + for (const index of classA) ready.push({ id: rows[index].id, userId }); + } + } + + const outcome = await purgeTombstonesByAccount({ + prisma, + batchSize: options.batchSize ?? PURGE_BATCH_SIZE, + maxBatches: options.maxBatches ?? PURGE_MAX_BATCHES, + shouldStop: options.shouldStop, + betweenBatches: + pauseMs > 0 + ? () => new Promise((resolve) => setTimeout(resolve, pauseMs)) + : undefined, + findRows: async (take, skipUserIds) => { + const skip = new Set(skipUserIds); + ready = ready.filter((row) => !skip.has(row.userId)); + while (ready.length < take && !exhausted) { + if (options.shouldStop?.()) break; + await classifyNextPage(skip); + } + return ready.splice(0, take); + }, + deleteIds: async (tx, ids) => { + // Re-checked at delete time: a row a restore wrote back, or one a + // person resurrected since the scan, is not a tombstone any more. + const result = await tx.measurement.deleteMany({ + where: { + id: { in: ids }, + deletedAt: { not: null }, + source: "APPLE_HEALTH", + }, + }); + return result.count; + }, + }); + + return { + deleted: outcome.deleted, + // A run the time budget stopped mid-scan has not seen every tombstone. + drained: outcome.drained && exhausted, + deferredAccounts: outcome.deferredAccounts, + scanned, + deferredUserIds: outcome.deferredUserIds, + }; +} + +/** A stable, non-identifying key for one account in the run report. */ +function accountKey(userId: string): string { + return createHash("sha256").update(userId).digest("hex").slice(0, 12); } /** - * Queue one purge run when the worker boots. The run is self-limiting, so a - * boot that finds no backlog costs one cheap query. Sends nothing until the - * purge is implemented. + * One purge run. The run's totals go out as job facts and on the run's wide + * event; the run report lists the accounts a restore made it skip, by hashed + * key, so a deferral that repeats run after run is visible. + */ +export async function handleCompactionTombstonePurge( + jobs: Job[], + prisma: PrismaClient = defaultPrisma, +): Promise { + const shouldStop = jobBudget(jobs); + const continuation = Math.max( + 0, + ...jobs.map((job) => job.data?.continuation ?? 0), + ); + return withBackgroundEvent("job.compaction_tombstone_purge", async (evt) => { + const startedAt = Date.now(); + const result = await purgeCompactionTombstones(prisma, { shouldStop }); + const candidates: JobRunCandidate[] = result.deferredUserIds.map( + (userId) => ({ + key: accountKey(userId), + outcome: "deferred", + cause: "restore_in_progress", + }), + ); + reportJobRun({ + queue: COMPACTION_TOMBSTONE_PURGE_QUEUE, + runId: jobs[0]?.id ?? "unknown", + candidates, + }); + evt.addMeta("compaction_purge_deleted", result.deleted); + evt.addMeta("compaction_purge_scanned", result.scanned); + evt.addMeta("compaction_purge_drained", result.drained); + evt.addMeta("compaction_purge_deferred_accounts", result.deferredAccounts); + evt.addMeta("compaction_purge_continuation", continuation); + + // A run that stopped with work left sends the next one, but only when it + // made progress, so a run that cannot delete anything cannot loop. + let continued = false; + if ( + !result.drained && + result.deleted > 0 && + continuation < MAX_CONTINUATIONS + ) { + const boss = getGlobalBoss(); + if (boss) { + await boss.send( + COMPACTION_TOMBSTONE_PURGE_QUEUE, + { continuation: continuation + 1 }, + { + singletonKey: COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY, + startAfter: CONTINUATION_DELAY_SECONDS, + retryLimit: 0, + }, + ); + continued = true; + } + } else if (!result.drained) { + evt.addWarning( + "compaction-tombstone-purge stopped with a backlog and sent no follow-up; the next boot resumes it", + ); + } + + return jobDone({ + deleted: result.deleted, + candidates_scanned: result.scanned, + drained: result.drained, + deferred_accounts: result.deferredAccounts, + continuation, + continued, + duration_ms: Date.now() - startedAt, + }); + }); +} + +/** + * Queue one purge run when the worker boots. The run is self-limiting and + * idempotent, so a boot that finds no backlog costs one cheap scan. */ export async function enqueueBootTimeCompactionTombstonePurge(): Promise<{ enqueued: boolean; }> { - return { enqueued: false }; + const boss = getGlobalBoss(); + if (!boss) return { enqueued: false }; + const id = await boss.send( + COMPACTION_TOMBSTONE_PURGE_QUEUE, + { continuation: 0 }, + { + singletonKey: COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY, + retryLimit: 0, + }, + ); + return { enqueued: id !== null }; } diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index 30eb96b14..a0c52a11a 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -5,8 +5,8 @@ * Modelled on `mean-consolidation.ts`: a discovery query enqueues one job * per user still holding live per-sample dense-tier rows OLDER than the * retention window, the per-user handler runs `runDenseIntradayRetention`, - * and the pass is idempotent across reboots (folded rows are soft-deleted - * so they drop off the discovery list). + * and the pass is idempotent across reboots (folded rows are deleted, before + * v1.42 soft-deleted, so they drop off the discovery list). * * Unlike the daily-mean consolidation, this pass scopes to the dense-tier * types (`HEART_RATE_VARIABILITY`, `PULSE`, `OXYGEN_SATURATION`) and keeps @@ -147,7 +147,8 @@ export async function runDenseIntradayRetentionForUser( * retention job per account. * * Idempotent across reboots: once a user's out-of-window per-sample rows - * are soft-deleted, the predicate drops them from the discovery set. + * are folded (deleted since v1.42), the predicate drops them from the + * discovery set. * pg-boss `singletonKey` coalesces duplicate sends. Best-effort: errors * are returned through the result value so worker boot never fails because * of a retention miss. diff --git a/src/lib/jobs/host-metric-sampler.ts b/src/lib/jobs/host-metric-sampler.ts index 2993b2db7..9a93f63cc 100644 --- a/src/lib/jobs/host-metric-sampler.ts +++ b/src/lib/jobs/host-metric-sampler.ts @@ -1,5 +1,5 @@ /** - * Per-minute host-load sampler that backs the admin /admin/system-status + * Host-load sampler (every five minutes since v1.42) that backs the admin /admin/system-status * "last 2 hours" chart (v1.4.16 phase B3). * * The maintainer asked for a small at-a-glance graph above the system-status facts @@ -8,7 +8,7 @@ * Sentinel collector, but the Coolify HTTP API does not expose a clean * read endpoint for per-server metrics — every approach ended up either * scraping the Coolify dashboard HTML or hitting a private RPC. An - * in-process sampler costs us a single 7-column row per minute, never + * in-process sampler costs us a single 7-column row per tick, never * leaves the app's own DB, and works identically on dev (macOS) and * production (Linux). * @@ -19,8 +19,8 @@ * of failing the whole chart. * * Retention: rows older than 7 days are deleted at the end of every - * tick. With one insert per minute that's 10,080 rows steady-state, - * indexed by `captured_at` so the 2h query stays under 200 rows. + * tick. With one insert every five minutes that's 2,016 rows steady-state, + * indexed by `captured_at` so the 2h query stays under 30 rows. */ import { loadavg, totalmem, freemem } from "node:os"; import { readFile } from "node:fs/promises"; diff --git a/src/lib/jobs/mean-consolidation.ts b/src/lib/jobs/mean-consolidation.ts index bbe5ab219..7c787755a 100644 --- a/src/lib/jobs/mean-consolidation.ts +++ b/src/lib/jobs/mean-consolidation.ts @@ -6,8 +6,8 @@ * Modelled on `step-consolidation.ts`: a discovery query enqueues one * job per user still holding live per-sample mean-type rows, the * per-user handler runs `consolidateDailyMean`, and the pass is - * idempotent across reboots (consolidated rows are soft-deleted so they - * drop off the discovery list). + * idempotent across reboots (consolidated rows are deleted, before v1.42 + * soft-deleted, so they drop off the discovery list). * * The queue name MUST be registered in `allQueues` in * `src/lib/jobs/reminder-worker.ts` so pg-boss provisions it at boot; @@ -84,8 +84,8 @@ export async function runMeanConsolidationForUser( * enqueues one consolidation job per account. * * Idempotent across reboots: once a user's per-sample rows are - * soft-deleted, the `deleted_at IS NULL` predicate drops them from the - * discovery set. pg-boss `singletonKey` coalesces duplicate sends. + * consolidated (deleted since v1.42), the `deleted_at IS NULL` predicate + * drops them from the discovery set. pg-boss `singletonKey` coalesces duplicate sends. * * Best-effort: errors are returned through the result value so the * worker boot never fails because of a consolidation miss. diff --git a/src/lib/jobs/measurement-maintenance.ts b/src/lib/jobs/measurement-maintenance.ts index 5feb29ff5..d76049dda 100644 --- a/src/lib/jobs/measurement-maintenance.ts +++ b/src/lib/jobs/measurement-maintenance.ts @@ -8,13 +8,44 @@ * (cookie-only admin), never on a schedule: the operator picks a quiet window. * Runbook: `docs/ops/measurement-maintenance.md`. * - * Contract stub: the queue is registered and bound; the handler refuses to - * claim success until the maintenance pass is implemented. + * Why the indexes: after the purge the heap is small, but a B-tree does not + * give back the pages its deleted entries held. VACUUM marks the heap space + * reusable and refreshes the planner statistics; only a rebuild returns the + * index space. `CONCURRENTLY` keeps reads and writes going during the + * rebuild; the price is a second pass over the table and, for a while, the + * disk for one more copy of the index being rebuilt. + * + * Safeguards: + * - One run at a time: the queue is `exclusive` on a fixed singleton key, + * and the run holds a session advisory lock besides (`withJobLock`), so a + * delivery that outlived its expiry never runs beside its retry. + * - It refuses to start while the compaction-tombstone purge is queued or + * running: rebuilding indexes under a bulk delete rebuilds them twice. + * - It runs on a connection of its own with no statement timeout (the app + * pool's 60 s would kill the first REINDEX), a session-only + * `maintenance_work_mem`, and a lock timeout, so a rebuild that cannot get + * its lock fails instead of queueing writers behind it. + * - One index at a time, largest first. A failed rebuild leaves an invalid + * `_ccnew` copy behind; it is dropped at once, and the pass stops + * there rather than trying the next index on a disk that just ran out. + * Leftover invalid copies from an earlier, interrupted run are dropped + * before the first rebuild. + * - Every step's size before and after, and its duration, are on the run's + * wide event; the totals are job facts. */ -import { jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { Client } from "pg"; +import type { Job } from "pg-boss"; + +import { withJobLock } from "@/lib/jobs/job-lock"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { reportJobRun, type JobRunCandidate } from "@/lib/jobs/job-run-report"; +import { withBackgroundEvent } from "@/lib/logging/background"; export const MEASUREMENT_MAINTENANCE_QUEUE = "measurement-maintenance"; +/** The singleton key every send carries (the queue is `exclusive`). */ +export const MEASUREMENT_MAINTENANCE_SINGLETON_KEY = "measurement-maintenance"; + /** * A REINDEX of the largest index on a big instance takes well past pg-boss's * default fifteen minutes. One-shot: a retry would start the pass over beside @@ -23,7 +54,7 @@ export const MEASUREMENT_MAINTENANCE_QUEUE = "measurement-maintenance"; export const MEASUREMENT_MAINTENANCE_SEND_OPTIONS = { retryLimit: 0, expireInSeconds: 6 * 60 * 60, - singletonKey: "measurement-maintenance", + singletonKey: MEASUREMENT_MAINTENANCE_SINGLETON_KEY, } as const; /** Which steps the operator asked for. */ @@ -32,6 +63,326 @@ export interface MeasurementMaintenancePayload { reindex: boolean; } -export async function handleMeasurementMaintenance(): Promise { - return jobFailed("measurement maintenance not implemented"); +/** Session settings of the maintenance connection. */ +export const MAINTENANCE_SESSION_OPTIONS = + "-c statement_timeout=0 -c idle_in_transaction_session_timeout=0 -c maintenance_work_mem=128MB -c lock_timeout=300000"; + +/** The only shape an index name may have before it is spliced into SQL. */ +const INDEX_NAME = /^[a-z0-9_]{1,63}$/; + +/** Postgres's suffixes for the copies a concurrent rebuild leaves behind. */ +const CONCURRENT_COPY = /_ccnew\d*$|_ccold\d*$/; + +const PURGE_QUEUE = "compaction-tombstone-purge"; + +export interface MaintenanceStep { + step: "vacuum" | "reindex" | "drop_invalid"; + target: string; + bytesBefore: number; + bytesAfter: number; + durationMs: number; + ok: boolean; + error?: string; +} + +export interface MeasurementMaintenanceSummary { + outcome: "completed" | "refused_purge_running" | "already_running" | "failed"; + steps: MaintenanceStep[]; + tableBytesBefore: number; + tableBytesAfter: number; + indexBytesBefore: number; + indexBytesAfter: number; +} + +interface IndexRow { + name: string; + bytes: string; + valid: boolean; +} + +/** What the pass needs from a connection: one query method. */ +export type MaintenanceConnection = Pick; + +async function relationSizes( + db: MaintenanceConnection, +): Promise<{ table: number; indexes: number }> { + const { rows } = await db.query<{ table: string; indexes: string }>( + `SELECT pg_total_relation_size('measurements'::regclass)::text AS "table", + pg_indexes_size('measurements'::regclass)::text AS "indexes"`, + ); + return { + table: Number(rows[0]?.table ?? 0), + indexes: Number(rows[0]?.indexes ?? 0), + }; +} + +async function measurementIndexes( + db: MaintenanceConnection, +): Promise { + const { rows } = await db.query( + `SELECT c.relname AS name, + pg_relation_size(c.oid)::text AS bytes, + i.indisvalid AS valid + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + WHERE i.indrelid = 'measurements'::regclass + ORDER BY pg_relation_size(c.oid) DESC, c.relname`, + ); + return rows; +} + +async function indexBytes( + db: MaintenanceConnection, + name: string, +): Promise { + const { rows } = await db.query<{ bytes: string | null }>( + `SELECT pg_relation_size(to_regclass($1))::text AS bytes`, + [name], + ); + return Number(rows[0]?.bytes ?? 0); +} + +function quoted(name: string): string { + // Whitelist-spliced: catalog names only, checked against a closed shape. + if (!INDEX_NAME.test(name)) { + throw new Error(`refusing to splice index name of unexpected shape`); + } + return `"${name}"`; +} + +function errorText(err: unknown): string { + const code = (err as { code?: unknown })?.code; + const message = err instanceof Error ? err.message : String(err); + return typeof code === "string" ? `${code}: ${message}` : message; +} + +/** Drop the invalid copies an interrupted concurrent rebuild left behind. */ +async function dropInvalidCopies( + db: MaintenanceConnection, + steps: MaintenanceStep[], +): Promise { + for (const index of await measurementIndexes(db)) { + if (index.valid || !CONCURRENT_COPY.test(index.name)) continue; + const startedAt = Date.now(); + const bytesBefore = Number(index.bytes); + try { + await db.query(`DROP INDEX CONCURRENTLY IF EXISTS ${quoted(index.name)}`); + steps.push({ + step: "drop_invalid", + target: index.name, + bytesBefore, + bytesAfter: 0, + durationMs: Date.now() - startedAt, + ok: true, + }); + } catch (err) { + steps.push({ + step: "drop_invalid", + target: index.name, + bytesBefore, + bytesAfter: bytesBefore, + durationMs: Date.now() - startedAt, + ok: false, + error: errorText(err), + }); + } + } +} + +/** + * The pass itself, on a connection the caller opened with + * {@link MAINTENANCE_SESSION_OPTIONS}. Exported for the integration test. + */ +export async function runMeasurementMaintenance( + db: MaintenanceConnection, + payload: MeasurementMaintenancePayload, + options: { shouldStop?: () => boolean } = {}, +): Promise { + const before = await relationSizes(db); + const steps: MaintenanceStep[] = []; + const summary = ( + outcome: MeasurementMaintenanceSummary["outcome"], + after = before, + ): MeasurementMaintenanceSummary => ({ + outcome, + steps, + tableBytesBefore: before.table, + tableBytesAfter: after.table, + indexBytesBefore: before.indexes, + indexBytesAfter: after.indexes, + }); + + // The queue table exists wherever a worker has started; a database without + // one has no purge to wait for. + const { rows: queueTable } = await db.query<{ present: boolean }>( + `SELECT to_regclass('pgboss.job') IS NOT NULL AS present`, + ); + if (queueTable[0]?.present) { + const { rows: purge } = await db.query<{ n: string }>( + `SELECT count(*)::text AS n FROM pgboss.job + WHERE name = $1 AND state IN ('created', 'retry', 'active')`, + [PURGE_QUEUE], + ); + if (Number(purge[0]?.n ?? 0) > 0) return summary("refused_purge_running"); + } + + if (payload.vacuum) { + const startedAt = Date.now(); + try { + await db.query(`VACUUM (ANALYZE) "measurements"`); + const after = await relationSizes(db); + steps.push({ + step: "vacuum", + target: "measurements", + bytesBefore: before.table, + bytesAfter: after.table, + durationMs: Date.now() - startedAt, + ok: true, + }); + } catch (err) { + steps.push({ + step: "vacuum", + target: "measurements", + bytesBefore: before.table, + bytesAfter: before.table, + durationMs: Date.now() - startedAt, + ok: false, + error: errorText(err), + }); + return summary("failed", await relationSizes(db)); + } + } + + if (payload.reindex) { + await dropInvalidCopies(db, steps); + const targets = (await measurementIndexes(db)).filter( + (index) => index.valid && !CONCURRENT_COPY.test(index.name), + ); + for (const index of targets) { + if (options.shouldStop?.()) break; + const startedAt = Date.now(); + const bytesBefore = Number(index.bytes); + try { + await db.query(`REINDEX INDEX CONCURRENTLY ${quoted(index.name)}`); + steps.push({ + step: "reindex", + target: index.name, + bytesBefore, + bytesAfter: await indexBytes(db, index.name), + durationMs: Date.now() - startedAt, + ok: true, + }); + } catch (err) { + steps.push({ + step: "reindex", + target: index.name, + bytesBefore, + bytesAfter: bytesBefore, + durationMs: Date.now() - startedAt, + ok: false, + error: errorText(err), + }); + // A failed concurrent rebuild leaves its half-built copy behind, + // holding disk. Drop it and stop: the likeliest cause is the disk. + await dropInvalidCopies(db, steps); + return summary("failed", await relationSizes(db)); + } + } + } + + return summary("completed", await relationSizes(db)); +} + +export async function handleMeasurementMaintenance( + jobs: Job[], +): Promise { + const job = jobs[0]; + const payload: MeasurementMaintenancePayload = { + vacuum: job?.data?.vacuum ?? true, + reindex: job?.data?.reindex ?? true, + }; + return withBackgroundEvent("job.measurement_maintenance", async (evt) => { + const startedAt = Date.now(); + const guarded = await withJobLock( + MEASUREMENT_MAINTENANCE_QUEUE, + async () => { + const client = new Client({ + connectionString: process.env.DATABASE_URL, + options: MAINTENANCE_SESSION_OPTIONS, + keepAlive: true, + }); + client.on("error", () => {}); + await client.connect(); + try { + return await runMeasurementMaintenance(client, payload); + } finally { + await client.end().catch(() => {}); + } + }, + ); + const result: MeasurementMaintenanceSummary = guarded.ran + ? guarded.result + : { + outcome: "already_running", + steps: [], + tableBytesBefore: 0, + tableBytesAfter: 0, + indexBytesBefore: 0, + indexBytesAfter: 0, + }; + + evt.addMeta("maintenance_outcome", result.outcome); + evt.addMeta("maintenance_vacuum", payload.vacuum); + evt.addMeta("maintenance_reindex", payload.reindex); + evt.addMeta("maintenance_table_bytes_before", result.tableBytesBefore); + evt.addMeta("maintenance_table_bytes_after", result.tableBytesAfter); + evt.addMeta("maintenance_index_bytes_before", result.indexBytesBefore); + evt.addMeta("maintenance_index_bytes_after", result.indexBytesAfter); + // Catalog names and sizes only: nothing account- or value-shaped. + evt.addMeta( + "maintenance_steps", + JSON.stringify( + result.steps.map((s) => ({ + step: s.step, + target: s.target, + before: s.bytesBefore, + after: s.bytesAfter, + ms: s.durationMs, + ok: s.ok, + })), + ), + ); + + const candidates: JobRunCandidate[] = result.steps.map((s) => ({ + key: `${s.step}:${s.target}`, + outcome: s.ok ? "ok" : "failed", + ...(s.ok ? {} : { cause: s.error?.slice(0, 120) }), + })); + reportJobRun({ + queue: MEASUREMENT_MAINTENANCE_QUEUE, + runId: job?.id ?? "unknown", + candidates, + }); + + const facts = { + outcome: result.outcome, + processed: result.steps.filter((s) => s.ok).length, + failed: result.steps.filter((s) => !s.ok).length, + duration_ms: Date.now() - startedAt, + }; + if (result.outcome === "failed") { + const firstError = result.steps.find((s) => !s.ok); + return jobFailed( + `measurement maintenance stopped at ${firstError?.step ?? "a step"} of ${firstError?.target ?? "measurements"}`, + firstError?.error, + facts, + ); + } + if (result.outcome === "refused_purge_running") { + evt.addWarning( + "measurement maintenance refused: the compaction-tombstone purge is still queued or running", + ); + } + return jobDone(facts); + }); } diff --git a/src/lib/jobs/measurement-tombstone-cleanup.ts b/src/lib/jobs/measurement-tombstone-cleanup.ts index 306b2bb49..bd84ad636 100644 --- a/src/lib/jobs/measurement-tombstone-cleanup.ts +++ b/src/lib/jobs/measurement-tombstone-cleanup.ts @@ -43,11 +43,13 @@ export interface TombstonePurgeOutcome extends PurgeOutcome { * account was running. The next night purges them. */ deferredAccounts: number; + /** The accounts behind `deferredAccounts`, for a per-account run report. */ + deferredUserIds: string[]; } type TombstoneRow = { id: string; userId: string }; -interface TombstonePurgeOptions { +export interface TombstonePurgeOptions { prisma: PrismaClient; /** Up to `take` expired tombstones, none of them owned by `skipUserIds`. */ findRows: (take: number, skipUserIds: string[]) => Promise; @@ -55,6 +57,17 @@ interface TombstonePurgeOptions { deleteIds: (tx: Prisma.TransactionClient, ids: string[]) => Promise; batchSize?: number; maxBatches?: number; + /** + * Awaited between two batches. The compaction-tombstone backlog purge uses + * it to pause, so a run of forty batches does not hold the disk and the WAL + * at full rate for its whole length. + */ + betweenBatches?: () => Promise; + /** + * Ends the walk before the next batch when it returns `true`; the outcome + * then reports `drained: false`, exactly like a run that hit the batch cap. + */ + shouldStop?: () => boolean; } /** @@ -74,12 +87,14 @@ interface TombstonePurgeOptions { * * Still one delete statement per account per batch, never one per row. */ -async function purgeTombstonesByAccount({ +export async function purgeTombstonesByAccount({ prisma, findRows, deleteIds, batchSize = PURGE_BATCH_SIZE, maxBatches = PURGE_MAX_BATCHES, + betweenBatches, + shouldStop, }: TombstonePurgeOptions): Promise { let deleted = 0; const deferred = new Set(); @@ -87,9 +102,14 @@ async function purgeTombstonesByAccount({ deleted, drained, deferredAccounts: deferred.size, + deferredUserIds: [...deferred], }); for (let batch = 0; batch < maxBatches; batch++) { + if (batch > 0) { + if (shouldStop?.()) return outcome(false); + await betweenBatches?.(); + } const rows = await findRows(batchSize, [...deferred]); if (rows.length === 0) return outcome(true); diff --git a/src/lib/jobs/pr-detection.ts b/src/lib/jobs/pr-detection.ts index 94f933fd3..abe18a1e1 100644 --- a/src/lib/jobs/pr-detection.ts +++ b/src/lib/jobs/pr-detection.ts @@ -11,13 +11,15 @@ * 30-minute fallback cron in `reminder-worker.ts` is the safety net * for any ingest path that forgets to enqueue. */ +import type { PrismaClient } from "@/generated/prisma/client"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; export const PR_DETECTION_QUEUE = "pr-detection"; /** - * The job's expiry: twenty-five minutes. The fallback cron walks every - * account's whole history every thirty minutes, which on a large instance + * The job's expiry: twenty-five minutes. The fallback cron walks the whole + * history of every account that changed, every thirty minutes (every account + * before v1.42), which on a large instance * outlasts pg-boss's fifteen-minute default; past it pg-boss retried the pass * beside itself. The pass stops at three quarters of this (`jobBudget`) and * holds a lock (`lockedPass`), so it ends before the next tick and a retry or @@ -33,13 +35,72 @@ export const PR_DETECTION_CONCURRENCY = 5; /** * Cron schedule (Europe/Berlin) for the fallback rescan. Every 30 - * minutes the worker re-runs detection for every user — protects - * against ingest paths that ship rows without enqueueing a job (a - * future Withings sync that lands without the hook wired, a manual - * row imported via SQL, a backfill from a future migration). + * minutes the worker re-runs detection for every user whose data changed + * since the last tick (v1.42; every user before) — protects against + * ingest paths that ship rows without enqueueing a job (a future Withings + * sync that lands without the hook wired, a backfill from a future + * migration). */ export const PR_DETECTION_FALLBACK_CRON = "*/30 * * * *"; +/** + * v1.42 — how far back the fallback tick looks for accounts with changed + * data: the thirty-minute cadence plus five minutes, so a write between two + * ticks is always inside one of them. + */ +export const PR_DETECTION_WATERMARK_MINUTES = 35; + +/** + * The first fallback tick after a worker boot looks back a day instead: the + * process that would have run the ticks in between was not there. + */ +export const PR_DETECTION_BOOT_LOOKBACK_HOURS = 24; + +/** + * The accounts the fallback tick re-scans (v1.42): those with a live + * measurement or a workout changed since `since`. Until v1.42 the tick walked + * every account's whole history every thirty minutes, though detection can + * only find something new where something was written. No column records the + * last run: the window is the cron's own cadence, and each probe is a range + * scan on `measurements (user_id, updated_at, id)` per account. + */ +export async function listPrDetectionFallbackUserIds( + prisma: Pick, + since: Date, +): Promise { + const rows = await prisma.$queryRaw>` + SELECT u."id" + FROM "users" u + WHERE EXISTS ( + SELECT 1 FROM "measurements" m + WHERE m."user_id" = u."id" + AND m."updated_at" > ${since} + AND m."deleted_at" IS NULL + ) + OR EXISTS ( + SELECT 1 FROM "workouts" w + WHERE w."user_id" = u."id" AND w."updated_at" > ${since} + ) + ORDER BY u."id" + `; + return rows.map((row) => row.id); +} + +let fallbackTicksSinceBoot = 0; + +/** + * The window start for the next fallback tick: a day back on the first tick + * of this process, {@link PR_DETECTION_WATERMARK_MINUTES} afterwards. + */ +export function nextPrDetectionFallbackSince(now: Date = new Date()): Date { + const first = fallbackTicksSinceBoot === 0; + fallbackTicksSinceBoot += 1; + const lookbackMs = first + ? PR_DETECTION_BOOT_LOOKBACK_HOURS * 3_600_000 + : PR_DETECTION_WATERMARK_MINUTES * 60_000; + return new Date(now.getTime() - lookbackMs); +} + export interface PrDetectionPayload { userId: string; /** diff --git a/src/lib/jobs/reminder/__tests__/queue-retention.test.ts b/src/lib/jobs/reminder/__tests__/queue-retention.test.ts new file mode 100644 index 000000000..aacdcb4da --- /dev/null +++ b/src/lib/jobs/reminder/__tests__/queue-retention.test.ts @@ -0,0 +1,87 @@ +/** + * v1.42 — shortened pg-boss retention for the pure volume queues. + * + * A failed job stays readable in `pgboss.job` only until + * `completed_on + deletion_seconds`, and the admin status reads failures from + * there over a 72-hour window (`job-failures.ts`). A volume queue kept for + * less than that window plus a day would let its failures vanish before they + * were shown, so the floor is pinned here, the list stays the four queues + * that were decided on, and the registrar applies the value on create and on + * every boot. + */ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/db", () => ({ + prisma: { $executeRaw: vi.fn().mockResolvedValue(0) }, +})); +vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); + +import { JOB_FAILURE_WINDOW_HOURS } from "@/lib/jobs/job-failures"; +import { + QUEUE_RETENTION_FLOOR_SECONDS, + VOLUME_QUEUE_DELETE_AFTER_SECONDS, + createAndSchedule, +} from "../registrar-shared"; + +describe("volume queue retention", () => { + it("never drops below the failure window plus a day", () => { + expect(QUEUE_RETENTION_FLOOR_SECONDS).toBeGreaterThanOrEqual( + (JOB_FAILURE_WINDOW_HOURS + 24) * 3600, + ); + for (const [queue, seconds] of Object.entries( + VOLUME_QUEUE_DELETE_AFTER_SECONDS, + )) { + expect(seconds, queue).toBeGreaterThanOrEqual( + QUEUE_RETENTION_FLOOR_SECONDS, + ); + } + }); + + it("covers exactly the four volume queues", () => { + expect(Object.keys(VOLUME_QUEUE_DELETE_AFTER_SECONDS).sort()).toEqual([ + "__pgboss__send-it", + "host-metric-sample", + "insight-status-generate", + "rollup-recompute", + ]); + }); + + it("creates a volume queue with its retention and updates it on an existing instance", async () => { + const boss = { + createQueue: vi.fn().mockResolvedValue(undefined), + updateQueue: vi.fn().mockResolvedValue(undefined), + schedule: vi.fn().mockResolvedValue(undefined), + }; + await createAndSchedule( + boss as never, + ["host-metric-sample", "some-other-queue"], + [["host-metric-sample", "*/5 * * * *"]], + ); + expect(boss.createQueue).toHaveBeenCalledWith("host-metric-sample", { + deleteAfterSeconds: QUEUE_RETENTION_FLOOR_SECONDS, + }); + expect(boss.createQueue).toHaveBeenCalledWith("some-other-queue", {}); + expect(boss.updateQueue).toHaveBeenCalledWith("host-metric-sample", { + deleteAfterSeconds: QUEUE_RETENTION_FLOOR_SECONDS, + }); + // The timekeeper's queue rides along whenever the registrar schedules. + expect(boss.updateQueue).toHaveBeenCalledWith("__pgboss__send-it", { + deleteAfterSeconds: QUEUE_RETENTION_FLOOR_SECONDS, + }); + expect(boss.updateQueue).not.toHaveBeenCalledWith( + "some-other-queue", + expect.anything(), + ); + }); + + it("does not fail the boot when an update is refused", async () => { + const boss = { + createQueue: vi.fn().mockResolvedValue(undefined), + updateQueue: vi.fn().mockRejectedValue(new Error("nope")), + schedule: vi.fn().mockResolvedValue(undefined), + }; + await expect( + createAndSchedule(boss as never, ["rollup-recompute"], []), + ).resolves.toBeUndefined(); + }); +}); diff --git a/src/lib/jobs/reminder/ops-handlers.ts b/src/lib/jobs/reminder/ops-handlers.ts index 7f944c1b8..608f3a9ee 100644 --- a/src/lib/jobs/reminder/ops-handlers.ts +++ b/src/lib/jobs/reminder/ops-handlers.ts @@ -10,7 +10,11 @@ import { runGeoBackfill } from "@/lib/jobs/geo-backfill"; import { fetchGeoLite2Databases } from "@/lib/geo/geolite2-fetch"; import { type Geolite2FetchPayload } from "@/lib/jobs/geolite2-fetch"; import { runTlsPinMonitor } from "@/lib/jobs/tls-pin-monitor"; -import { type PrDetectionPayload } from "@/lib/jobs/pr-detection"; +import { + listPrDetectionFallbackUserIds, + nextPrDetectionFallbackSince, + type PrDetectionPayload, +} from "@/lib/jobs/pr-detection"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runHostMetricTick } from "@/lib/jobs/host-metric-sampler"; @@ -225,8 +229,8 @@ export async function handlePrDetection( let tiesAcrossJobs = 0; let usersFailed = 0; let stoppedEarly = false; - // The cron pass walks every account's all-time history and ticks every - // thirty minutes. It stops between accounts once the job's budget is spent; + // The cron pass walks the all-time history of every account whose data + // changed since the last tick, every thirty minutes. It stops between accounts once the job's budget is spent; // the ingest jobs and the next tick carry the rest. const shouldStop = jobBudget(jobs); for (const job of jobs) { @@ -240,9 +244,14 @@ export async function handlePrDetection( ?.userId; const silent = (job.data as PrDetectionPayload | undefined)?.silent ?? false; + // v1.42 — the cron path re-scans only accounts whose measurements or + // workouts changed since the previous tick (`pr-detection.ts`). const userIds: string[] = payloadUserId ? [payloadUserId] - : (await p.user.findMany({ select: { id: true } })).map((u) => u.id); + : await listPrDetectionFallbackUserIds( + p, + nextPrDetectionFallbackSince(), + ); let insertedTotal = 0; let tiesTotal = 0; diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 01ace3f8d..6c1d2d210 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -790,6 +790,22 @@ const queuePolicies: QueuePolicyTable = { reason: "Fixed singleton key, admin-triggered. Two concurrent corpus rotations must never overlap; the route already reports a suppressed send back as alreadyQueued.", }, + // v1.42 — operator-triggered VACUUM / REINDEX of `measurements`. + [MEASUREMENT_MAINTENANCE_QUEUE]: { + policy: "exclusive", + reason: + "Fixed singleton key, admin-triggered. Two maintenance passes would rebuild the same indexes side by side; the route reports a suppressed send back as enqueued: false.", + }, + // v1.42 — compaction-tombstone backlog purge. `short`, not `exclusive`: + // a run that stops early sends its own follow-up while it is still active, + // and `exclusive` would swallow that send. A boot's send while a run is + // queued collapses into it; the delete predicate is idempotent, so a boot + // that lands beside an active run costs a duplicate scan, never a wrong row. + [COMPACTION_TOMBSTONE_PURGE_QUEUE]: { + policy: "short", + reason: + "Fixed singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, // Per-document, enqueued on upload. `short`, NOT `exclusive`: each handler // re-reads the document when it starts, so collapsing sends that arrive while diff --git a/src/lib/jobs/reminder/registrar-shared.ts b/src/lib/jobs/reminder/registrar-shared.ts index 6841b0111..454943d81 100644 --- a/src/lib/jobs/reminder/registrar-shared.ts +++ b/src/lib/jobs/reminder/registrar-shared.ts @@ -210,6 +210,82 @@ export const nightlyInsightPassOptions = { */ export const cronIsTheRetry = { retryLimit: 0 } as const; +/** + * pg-boss's internal queue the cron timekeeper sends through: one job per + * schedule tick, for every schedule. + */ +const PG_BOSS_SEND_IT_QUEUE = "__pgboss__send-it"; + +/** + * The shortest retention a queue may be given: the 72-hour failure window + * `job-failures.ts` reads back from pg-boss's terminal rows, plus a day. A + * failed job stays in `pgboss.job` until `completed_on + deletion_seconds`, + * so a queue kept for less than the window would make its failures vanish + * from the admin status before they were ever shown. + */ +export const QUEUE_RETENTION_FLOOR_SECONDS = 96 * 60 * 60; + +/** + * v1.42 — pure volume queues and their terminal-row retention + * (`deleteAfterSeconds`). pg-boss keeps every completed and failed job for + * seven days by default; these queues mint a job every few minutes, or one + * per write, and nothing reads their history beyond the failure window, so + * their rows were most of `pgboss.job` and of its sequential scans. Every + * other queue keeps the default. Each value is held at or above + * {@link QUEUE_RETENTION_FLOOR_SECONDS} by `queue-retention.test.ts`. + */ +export const VOLUME_QUEUE_DELETE_AFTER_SECONDS: Readonly< + Record +> = { + // Every five minutes, all day. + "host-metric-sample": QUEUE_RETENTION_FLOOR_SECONDS, + // One job per (user, type, day) a write touched. + "rollup-recompute": QUEUE_RETENTION_FLOOR_SECONDS, + // One job per write that dirties a status card. + "insight-status-generate": QUEUE_RETENTION_FLOOR_SECONDS, + // One job per schedule tick, for every cron in the tree. + [PG_BOSS_SEND_IT_QUEUE]: QUEUE_RETENTION_FLOOR_SECONDS, +}; + +/** + * Bring the retention of already-existing volume queues in line with the + * table above. `createQueue` only applies its options when it creates the + * queue, so an upgraded instance needs the update; `updateQueue` may change + * `deleteAfterSeconds` (unlike the policy). A job copies the value when it is + * inserted, so rows already in the table keep theirs and age out as before. + * Never fails a boot. + */ +async function reconcileVolumeQueueRetention( + boss: PgBoss, + queues: readonly string[], +): Promise { + const reconciled: string[] = []; + const failed: string[] = []; + for (const name of queues) { + const deleteAfterSeconds = VOLUME_QUEUE_DELETE_AFTER_SECONDS[name]; + if (deleteAfterSeconds === undefined) continue; + try { + await boss.updateQueue(name, { deleteAfterSeconds }); + reconciled.push(name); + } catch (err) { + failed.push(name); + workerLog("error", `[queue-retention] failed to update ${name}`, err); + } + } + if (reconciled.length === 0 && failed.length === 0) return; + await withBackgroundEvent( + "worker.boot.queue_retention_reconcile", + async () => { + annotate({ + meta: { + queue_retention_reconciled: reconciled.join(",") || "none", + queue_retention_failed_count: failed.length, + }, + }); + }, + ); +} + /** * Create every queue in `queues`, then schedule every cron in `schedules`. * Centralised so each registrar provisions before it schedules in the exact @@ -231,9 +307,19 @@ export async function createAndSchedule( ): Promise { for (const q of queues) { const decision = policies[q]; - await boss.createQueue(q, decision ? { policy: decision.policy } : {}); + const deleteAfterSeconds = VOLUME_QUEUE_DELETE_AFTER_SECONDS[q]; + await boss.createQueue(q, { + ...(decision ? { policy: decision.policy } : {}), + ...(deleteAfterSeconds !== undefined ? { deleteAfterSeconds } : {}), + }); } await reconcileQueuePolicies(policies); + // The cron timekeeper's own queue exists once pg-boss has started, and only + // a registrar that schedules something sends through it. + await reconcileVolumeQueueRetention(boss, [ + ...queues, + ...(schedules.length > 0 ? [PG_BOSS_SEND_IT_QUEUE] : []), + ]); for (const [name, cron, sendOptions] of schedules) { await boss.schedule( name, diff --git a/src/lib/measurements/__tests__/consolidate-daily-mean.test.ts b/src/lib/measurements/__tests__/consolidate-daily-mean.test.ts index 4eec27d3b..53bf25ef6 100644 --- a/src/lib/measurements/__tests__/consolidate-daily-mean.test.ts +++ b/src/lib/measurements/__tests__/consolidate-daily-mean.test.ts @@ -94,6 +94,7 @@ describe("consolidateDailyMean — drain flow (mocked Prisma)", () => { function buildPrismaMock(rowsByType: Record) { const upsert = vi.fn().mockResolvedValue({}); const updateMany = vi.fn().mockResolvedValue({ count: 0 }); + const deleteMany = vi.fn().mockResolvedValue({ count: 0 }); const update = vi.fn().mockResolvedValue({}); // Canonical-slot probe inside the write transaction — defaults to an // unoccupied slot (the pre-collision-fix happy path). @@ -103,7 +104,13 @@ describe("consolidateDailyMean — drain flow (mocked Prisma)", () => { rowsByType[args.where.type] ?? [], ); const tx = { - measurement: { upsert, updateMany, update, findFirst: txFindFirst }, + measurement: { + upsert, + updateMany, + deleteMany, + update, + findFirst: txFindFirst, + }, }; return { mock: { @@ -121,6 +128,7 @@ describe("consolidateDailyMean — drain flow (mocked Prisma)", () => { }, upsert, updateMany, + deleteMany, update, txFindFirst, findManyMeasurement, @@ -141,12 +149,12 @@ describe("consolidateDailyMean — drain flow (mocked Prisma)", () => { } }); - it("upserts the per-day mean and soft-deletes the source rows", async () => { + it("upserts the per-day mean and deletes the source rows", async () => { const walkingSpeedRows = [ row("a", 1.0, "2026-05-01T08:00:00.000Z"), row("b", 1.4, "2026-05-01T09:00:00.000Z"), ]; - const { mock, upsert, updateMany } = buildPrismaMock({ + const { mock, upsert, updateMany, deleteMany } = buildPrismaMock({ WALKING_SPEED: walkingSpeedRows, }); @@ -161,11 +169,12 @@ describe("consolidateDailyMean — drain flow (mocked Prisma)", () => { // Unit is read straight off the day's rows (no separate query). expect(upsertArg.create.unit).toBe("m/s"); - // soft-delete: updateMany sets deletedAt, never a hard delete. - const updArg = updateMany.mock.calls[0]?.[0] as { - data: { deletedAt: Date }; + // v1.42 — the folded rows are deleted outright, no longer tombstoned. + const delArg = deleteMany.mock.calls[0]?.[0] as { + where: { id: { in: string[] } }; }; - expect(updArg.data.deletedAt).toBeInstanceOf(Date); + expect(delArg.where.id.in).toEqual(["a", "b"]); + expect(updateMany).not.toHaveBeenCalled(); expect(summary.totals.daysConsolidated).toBe(1); // T3 — the rollup DAY bucket must be recomputed for the touched @@ -277,6 +286,7 @@ describe("consolidateDailyMean — canonical-slot collision (second unique index function buildPrismaMock(rowsByType: Record) { const upsert = vi.fn().mockResolvedValue({}); const updateMany = vi.fn().mockResolvedValue({ count: 0 }); + const deleteMany = vi.fn().mockResolvedValue({ count: 0 }); const update = vi.fn().mockResolvedValue({}); const txFindFirst = vi.fn().mockResolvedValue(null); const findManyMeasurement = vi.fn( @@ -284,7 +294,13 @@ describe("consolidateDailyMean — canonical-slot collision (second unique index rowsByType[args.where.type] ?? [], ); const tx = { - measurement: { upsert, updateMany, update, findFirst: txFindFirst }, + measurement: { + upsert, + updateMany, + deleteMany, + update, + findFirst: txFindFirst, + }, }; return { mock: { @@ -300,6 +316,7 @@ describe("consolidateDailyMean — canonical-slot collision (second unique index } as unknown as PrismaClient, upsert, updateMany, + deleteMany, update, txFindFirst, }; diff --git a/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts b/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts index ea5f764ba..aee9a1c02 100644 --- a/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-pulse-summary.test.ts @@ -57,7 +57,7 @@ function pulseRow(id: string, value: number, iso: string): PerSampleRow { /** * Prisma mock covering the full PULSE-summary flow: the scan (per-type rows), * the per-user native-resting probe (top-level `findFirst`), the fold - * transaction (`create`/`update`/`findFirst`/`updateMany`), and the derived + * transaction (`create`/`update`/`findFirst`/`updateMany`/`deleteMany`), and the derived * resting `upsert` (top-level). */ function buildPrismaMock(opts: { @@ -68,7 +68,7 @@ function buildPrismaMock(opts: { const txCreate = vi.fn().mockResolvedValue({ id: "minted-daily" }); const txUpdate = vi.fn().mockResolvedValue({}); const txFindFirst = vi.fn().mockResolvedValue(null); - const txUpdateMany = vi + const txDeleteMany = vi .fn() .mockResolvedValue({ count: opts.pulseRows.length }); const tx = { @@ -76,7 +76,8 @@ function buildPrismaMock(opts: { create: txCreate, update: txUpdate, findFirst: txFindFirst, - updateMany: txUpdateMany, + updateMany: vi.fn().mockResolvedValue({ count: 0 }), + deleteMany: txDeleteMany, findMany: candidateLookup(), createManyAndReturn: createManyVia(txCreate), }, @@ -102,7 +103,7 @@ function buildPrismaMock(opts: { $transaction: vi.fn(async (cb: (t: unknown) => Promise) => cb(tx)), } as unknown as PrismaClient; - return { mock, txCreate, txUpdateMany, upsert, topFindFirst }; + return { mock, txCreate, txDeleteMany, upsert, topFindFirst }; } beforeEach(() => { @@ -335,7 +336,8 @@ describe("PULSE fold — per-day failure boundary", () => { create: vi.fn().mockResolvedValue({ id: "minted" }), update: vi.fn(), findFirst: vi.fn().mockResolvedValue(null), - updateMany: vi.fn().mockResolvedValue({ count: 5 }), + updateMany: vi.fn().mockResolvedValue({ count: 0 }), + deleteMany: vi.fn().mockResolvedValue({ count: 5 }), findMany: candidateLookup(), createManyAndReturn: createManyVia( vi.fn().mockResolvedValue({ id: "minted" }), diff --git a/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts b/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts index dd7c9218c..e72ae5db1 100644 --- a/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-resting-mint.test.ts @@ -71,6 +71,7 @@ function buildPrismaMock(existingResting: Array<{ source: string }>) { const create = vi.fn().mockResolvedValue({ id: "minted-hourly" }); const update = vi.fn().mockResolvedValue({}); const updateMany = vi.fn().mockResolvedValue({ count: 0 }); + const deleteMany = vi.fn().mockResolvedValue({ count: 0 }); const txFindFirst = vi.fn().mockResolvedValue(null); // The probe. Honours a `source: { not: X }` clause exactly like the DB. @@ -90,6 +91,7 @@ function buildPrismaMock(existingResting: Array<{ source: string }>) { update, findFirst: txFindFirst, updateMany, + deleteMany, upsert, findMany: candidateLookup(), createManyAndReturn: createManyVia(create), diff --git a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts index d4e747507..740f40d21 100644 --- a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts @@ -6,7 +6,7 @@ * OLDER than the retention window (`measuredAt < now - retentionDays`), so * the recent intra-day shape the Stress engine reads is never collapsed. * Also pins the dense-tier scope (HRV + PULSE + SpO2), the APPLE_HEALTH - * source scope, the per-local-hour MEAN reduction + soft-delete, the + * source scope, the per-local-hour MEAN reduction + delete, the * hourly `stats:` externalId shape, and the pre-fold rollup recompute. */ import { describe, it, expect, vi, beforeEach } from "vitest"; @@ -61,6 +61,7 @@ function buildPrismaMock( existingCanonicalId ? { id: existingCanonicalId } : null, ); const updateMany = vi.fn().mockResolvedValue({ count: 0 }); + const deleteMany = vi.fn().mockResolvedValue({ count: 0 }); const findManyMeasurement = vi.fn( async (args: { where: { type: string } }) => rowsByType[args.where.type] ?? [], @@ -71,6 +72,7 @@ function buildPrismaMock( update, findFirst, updateMany, + deleteMany, findMany: candidateLookup(existingCanonicalId), createManyAndReturn: createManyVia(create), }, @@ -91,6 +93,7 @@ function buildPrismaMock( update, findFirst, updateMany, + deleteMany, findManyMeasurement, }; } @@ -155,14 +158,14 @@ describe("runDenseIntradayRetention — retention bound", () => { }); describe("runDenseIntradayRetention — hourly fold flow", () => { - it("creates one MEAN row per LOCAL hour and soft-deletes the out-of-window rows", async () => { + it("creates one MEAN row per LOCAL hour and deletes the out-of-window rows", async () => { // Two HRV samples in DIFFERENT local hours (Berlin is UTC+2 on this // date: 08:00Z → 10:xx local, 09:00Z → 11:xx local). const hrvRows = [ row("a", 40, "2026-05-01T08:00:00.000Z", "HEART_RATE_VARIABILITY"), row("b", 60, "2026-05-01T09:00:00.000Z", "HEART_RATE_VARIABILITY"), ]; - const { mock, create, update, updateMany } = buildPrismaMock({ + const { mock, create, update, updateMany, deleteMany } = buildPrismaMock({ HEART_RATE_VARIABILITY: hrvRows, }); @@ -205,11 +208,12 @@ describe("runDenseIntradayRetention — hourly fold flow", () => { expect(d.type).toBe("HEART_RATE_VARIABILITY"); } - // soft-delete, never hard delete. - const updArg = updateMany.mock.calls[0]?.[0] as { - data: { deletedAt: Date }; + // v1.42 — the raw rows are deleted outright, no longer tombstoned. + const delArg = deleteMany.mock.calls[0]?.[0] as { + where: { id: { in: string[] } }; }; - expect(updArg.data.deletedAt).toBeInstanceOf(Date); + expect(delArg.where.id.in).toEqual(["a", "b"]); + expect(updateMany).not.toHaveBeenCalled(); expect(summary.totals.daysConsolidated).toBe(1); expect(summary.totals.hourlyRowsUpserted).toBe(2); }); @@ -242,7 +246,7 @@ describe("runDenseIntradayRetention — hourly fold flow", () => { row("a", 40, "2026-05-01T08:00:00.000Z", "HEART_RATE_VARIABILITY"), ]; // A row already sits on the hourly slot (externalId or anchor). - const { mock, create, update, updateMany } = buildPrismaMock( + const { mock, create, update, deleteMany } = buildPrismaMock( { HEART_RATE_VARIABILITY: hrvRows }, "existing-canonical-row", ); @@ -264,11 +268,11 @@ describe("runDenseIntradayRetention — hourly fold flow", () => { ); expect(updateArg.data.deletedAt).toBeNull(); - // The soft-delete excludes the adopted canonical row ids. - const updManyArg = updateMany.mock.calls[0]?.[0] as { + // The delete excludes the adopted canonical row ids. + const delArg = deleteMany.mock.calls[0]?.[0] as { where: { id: { notIn: string[] } }; }; - expect(updManyArg.where.id.notIn).toContain("existing-canonical-row"); + expect(delArg.where.id.notIn).toContain("existing-canonical-row"); }); it("retires a live pre-hourly DAILY stats row in the same fold transaction", async () => { diff --git a/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts b/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts index 32f5a1478..6ae4152e5 100644 --- a/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-spo2-summary.test.ts @@ -13,7 +13,7 @@ * mean to an unweighted mean-of-hourly-means) is skipped. * * This suite pins the SpO2 facet: the pre-fold (and only) DAY-rollup recompute, - * the hourly MEAN fold + soft-delete, the no-derived-resting-row guard + * the hourly MEAN fold + delete, the no-derived-resting-row guard * (resting HR is a PULSE-only concern), idempotency, and the per-day failure * boundary. */ @@ -52,7 +52,7 @@ function spo2Row(id: string, value: number, iso: string): PerSampleRow { /** * Prisma mock covering the SpO2 fold flow: the scan (per-type rows), the fold - * transaction (`create`/`update`/`findFirst`/`updateMany`), and the top-level + * transaction (`create`/`update`/`findFirst`/`updateMany`/`deleteMany`), and the top-level * surfaces a PULSE day would touch (`findFirst` for the native-resting probe, * `upsert` for the derived resting row) — both must stay UNCALLED for SpO2. */ @@ -60,7 +60,7 @@ function buildPrismaMock(opts: { spo2Rows: PerSampleRow[] }) { const txCreate = vi.fn().mockResolvedValue({ id: "minted-daily" }); const txUpdate = vi.fn().mockResolvedValue({}); const txFindFirst = vi.fn().mockResolvedValue(null); - const txUpdateMany = vi + const txDeleteMany = vi .fn() .mockResolvedValue({ count: opts.spo2Rows.length }); const tx = { @@ -68,7 +68,8 @@ function buildPrismaMock(opts: { spo2Rows: PerSampleRow[] }) { create: txCreate, update: txUpdate, findFirst: txFindFirst, - updateMany: txUpdateMany, + updateMany: vi.fn().mockResolvedValue({ count: 0 }), + deleteMany: txDeleteMany, findMany: candidateLookup(), createManyAndReturn: createManyVia(txCreate), }, @@ -91,15 +92,15 @@ function buildPrismaMock(opts: { spo2Rows: PerSampleRow[] }) { $transaction: vi.fn(async (cb: (t: unknown) => Promise) => cb(tx)), } as unknown as PrismaClient; - return { mock, txCreate, txUpdate, txUpdateMany, upsert, topFindFirst }; + return { mock, txCreate, txUpdate, txDeleteMany, upsert, topFindFirst }; } beforeEach(() => { recomputeBucketsForMeasurement.mockClear(); }); -describe("SpO2 fold — hourly MEAN collapse + soft-delete", () => { - it("folds each LOCAL hour to its own MEAN row and soft-deletes the raw rows", async () => { +describe("SpO2 fold — hourly MEAN collapse + delete", () => { + it("folds each LOCAL hour to its own MEAN row and deletes the raw rows", async () => { // Overnight dip to 91, daytime band 96..98 — four distinct local hours. const rows = [ spo2Row("a", 91, "2026-05-01T03:00:00.000Z"), @@ -107,7 +108,7 @@ describe("SpO2 fold — hourly MEAN collapse + soft-delete", () => { spo2Row("c", 97, "2026-05-01T13:00:00.000Z"), spo2Row("d", 98, "2026-05-01T19:00:00.000Z"), ]; - const { mock, txCreate, txUpdateMany } = buildPrismaMock({ + const { mock, txCreate, txDeleteMany } = buildPrismaMock({ spo2Rows: rows, }); @@ -134,11 +135,11 @@ describe("SpO2 fold — hourly MEAN collapse + soft-delete", () => { expect(d.unit).toBe("%"); } - // Soft-delete, never hard delete. - const updArg = txUpdateMany.mock.calls[0]?.[0] as { - data: { deletedAt: Date }; + // v1.42 — the raw rows are deleted outright, no longer tombstoned. + const delArg = txDeleteMany.mock.calls[0]?.[0] as { + where: { id: { in: string[] } }; }; - expect(updArg.data.deletedAt).toBeInstanceOf(Date); + expect(delArg.where.id.in).toHaveLength(rows.length); expect(summary.totals.daysConsolidated).toBe(1); expect(summary.totals.perSampleRowsSoftDeleted).toBe(rows.length); expect(summary.totals.hourlyRowsUpserted).toBe(4); @@ -191,13 +192,13 @@ describe("SpO2 fold — no resting derivation (PULSE-only concern)", () => { describe("SpO2 fold — idempotency + dry-run + no double-collapse", () => { it("converges to zero work when the scan returns no live rows (re-run)", async () => { - const { mock, txCreate, txUpdateMany } = buildPrismaMock({ spo2Rows: [] }); + const { mock, txCreate, txDeleteMany } = buildPrismaMock({ spo2Rows: [] }); const summary = await runDenseIntradayRetention(mock, { retentionDays: 0, log: () => {}, }); expect(txCreate).not.toHaveBeenCalled(); - expect(txUpdateMany).not.toHaveBeenCalled(); + expect(txDeleteMany).not.toHaveBeenCalled(); expect(summary.totals.daysConsolidated).toBe(0); expect(recomputeBucketsForMeasurement).not.toHaveBeenCalled(); }); @@ -264,7 +265,8 @@ describe("SpO2 fold — per-day failure boundary", () => { create: vi.fn().mockResolvedValue({ id: "minted" }), update: vi.fn(), findFirst: vi.fn().mockResolvedValue(null), - updateMany: vi.fn().mockResolvedValue({ count: 2 }), + updateMany: vi.fn().mockResolvedValue({ count: 0 }), + deleteMany: vi.fn().mockResolvedValue({ count: 2 }), findMany: candidateLookup(), createManyAndReturn: createManyVia( vi.fn().mockResolvedValue({ id: "minted" }), diff --git a/src/lib/measurements/__tests__/folded-window.test.ts b/src/lib/measurements/__tests__/folded-window.test.ts new file mode 100644 index 000000000..f75359642 --- /dev/null +++ b/src/lib/measurements/__tests__/folded-window.test.ts @@ -0,0 +1,228 @@ +/** + * v1.42 — the `folded_window` rule and the compaction-tombstone class. + * + * Pins what counts as a folded sample (source, type, age, id shape), which + * `stats:` id covers it (the fold's own hourly or daily id, in the account's + * zone), and that a tombstone is class A only when it was already past the + * fold threshold at the moment it was deleted, which is what keeps a + * person's deletion of a recent sample out of the purge. + */ +import { describe, expect, it, vi } from "vitest"; + +import { + coveringStatsExternalId, + findCompactionTombstones, + findFoldedWindowDuplicates, + isFoldedWindowCandidate, +} from "../folded-window"; + +const DAY = 86_400_000; +const NOW = new Date("2026-10-08T12:00:00.000Z"); + +function raw( + type: string, + measuredAt: Date, + overrides: Record = {}, +) { + return { + type, + source: "APPLE_HEALTH", + externalId: "uuid-1", + measuredAt, + ...overrides, + } as Parameters[0]; +} + +describe("isFoldedWindowCandidate", () => { + it("takes dense samples past 90 days and mean samples past 36 hours", () => { + expect( + isFoldedWindowCandidate( + raw("PULSE", new Date(NOW.getTime() - 91 * DAY)), + NOW, + ), + ).toBe(true); + expect( + isFoldedWindowCandidate( + raw("PULSE", new Date(NOW.getTime() - 89 * DAY)), + NOW, + ), + ).toBe(false); + expect( + isFoldedWindowCandidate( + raw("RESPIRATORY_RATE", new Date(NOW.getTime() - 2 * DAY)), + NOW, + ), + ).toBe(true); + expect( + isFoldedWindowCandidate( + raw("RESPIRATORY_RATE", new Date(NOW.getTime() - DAY)), + NOW, + ), + ).toBe(false); + }); + + it("refuses other sources, other types, `stats:` and `retired:` ids", () => { + const old = new Date(NOW.getTime() - 200 * DAY); + expect( + isFoldedWindowCandidate(raw("PULSE", old, { source: "MANUAL" }), NOW), + ).toBe(false); + expect(isFoldedWindowCandidate(raw("WEIGHT", old), NOW)).toBe(false); + expect( + isFoldedWindowCandidate( + raw("PULSE", old, { + externalId: "stats:HKQuantityTypeIdentifierHeartRate:2026-01-01T10", + }), + NOW, + ), + ).toBe(false); + expect( + isFoldedWindowCandidate( + raw("PULSE", old, { externalId: "retired:m1:uuid-1" }), + NOW, + ), + ).toBe(false); + expect( + isFoldedWindowCandidate(raw("PULSE", old, { externalId: null }), NOW), + ).toBe(false); + }); +}); + +describe("coveringStatsExternalId", () => { + it("names the local hour for a dense type and the local day for a mean type", () => { + // 22:30 UTC on 1 Jan is 23:30 in Berlin (UTC+1). + const at = new Date("2026-01-01T22:30:00.000Z"); + expect( + coveringStatsExternalId( + { type: "PULSE", measuredAt: at }, + "Europe/Berlin", + ), + ).toBe("stats:HKQuantityTypeIdentifierHeartRate:2026-01-01T23"); + // 23:30 UTC is already 2 Jan in Berlin. + const late = new Date("2026-01-01T23:30:00.000Z"); + expect( + coveringStatsExternalId( + { type: "RESPIRATORY_RATE", measuredAt: late }, + "Europe/Berlin", + ), + ).toBe("stats:HKQuantityTypeIdentifierRespiratoryRate:2026-01-02"); + expect( + coveringStatsExternalId( + { type: "WEIGHT", measuredAt: at }, + "Europe/Berlin", + ), + ).toBeNull(); + }); +}); + +function client(live: Array<{ type: string; externalId: string }>) { + const findMany = vi.fn(async () => live); + return { + findMany, + client: { + measurement: { findMany }, + user: { + findUnique: vi.fn(async () => ({ timezone: "UTC" })), + }, + } as unknown as Parameters[0], + }; +} + +describe("findFoldedWindowDuplicates", () => { + const old = new Date("2026-03-01T10:15:00.000Z"); + + it("asks nothing when no row is old enough", async () => { + const { client: c, findMany } = client([]); + const found = await findFoldedWindowDuplicates( + c, + "u1", + [raw("PULSE", new Date(NOW.getTime() - DAY))], + { now: NOW }, + ); + expect(found.size).toBe(0); + expect(findMany).not.toHaveBeenCalled(); + }); + + it("returns the rows a live `stats:` row covers, in one live-only read", async () => { + const { client: c, findMany } = client([ + { + type: "PULSE", + externalId: "stats:HKQuantityTypeIdentifierHeartRate:2026-03-01T10", + }, + ]); + const found = await findFoldedWindowDuplicates( + c, + "u1", + [ + raw("PULSE", old), + raw("PULSE", new Date("2026-03-01T11:15:00.000Z"), { + externalId: "uuid-2", + }), + raw("WEIGHT", old), + ], + { now: NOW }, + ); + expect([...found]).toEqual([0]); + expect(findMany).toHaveBeenCalledTimes(1); + const where = ( + findMany.mock.calls[0] as unknown as [{ where: Record }] + )[0].where; + expect(where.deletedAt).toBeNull(); + expect(where.source).toBe("APPLE_HEALTH"); + }); +}); + +describe("findCompactionTombstones", () => { + const measuredAt = new Date("2026-03-01T10:15:00.000Z"); + const covering = [ + { + type: "PULSE", + externalId: "stats:HKQuantityTypeIdentifierHeartRate:2026-03-01T10", + }, + ]; + + it("takes a tombstone deleted after the fold threshold under a live `stats:` row", async () => { + const { client: c } = client(covering); + const found = await findCompactionTombstones(c, "u1", "UTC", [ + { + ...raw("PULSE", measuredAt), + deletedAt: new Date(measuredAt.getTime() + 91 * DAY), + }, + ]); + expect([...found]).toEqual([0]); + }); + + it("leaves a person's deletion of a then-recent sample alone, even under a `stats:` row", async () => { + const { client: c } = client(covering); + const found = await findCompactionTombstones(c, "u1", "UTC", [ + { + ...raw("PULSE", measuredAt), + deletedAt: new Date(measuredAt.getTime() + 2 * DAY), + }, + ]); + expect(found.size).toBe(0); + }); + + it("leaves a dense tombstone alone when only a pre-hourly daily row covers it", async () => { + const { client: c } = client([ + { + type: "PULSE", + externalId: "stats:HKQuantityTypeIdentifierHeartRate:2026-03-01", + }, + ]); + const found = await findCompactionTombstones(c, "u1", "UTC", [ + { + ...raw("PULSE", measuredAt), + deletedAt: new Date(measuredAt.getTime() + 91 * DAY), + }, + ]); + expect(found.size).toBe(0); + }); + + it("leaves a live row alone", async () => { + const { client: c } = client(covering); + const found = await findCompactionTombstones(c, "u1", "UTC", [ + { ...raw("PULSE", measuredAt), deletedAt: null }, + ]); + expect(found.size).toBe(0); + }); +}); diff --git a/src/lib/measurements/consolidate-daily-mean.ts b/src/lib/measurements/consolidate-daily-mean.ts index 9b90312a4..46d8f27cf 100644 --- a/src/lib/measurements/consolidate-daily-mean.ts +++ b/src/lib/measurements/consolidate-daily-mean.ts @@ -31,13 +31,13 @@ * Because mean types are NOT in `CUMULATIVE_HK_TYPES`, the daily * read path averages over the single stats row (count = 1) and * returns it unchanged — no reader change needed. - * 5. SOFT-DELETE (set `deletedAt`) the per-sample rows — they stay in - * the table as an audit/backup trail. Sub-day detail loss is - * accepted, matching the legacy-step consolidation choice. + * 5. DELETE the per-sample rows (since v1.42; they used to be + * soft-deleted and kept for the tombstone retention). Sub-day detail + * loss is accepted, matching the legacy-step consolidation choice. * * Idempotent: the discovery query matches only users still holding live * per-sample mean-type rows, so a second run converges to zero work. - * Within a run, soft-deleted rows are excluded from the scan, and the + * Within a run, already-removed rows are excluded from the scan, and the * single minted stats row is excluded by the `NOT externalId LIKE * 'stats:%'` predicate so it is never re-folded. * @@ -369,12 +369,14 @@ export async function consolidateDailyMean( }, }); - // Soft-delete the per-sample rows in the same transaction — - // tombstone, never hard-delete; they remain as an audit trail and - // drop off the live read + this pass's re-run discovery. - const del = await tx.measurement.updateMany({ + // Delete the per-sample rows in the same transaction. + // + // v1.42 — deleted outright, where they used to be tombstoned for the + // 75-day retention. No client ever pulled them; the tombstone only + // held a re-upload of a folded sample off, which the `folded_window` + // ingest guard (`folded-window.ts`) now does from the live daily row. + const del = await tx.measurement.deleteMany({ where: { id: { in: sourceRowIds }, deletedAt: null }, - data: { deletedAt: new Date() }, }); removed = del.count; }); diff --git a/src/lib/measurements/dense-intraday-retention.ts b/src/lib/measurements/dense-intraday-retention.ts index e5e939613..f13ec2fb9 100644 --- a/src/lib/measurements/dense-intraday-retention.ts +++ b/src/lib/measurements/dense-intraday-retention.ts @@ -29,7 +29,8 @@ * widened from 36 hours to the retention window, and the fold grain * set to the LOCAL HOUR: per-sample rows OLDER than the window fold * to one `stats::T` hourly-mean row per user × - * type × local hour and the raw rows are tombstoned; rows INSIDE the + * type × local hour and the raw rows are deleted (tombstoned before + * v1.42); rows INSIDE the * window stay raw so the Stress engine always has its per-sample * inputs for the days it scores. * @@ -137,7 +138,7 @@ export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = /** * Retention bound (days). Per-sample dense-tier rows older than this window - * fold to hourly-mean rows and the raw rows tombstone; rows inside the + * fold to hourly-mean rows and the raw rows are deleted; rows inside the * window stay raw so the Stress engine has its intra-day inputs. * * 90 days comfortably covers the Stress engine's 7-day reference window and @@ -464,10 +465,10 @@ export interface DenseIntradayRetentionOptions { /** * Run the dense intra-day retention drain. Folds per-sample dense-tier * rows older than the retention window into hourly-mean `stats:` rows - * (user-local hours, anchored at local HH:30) and soft-deletes the raw - * rows; rows inside the window stay raw. Idempotent — re-invocation after - * a successful pass converges to zero work because the folded rows are - * soft-deleted and so excluded from the live scan, and the minted hourly + * (user-local hours, anchored at local HH:30) and deletes the raw rows + * (soft-deleted before v1.42); rows inside the window stay raw. Idempotent — + * re-invocation after a successful pass converges to zero work because the + * folded rows are gone from the live scan, and the minted hourly * rows are excluded by the `NOT startsWith('stats:')` predicate. * * Does NOT enforce any auth gate — the queue handler owns that concern. @@ -655,19 +656,23 @@ export async function runDenseIntradayRetention( retiredDaily = true; } - // Soft-delete the out-of-window per-sample rows in the same - // transaction — tombstone, never hard-delete; they remain until - // the tombstone-retention prune and drop off the live read + this - // pass's re-run discovery. EXCLUDE the adopted canonical rows: a - // per-sample row that happened to fall on an hourly anchor is the - // row just adopted as that hour's mean, so tombstoning it would - // erase the fold. - const del = await tx.measurement.updateMany({ + // Delete the out-of-window per-sample rows in the same + // transaction. EXCLUDE the adopted canonical rows: a per-sample row + // that happened to fall on an hourly anchor is the row just adopted + // as that hour's mean, so removing it would erase the fold. + // + // v1.42 — deleted outright, like the cumulative drain's rows, + // where they used to be tombstoned for the 75-day retention. No + // client ever pulled these rows, so the tombstone told nobody + // anything; it only held the re-upload of a folded sample off, + // and the `folded_window` ingest guard (`folded-window.ts`) does + // that now from the live hourly row. The tombstones made up most + // of the table and its indexes. + const del = await tx.measurement.deleteMany({ where: { id: { in: sourceRowIds, notIn: canonicalRowIds }, deletedAt: null, }, - data: { deletedAt: new Date() }, }); removed = del.count; }); @@ -777,7 +782,7 @@ export async function runDenseIntradayRetention( // not absorb, or a derived-resting upsert that races a native row — is // logged and STEPPED OVER so the global walk keeps draining every other // user / type / day. The failed day keeps its raw rows live (the fold - // soft-deletes only on a committed transaction), so the next nightly run + // deletes only on a committed transaction), so the next nightly run // retries it. Without this, one poisoned day aborted the whole walk and // stranded every later account. onBucketError: ({ userId, type, dateKey, error }) => { diff --git a/src/lib/measurements/folded-window.ts b/src/lib/measurements/folded-window.ts index 97291771b..2aeff4167 100644 --- a/src/lib/measurements/folded-window.ts +++ b/src/lib/measurements/folded-window.ts @@ -1,5 +1,5 @@ /** - * The `folded_window` ingest guard (v1.42). + * The `folded_window` ingest guard and the compaction-tombstone class (v1.42). * * The dense-intraday consolidation folds the raw Apple Health samples of a * DENSE type into one `stats:` row per local hour once they are older than @@ -12,27 +12,51 @@ * live `stats:` row covers its hour or day. The batch route and the ZIP import * apply the same rule. * + * The same question, asked of a tombstone, names the compaction tombstones + * ("class A"): a soft-deleted raw sample that was already old enough to fold + * when it was deleted, and whose hour or day a live `stats:` row covers. The + * backlog purge, the sync feed and the restore all classify through + * {@link findCompactionTombstones}, so the three can never disagree about + * which rows are compaction leftovers and which are a person's deletions. + * + * Which `stats:` row covers a sample is decided by its externalId, built from + * the account's current timezone exactly as the fold builds it + * (`stats::T` for a dense hour, `stats::` + * for a mean day). A dense day still folded at the pre-v1.28.31 daily grain is + * deliberately NOT covered: the one-shot hourly rebuild reads those days' + * tombstones, so they are neither refused at ingest by this rule nor purged. + * An account whose timezone changed after a fold may see an old day keyed in + * the other zone; such a row is then simply not recognised, which is the + * pre-v1.42 behaviour, never a wrong refusal. + * * Health Connect does not use this: its importer writes the folded shape * itself. - * - * Contract stub: the rules and the pure candidate check are final; the lookup - * that finds the covering `stats:` rows returns nothing until it is built, so - * nothing is refused yet. */ import type { MeasurementSource, MeasurementType, Prisma, } from "@/generated/prisma/client"; -import { HIGH_FREQUENCY_MEAN_TYPES } from "@/lib/measurements/apple-health-mapping"; -import { CONSOLIDATION_GRACE_CUTOFF_HOURS } from "@/lib/measurements/consolidation-tz"; +import { + HIGH_FREQUENCY_MEAN_TYPES, + dailyStatsExternalId, + hkIdentifierForType, +} from "@/lib/measurements/apple-health-mapping"; +import { resolveUserTimezone } from "@/lib/measurements/consolidation-base"; +import { + CONSOLIDATION_GRACE_CUTOFF_HOURS, + dayKeyForUserTz, + hourOfDayForUserTz, +} from "@/lib/measurements/consolidation-tz"; import { DENSE_INTRADAY_RETENTION_DAYS, DENSE_INTRADAY_RETENTION_TYPES, + hourlyStatsExternalId, } from "@/lib/measurements/dense-intraday-retention"; /** The per-entry status reason the batch result carries for such a row. */ export type FoldedWindowReason = "folded_window"; +export const FOLDED_WINDOW_REASON: FoldedWindowReason = "folded_window"; const HOUR_MS = 3_600_000; @@ -50,6 +74,12 @@ export const FOLDED_WINDOW_RULES = [ }, ] as const; +/** Every type either rule folds. */ +export const FOLDED_TYPES: readonly MeasurementType[] = [ + ...DENSE_INTRADAY_RETENTION_TYPES, + ...HIGH_FREQUENCY_MEAN_TYPES, +]; + /** The fields of an incoming row the guard reads. */ export interface FoldedWindowCandidateRow { type: MeasurementType; @@ -61,14 +91,19 @@ export interface FoldedWindowCandidateRow { /** * Whether a row is old enough, and of the right kind, for its hour or day to * have been folded already. Pure: whether a live `stats:` row actually covers - * it is the lookup's question. + * it is the lookup's question. `now` is the instant the age is measured at: + * the request time for an upload, the deletion time for a tombstone. */ export function isFoldedWindowCandidate( row: FoldedWindowCandidateRow, now: Date = new Date(), ): boolean { if (row.source !== "APPLE_HEALTH") return false; - if (row.externalId?.startsWith("stats:")) return false; + if (row.externalId === null) return false; + if (row.externalId.startsWith("stats:")) return false; + // A collision retirement (`reconcile-external-measurement.ts`) parks the + // losing row at the epoch under a `retired:` id. It is not a folded sample. + if (row.externalId.startsWith("retired:")) return false; const age = now.getTime() - row.measuredAt.getTime(); return FOLDED_WINDOW_RULES.some( (rule) => rule.types.has(row.type) && age > rule.foldedAfterMs, @@ -76,16 +111,140 @@ export function isFoldedWindowCandidate( } /** - * Indexes (into `rows`) of the candidates whose hour or day a live `stats:` - * row already covers. Stub: returns none. + * The externalId of the `stats:` row the fold writes for a sample's hour + * (dense types) or day (mean types), in `tz`. Null for a type outside both + * rules or without an HK identifier. Pure. + */ +export function coveringStatsExternalId( + row: Pick, + tz: string, +): string | null { + const hk = hkIdentifierForType(row.type); + if (!hk) return null; + const dateKey = dayKeyForUserTz(row.measuredAt, tz); + if (DENSE_INTRADAY_RETENTION_TYPES.has(row.type)) { + return hourlyStatsExternalId( + hk, + dateKey, + hourOfDayForUserTz(row.measuredAt, tz), + ); + } + if (HIGH_FREQUENCY_MEAN_TYPES.has(row.type)) { + return dailyStatsExternalId(hk, dateKey); + } + return null; +} + +type LookupClient = { + measurement: Pick; + user: Pick; +}; + +/** The account's timezone, resolved the way the fold resolves it. */ +export async function loadFoldTimezone( + client: Pick, + userId: string, +): Promise { + const user = await client.user.findUnique({ + where: { id: userId }, + select: { timezone: true }, + }); + return resolveUserTimezone(user?.timezone ?? null); +} + +/** + * Indexes (into `rows`) whose covering `stats:` row is live. One query over + * the unique `(user_id, type, source, external_id)` index for the whole set. + */ +async function coveredIndexes( + client: Pick, + userId: string, + tz: string, + rows: ReadonlyArray<{ + index: number; + type: MeasurementType; + measuredAt: Date; + }>, +): Promise> { + const covered = new Set(); + if (rows.length === 0) return covered; + const keyOf = new Map(); + const ids = new Set(); + const types = new Set(); + for (const row of rows) { + const id = coveringStatsExternalId(row, tz); + if (!id) continue; + keyOf.set(row.index, `${row.type}|${id}`); + ids.add(id); + types.add(row.type); + } + if (ids.size === 0) return covered; + const live = await client.measurement.findMany({ + where: { + userId, + type: { in: [...types] }, + source: "APPLE_HEALTH", + externalId: { in: [...ids] }, + deletedAt: null, + }, + select: { type: true, externalId: true }, + }); + const liveKeys = new Set(live.map((row) => `${row.type}|${row.externalId}`)); + for (const [index, key] of keyOf) { + if (liveKeys.has(key)) covered.add(index); + } + return covered; +} + +/** + * Indexes (into `rows`) of the incoming rows whose hour or day a live `stats:` + * row already covers. Rows too young to have been folded are never looked up, + * so a batch of current samples costs no query at all. */ export async function findFoldedWindowDuplicates( - tx: Prisma.TransactionClient, + client: Pick & + Partial>, userId: string, rows: readonly FoldedWindowCandidateRow[], + options: { now?: Date; tz?: string } = {}, ): Promise> { - void tx; - void userId; - void rows; - return new Set(); + const now = options.now ?? new Date(); + const candidates = rows.flatMap((row, index) => + isFoldedWindowCandidate(row, now) + ? [{ index, type: row.type, measuredAt: row.measuredAt }] + : [], + ); + if (candidates.length === 0) return new Set(); + let tz = options.tz; + if (tz === undefined) { + if (!client.user) throw new Error("folded_window lookup needs a timezone"); + tz = await loadFoldTimezone({ user: client.user }, userId); + } + return coveredIndexes(client, userId, tz, candidates); +} + +/** A tombstone as the classifier reads it. */ +export interface TombstoneCandidateRow extends FoldedWindowCandidateRow { + deletedAt: Date | null; +} + +/** + * Indexes (into `rows`) of the compaction tombstones ("class A"): soft-deleted + * raw Apple Health samples of a folded type that were already past the fold + * threshold when they were deleted, and whose hour or day a live `stats:` row + * covers. Everything else, in particular a person's own deletions of recent + * samples, is class B and keeps the 75-day tombstone retention. + */ +export async function findCompactionTombstones( + client: Pick, + userId: string, + tz: string, + rows: readonly TombstoneCandidateRow[], +): Promise> { + const candidates = rows.flatMap((row, index) => + row.deletedAt !== null && isFoldedWindowCandidate(row, row.deletedAt) + ? [{ index, type: row.type, measuredAt: row.measuredAt }] + : [], + ); + return coveredIndexes(client, userId, tz, candidates); } diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index f75e2a53e..1e3c3bab2 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -46,6 +46,11 @@ import { MEASURED_AT_TOLERANCE_MS, } from "@/lib/measurements/cross-source-merge"; import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; +import { + FOLDED_WINDOW_REASON, + findFoldedWindowDuplicates, + isFoldedWindowCandidate, +} from "@/lib/measurements/folded-window"; import { insertNewMeasurementRows, type InsertedMeasurementRow, @@ -704,11 +709,46 @@ export async function streamParseExportXml( }); }; + // v1.42 — the `folded_window` guard, the same rule the batch route applies + // (`folded-window.ts`). A raw sample of a dense or mean type whose hour or + // day the consolidation has already folded into a live `stats:` row is left + // out: the fold deletes its raw rows outright since v1.42, so re-importing + // the export would otherwise put them back beside the mean that already + // accounts for them. One read per flush, none when nothing is old enough. + const withoutFoldedWindows = async ( + rows: PreparedMeasurement[], + ): Promise => { + if (rows.length === 0) return rows; + const now = new Date(); + const candidates = rows.map((row) => ({ + type: row.type, + source: "APPLE_HEALTH" as const, + externalId: row.externalId, + measuredAt: row.measuredAt, + })); + if (!candidates.some((row) => isFoldedWindowCandidate(row, now))) { + return rows; + } + const folded = await findFoldedWindowDuplicates( + prisma, + userId, + candidates, + { now, tz: userTimezone }, + ); + if (folded.size === 0) return rows; + return rows.filter((row, index) => { + if (!folded.has(index)) return true; + unknown[`${row.type}::${FOLDED_WINDOW_REASON}`] = + (unknown[`${row.type}::${FOLDED_WINDOW_REASON}`] ?? 0) + 1; + return false; + }); + }; + const flushSpotBatch = async (): Promise => { if (spotBatch.length === 0) return; const incoming = spotBatch.splice(0, spotBatch.length); - const chunk = await withoutManualMirrors( - await withoutHealthLogIds(incoming), + const chunk = await withoutFoldedWindows( + await withoutManualMirrors(await withoutHealthLogIds(incoming)), ); for (const row of incoming) { healthLogIdOf.delete(row); diff --git a/src/lib/measurements/live-types.ts b/src/lib/measurements/live-types.ts index ff7c52431..8df85fe1a 100644 --- a/src/lib/measurements/live-types.ts +++ b/src/lib/measurements/live-types.ts @@ -2,16 +2,26 @@ * Which measurement types an account has live rows of (v1.42). * * Several readers ask this ("which types exist?") on the request path and in - * nightly passes. Since v1.42 they ask through this one helper, so the query - * can move to a loose index scan over `measurements_live_covering_idx` - * without each caller re-learning the plan. + * nightly passes. A `groupBy` answers it by reading every live row of the + * account, which on a dense multi-year Apple Health history is a six-figure + * row set for a two-digit answer. This helper walks the partial + * `measurements_live_covering_idx (user_id, type, measured_at DESC) WHERE + * deleted_at IS NULL` (migration 0183) as a loose index scan instead: a + * recursive CTE that asks for the first type, then the first type greater + * than it, once per distinct type, each an index probe. * - * Contract stub: the signature is final; the body is the `groupBy` the callers - * run today. + * `MeasurementType` is a Postgres enum, so both `ORDER BY type` and + * `type > …` follow the enum's declaration order, which is the order the + * `groupBy … orderBy: { type: "asc" }` it replaces returned. + * + * A `source` filter is not in the index and is checked against the heap for + * the rows each probe visits; it stays correct, and is cheap for the sources + * that make up most of a type's rows. */ -import type { - MeasurementSource, - MeasurementType, +import { + Prisma, + type MeasurementSource, + type MeasurementType, } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; @@ -22,15 +32,38 @@ export async function listLiveMeasurementTypes( types?: readonly MeasurementType[]; } = {}, ): Promise { - const rows = await prisma.measurement.groupBy({ - by: ["type"], - where: { - userId, - deletedAt: null, - ...(options.source ? { source: options.source } : {}), - ...(options.types ? { type: { in: [...options.types] } } : {}), - }, - orderBy: { type: "asc" }, - }); + if (options.types && options.types.length === 0) return []; + // Both fragments are parameter-bound; the enum casts are literal SQL. + const sourceFilter = options.source + ? Prisma.sql`AND "source" = ${options.source}::"measurement_source"` + : Prisma.empty; + const typeFilter = options.types + ? Prisma.sql`AND "type" = ANY(${[...options.types]}::"measurement_type"[])` + : Prisma.empty; + const rows = await prisma.$queryRaw>` + WITH RECURSIVE live_types AS ( + ( + SELECT "type" + FROM "measurements" + WHERE "user_id" = ${userId} AND "deleted_at" IS NULL + ${sourceFilter} ${typeFilter} + ORDER BY "type" + LIMIT 1 + ) + UNION ALL + SELECT ( + SELECT m."type" + FROM "measurements" m + WHERE m."user_id" = ${userId} AND m."deleted_at" IS NULL + AND m."type" > live_types."type" + ${sourceFilter} ${typeFilter} + ORDER BY m."type" + LIMIT 1 + ) + FROM live_types + WHERE live_types."type" IS NOT NULL + ) + SELECT "type"::text AS "type" FROM live_types WHERE "type" IS NOT NULL + `; return rows.map((row) => row.type); } diff --git a/tests/integration/compaction-tombstones.test.ts b/tests/integration/compaction-tombstones.test.ts new file mode 100644 index 000000000..6fd879a8f --- /dev/null +++ b/tests/integration/compaction-tombstones.test.ts @@ -0,0 +1,693 @@ +/** + * v1.42 — compaction tombstones, end to end against Postgres. + * + * The folds used to soft-delete the raw Apple Health samples they folded into + * an hourly or daily `stats:` row ("class A"). Since v1.42: + * - the folds delete those rows outright; + * - a re-upload of such a sample is a `duplicate` with reason + * `folded_window` while a live `stats:` row covers its hour or day; + * - the backlog purge removes the class-A tombstones already in the table, + * in batches, one account at a time under the restore lock, and leaves + * every other tombstone ("class B") to the 75-day retention; + * - `/api/sync/changes` no longer reports class-A tombstones; + * - a restore does not write class-A tombstones back. + * Plus the exact-duplicate prefilter (no transaction for a re-sent batch, and + * two identical batches racing insert every row once), the live-type + * discovery helper, and the PR-detection watermark. + */ +import { NextRequest } from "next/server"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const NOW = Date.now(); +/** An hour 120 days back, on the hour, in UTC (the accounts' zone). */ +const OLD_HOUR = new Date(Math.floor((NOW - 120 * DAY) / HOUR) * HOUR); +const OLD_DAY = OLD_HOUR.toISOString().slice(0, 10); +const OLD_HH = OLD_HOUR.toISOString().slice(11, 13); +const HOURLY_ID = `stats:HKQuantityTypeIdentifierHeartRate:${OLD_DAY}T${OLD_HH}`; +const at = (minutes: number) => new Date(OLD_HOUR.getTime() + minutes * 60_000); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterAll(async () => { + await truncateAllTables(getPrismaClient()); +}); + +async function account(id: string, signIn = false) { + const prisma = getPrismaClient(); + await prisma.user.create({ + data: { + id, + username: id, + email: `${id}@example.test`, + role: "ADMIN", + timezone: "UTC", + }, + }); + if (signIn) { + const session = await prisma.session.create({ + data: { userId: id, expiresAt: new Date(NOW + HOUR) }, + }); + cookieJar.set("healthlog_session", session.id); + } +} + +/** + * One account's mix: the live hourly row, five class-A tombstones under it, + * and class-B rows the purge must leave alone. + */ +async function seedMix(userId: string) { + const prisma = getPrismaClient(); + const compactedAt = new Date(NOW - DAY); + await prisma.measurement.create({ + data: { + id: `${userId}-hourly`, + userId, + type: "PULSE", + value: 62, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: at(30), + externalId: HOURLY_ID, + }, + }); + await prisma.measurement.createMany({ + data: Array.from({ length: 5 }, (_, i) => ({ + id: `${userId}-classA-${i}`, + userId, + type: "PULSE" as const, + value: 60 + i, + unit: "bpm", + source: "APPLE_HEALTH" as const, + measuredAt: at(1 + i), + externalId: `uuid-${userId}-a-${i}`, + deletedAt: compactedAt, + })), + }); + // A mean-type day folded into its daily row: class A too. + const meanAt = new Date(NOW - 5 * DAY); + const meanDay = meanAt.toISOString().slice(0, 10); + await prisma.measurement.create({ + data: { + id: `${userId}-daily-rr`, + userId, + type: "RESPIRATORY_RATE", + value: 14, + unit: "count/min", + source: "APPLE_HEALTH", + measuredAt: new Date(`${meanDay}T12:00:00.000Z`), + externalId: `stats:HKQuantityTypeIdentifierRespiratoryRate:${meanDay}`, + }, + }); + await prisma.measurement.create({ + data: { + id: `${userId}-classA-rr`, + userId, + type: "RESPIRATORY_RATE", + value: 15, + unit: "count/min", + source: "APPLE_HEALTH", + measuredAt: meanAt, + externalId: `uuid-${userId}-rr`, + deletedAt: new Date(meanAt.getTime() + 2 * DAY), + }, + }); + await prisma.measurement.createMany({ + data: [ + // A person deleted this sample the day after it was taken, before any + // fold could have reached it: class B although the hour is covered. + { + id: `${userId}-classB-user`, + userId, + type: "PULSE", + value: 99, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: at(40), + externalId: `uuid-${userId}-b-user`, + deletedAt: new Date(at(40).getTime() + DAY), + }, + // A dense day still at the pre-hourly daily grain: the one-shot + // hourly rebuild reads these tombstones, so they stay. + { + id: `${userId}-daily-legacy`, + userId, + type: "PULSE", + value: 61, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: new Date(OLD_HOUR.getTime() - 3 * DAY + 12 * HOUR), + externalId: `stats:HKQuantityTypeIdentifierHeartRate:${new Date( + OLD_HOUR.getTime() - 3 * DAY, + ) + .toISOString() + .slice(0, 10)}`, + }, + { + id: `${userId}-classB-legacy`, + userId, + type: "PULSE", + value: 61, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: new Date(OLD_HOUR.getTime() - 3 * DAY + 10 * HOUR), + externalId: `uuid-${userId}-b-legacy`, + deletedAt: compactedAt, + }, + // A hand-entered reading someone deleted. + { + id: `${userId}-classB-manual`, + userId, + type: "WEIGHT", + value: 80, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(NOW - 2 * DAY), + deletedAt: new Date(NOW - DAY), + }, + // A live raw sample. + { + id: `${userId}-live`, + userId, + type: "PULSE", + value: 70, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: new Date(NOW - 2 * DAY), + externalId: `uuid-${userId}-live`, + }, + ], + }); +} + +const CLASS_A = (userId: string) => [ + ...Array.from({ length: 5 }, (_, i) => `${userId}-classA-${i}`), + `${userId}-classA-rr`, +]; +const KEPT = (userId: string) => [ + `${userId}-hourly`, + `${userId}-daily-rr`, + `${userId}-classB-user`, + `${userId}-daily-legacy`, + `${userId}-classB-legacy`, + `${userId}-classB-manual`, + `${userId}-live`, +]; + +async function ids(userId: string): Promise { + const rows = await getPrismaClient().measurement.findMany({ + where: { userId }, + select: { id: true }, + orderBy: { id: "asc" }, + }); + return rows.map((row) => row.id); +} + +describe("compaction-tombstone purge", () => { + it("deletes class A in batches and leaves every class-B row", async () => { + const prisma = getPrismaClient(); + await account("purge-a"); + await seedMix("purge-a"); + const { purgeCompactionTombstones } = + await import("@/lib/jobs/compaction-tombstone-purge"); + + const outcome = await purgeCompactionTombstones(prisma, { + batchSize: 2, + scanPageSize: 3, + pauseMs: 0, + }); + + expect(outcome.deleted).toBe(6); + expect(outcome.drained).toBe(true); + expect(outcome.deferredAccounts).toBe(0); + expect(await ids("purge-a")).toEqual([...KEPT("purge-a")].sort()); + + // Idempotent: a second run finds nothing. + const again = await purgeCompactionTombstones(prisma, { pauseMs: 0 }); + expect(again.deleted).toBe(0); + expect(again.drained).toBe(true); + }); + + it("stops at the batch cap and says so; the next run resumes", async () => { + const prisma = getPrismaClient(); + await account("purge-cap"); + await seedMix("purge-cap"); + const { purgeCompactionTombstones } = + await import("@/lib/jobs/compaction-tombstone-purge"); + + const first = await purgeCompactionTombstones(prisma, { + batchSize: 2, + maxBatches: 1, + pauseMs: 0, + }); + expect(first.deleted).toBe(2); + expect(first.drained).toBe(false); + + const rest = await purgeCompactionTombstones(prisma, { pauseMs: 0 }); + expect(rest.deleted).toBe(4); + expect(rest.drained).toBe(true); + }); + + it("skips an account under restore without waiting and purges the others", async () => { + const prisma = getPrismaClient(); + await account("purge-restoring"); + await account("purge-other"); + await seedMix("purge-restoring"); + await seedMix("purge-other"); + const { takeRestoreLock } = await import("@/lib/export/restore-lock"); + const { purgeCompactionTombstones } = + await import("@/lib/jobs/compaction-tombstone-purge"); + + let holding!: () => void; + const isHolding = new Promise((resolve) => (holding = resolve)); + const restore = prisma + .$transaction( + async (tx) => { + await takeRestoreLock(tx, "purge-restoring"); + holding(); + await new Promise((resolve) => setTimeout(resolve, 3_000)); + throw new Error("rolled back"); + }, + { timeout: 30_000 }, + ) + .catch(() => undefined); + await isHolding; + + const started = Date.now(); + const outcome = await purgeCompactionTombstones(prisma, { pauseMs: 0 }); + expect(Date.now() - started).toBeLessThan(1_500); + await restore; + + expect(outcome.deleted).toBe(6); + expect(outcome.deferredAccounts).toBe(1); + expect(outcome.deferredUserIds).toEqual(["purge-restoring"]); + expect(await ids("purge-other")).toEqual([...KEPT("purge-other")].sort()); + for (const id of CLASS_A("purge-restoring")) { + expect(await ids("purge-restoring")).toContain(id); + } + + const next = await purgeCompactionTombstones(prisma, { pauseMs: 0 }); + expect(next.deleted).toBe(6); + expect(await ids("purge-restoring")).toEqual( + [...KEPT("purge-restoring")].sort(), + ); + }, 60_000); +}); + +describe("/api/sync/changes", () => { + it("leaves compaction tombstones out and still reports a person's deletion", async () => { + await account("feed", true); + await seedMix("feed"); + const { GET } = await import("@/app/api/sync/changes/route"); + const tombstones: string[] = []; + const upserts: string[] = []; + let cursor: string | undefined; + for (;;) { + const query = cursor + ? `?limit=3&cursor=${encodeURIComponent(cursor)}` + : "?limit=3"; + const res = await GET( + new NextRequest(`http://localhost/api/sync/changes${query}`), + ); + expect(res.status).toBe(200); + const page = ( + (await res.json()) as { + data: { + cursor: string; + hasMore: boolean; + changes: { + measurements: { + upserts: Array<{ id: string }>; + tombstones: Array<{ id: string }>; + }; + }; + }; + } + ).data; + tombstones.push(...page.changes.measurements.tombstones.map((t) => t.id)); + upserts.push(...page.changes.measurements.upserts.map((u) => u.id)); + cursor = page.cursor; + if (!page.hasMore) break; + } + expect(tombstones.sort()).toEqual( + ["feed-classB-legacy", "feed-classB-manual", "feed-classB-user"].sort(), + ); + for (const id of CLASS_A("feed")) expect(tombstones).not.toContain(id); + expect(upserts).toContain("feed-hourly"); + }); +}); + +describe("restore", () => { + it("does not write class-A tombstones back", async () => { + const prisma = getPrismaClient(); + await account("restore-a", true); + await seedMix("restore-a"); + const { storeBackupBlob } = await import("@/lib/export/store-backup-blob"); + const { streamFullBackupJson } = + await import("@/lib/export/full-backup-stream"); + const { id: backupId } = await storeBackupBlob( + prisma, + { userId: "restore-a", type: "WEEKLY_AUTO" }, + (write) => + streamFullBackupJson(prisma, "restore-a", write, { + purpose: "disaster-recovery", + }), + ); + const { POST } = await import("./restore-job-driver"); + const res = await POST( + new Request(`http://localhost/api/admin/backups/${backupId}/restore`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ confirm: "RESTORE" }), + }) as never, + { params: Promise.resolve({ id: backupId }) }, + ); + expect(res.status).toBe(200); + + const restored = await ids("restore-a"); + for (const id of CLASS_A("restore-a")) expect(restored).not.toContain(id); + // The class-B tombstones inside the retention come back; the user + // deletion from 119 days ago is past it and is skipped as before. + expect(restored).toContain("restore-a-classB-manual"); + expect(restored).toContain("restore-a-classB-legacy"); + expect(restored).toContain("restore-a-hourly"); + expect(restored).toContain("restore-a-live"); + }, 120_000); +}); + +type BatchBody = { + data: { + inserted: number; + duplicates: number; + entries: Array<{ index: number; status: string; reason?: string }>; + }; +}; + +function pulseEntry(externalId: string, when: Date, value = 60) { + return { + hkIdentifier: "HKQuantityTypeIdentifierHeartRate", + value, + unit: "count/min", + startDate: when.toISOString(), + endDate: when.toISOString(), + externalId, + }; +} + +async function postBatch(entries: unknown[]): Promise { + const { POST } = await import("@/app/api/measurements/batch/route"); + const res = await POST( + new NextRequest("http://localhost/api/measurements/batch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ entries }), + }), + ); + expect(res.status).toBe(200); + return ((await res.json()) as BatchBody).data; +} + +describe("folds delete, and the folded_window guard holds the re-upload off", () => { + it("dense retention removes the raw rows outright; re-sending one is a folded duplicate", async () => { + const prisma = getPrismaClient(); + await account("fold-dense", true); + const raws = Array.from({ length: 4 }, (_, i) => + pulseEntry(`uuid-fold-${i}`, at(5 + i * 10), 60 + i), + ); + expect((await postBatch(raws)).inserted).toBe(4); + + const { runDenseIntradayRetention } = + await import("@/lib/measurements/dense-intraday-retention"); + await runDenseIntradayRetention(prisma, { + userId: "fold-dense", + log: () => {}, + }); + + const rows = await prisma.measurement.findMany({ + where: { userId: "fold-dense", type: "PULSE" }, + select: { externalId: true, deletedAt: true }, + }); + expect(rows).toEqual([{ externalId: HOURLY_ID, deletedAt: null }]); + + const again = await postBatch(raws); + expect(again.entries.every((e) => e.reason === "folded_window")).toBe(true); + expect(again.duplicates).toBe(4); + expect( + await prisma.measurement.count({ + where: { userId: "fold-dense", type: "PULSE" }, + }), + ).toBe(1); + }); + + it("mean consolidation removes the raw rows outright", async () => { + const prisma = getPrismaClient(); + await account("fold-mean", true); + const when = new Date(NOW - 5 * DAY); + await postBatch([ + { + hkIdentifier: "HKQuantityTypeIdentifierRespiratoryRate", + value: 14, + unit: "count/min", + startDate: when.toISOString(), + endDate: when.toISOString(), + externalId: "uuid-rr-1", + }, + ]); + const { consolidateDailyMean } = + await import("@/lib/measurements/consolidate-daily-mean"); + await consolidateDailyMean(prisma, { userId: "fold-mean", log: () => {} }); + const rows = await prisma.measurement.findMany({ + where: { userId: "fold-mean", type: "RESPIRATORY_RATE" }, + select: { externalId: true, deletedAt: true }, + }); + expect(rows).toHaveLength(1); + expect(rows[0].externalId).toMatch(/^stats:/); + expect(rows[0].deletedAt).toBeNull(); + }); +}); + +describe("exact-duplicate prefilter against Postgres", () => { + it("answers a re-sent 500-entry batch without a transaction", async () => { + const prisma = getPrismaClient(); + await account("prefilter", true); + const entries = Array.from({ length: 500 }, (_, i) => ({ + hkIdentifier: "HKQuantityTypeIdentifierBodyMass", + value: 70 + (i % 10), + unit: "kg", + startDate: new Date(NOW - DAY - i * 60_000).toISOString(), + endDate: new Date(NOW - DAY - i * 60_000).toISOString(), + externalId: `uuid-weight-${i}`, + })); + expect((await postBatch(entries)).inserted).toBe(500); + + const spy = vi.spyOn(prisma, "$transaction"); + const again = await postBatch(entries); + expect(again.duplicates).toBe(500); + expect(again.entries.every((e) => e.status === "duplicate")).toBe(true); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); + + it("11: two identical batches racing insert every row exactly once", async () => { + const prisma = getPrismaClient(); + await account("race", true); + const entries = Array.from({ length: 50 }, (_, i) => ({ + hkIdentifier: "HKQuantityTypeIdentifierBodyMass", + value: 70, + unit: "kg", + startDate: new Date(NOW - DAY - i * 60_000).toISOString(), + endDate: new Date(NOW - DAY - i * 60_000).toISOString(), + externalId: `uuid-race-${i}`, + })); + const [a, b] = await Promise.all([postBatch(entries), postBatch(entries)]); + expect(a.inserted + b.inserted).toBe(50); + expect(a.duplicates + b.duplicates).toBe(50); + for (let i = 0; i < 50; i++) { + expect([a.entries[i].status, b.entries[i].status].sort()).toEqual([ + "duplicate", + "inserted", + ]); + } + expect(await prisma.measurement.count({ where: { userId: "race" } })).toBe( + 50, + ); + }); +}); + +describe("listLiveMeasurementTypes", () => { + it("answers what the GROUP BY answered, with and without filters", async () => { + const prisma = getPrismaClient(); + await account("types"); + await seedMix("types"); + const { listLiveMeasurementTypes } = + await import("@/lib/measurements/live-types"); + const grouped = async (where: Record) => + ( + await prisma.measurement.groupBy({ + by: ["type"], + where: { userId: "types", deletedAt: null, ...where }, + orderBy: { type: "asc" }, + }) + ).map((row) => row.type); + + expect(await listLiveMeasurementTypes("types")).toEqual(await grouped({})); + expect(await listLiveMeasurementTypes("types")).toEqual([ + "PULSE", + "RESPIRATORY_RATE", + ]); + expect( + await listLiveMeasurementTypes("types", { source: "MANUAL" }), + ).toEqual([]); + expect( + await listLiveMeasurementTypes("types", { + types: ["RESPIRATORY_RATE", "WEIGHT"], + }), + ).toEqual(["RESPIRATORY_RATE"]); + expect(await listLiveMeasurementTypes("types", { types: [] })).toEqual([]); + }); +}); + +describe("PR-detection watermark", () => { + it("re-scans only accounts whose readings or workouts changed", async () => { + const prisma = getPrismaClient(); + await account("pr-fresh"); + await account("pr-stale"); + await prisma.measurement.create({ + data: { + userId: "pr-fresh", + type: "WEIGHT", + value: 80, + unit: "kg", + measuredAt: new Date(NOW - DAY), + }, + }); + await prisma.measurement.create({ + data: { + userId: "pr-stale", + type: "WEIGHT", + value: 80, + unit: "kg", + measuredAt: new Date(NOW - DAY), + updatedAt: new Date(NOW - 3 * HOUR), + }, + }); + const { listPrDetectionFallbackUserIds } = + await import("@/lib/jobs/pr-detection"); + expect( + await listPrDetectionFallbackUserIds(prisma, new Date(NOW - 35 * 60_000)), + ).toEqual(["pr-fresh"]); + expect( + await listPrDetectionFallbackUserIds(prisma, new Date(NOW - DAY)), + ).toEqual(["pr-fresh", "pr-stale"]); + }); +}); + +describe("measurement maintenance", () => { + it("vacuums and rebuilds every index, one at a time, largest first", async () => { + await account("maint"); + await seedMix("maint"); + const { Client } = await import("pg"); + const { MAINTENANCE_SESSION_OPTIONS, runMeasurementMaintenance } = + await import("@/lib/jobs/measurement-maintenance"); + const client = new Client({ + connectionString: process.env.DATABASE_URL, + options: MAINTENANCE_SESSION_OPTIONS, + }); + await client.connect(); + try { + const { rows } = await client.query<{ setting: string }>( + "SELECT current_setting('statement_timeout') AS setting", + ); + expect(rows[0].setting).toBe("0"); + const summary = await runMeasurementMaintenance(client, { + vacuum: true, + reindex: true, + }); + expect(summary.outcome).toBe("completed"); + expect(summary.steps[0]).toMatchObject({ step: "vacuum", ok: true }); + const rebuilt = summary.steps.filter((s) => s.step === "reindex"); + expect(rebuilt.length).toBeGreaterThan(5); + expect(rebuilt.every((s) => s.ok)).toBe(true); + expect(rebuilt.map((s) => s.target)).toContain( + "measurements_live_covering_idx", + ); + const sizes = rebuilt.map((s) => s.bytesBefore); + expect([...sizes].sort((a, b) => b - a)).toEqual(sizes); + } finally { + await client.end(); + } + }, 120_000); + + it("drops a leftover invalid copy before rebuilding", async () => { + await account("maint-copy"); + const { Client } = await import("pg"); + const { MAINTENANCE_SESSION_OPTIONS, runMeasurementMaintenance } = + await import("@/lib/jobs/measurement-maintenance"); + const client = new Client({ + connectionString: process.env.DATABASE_URL, + options: MAINTENANCE_SESSION_OPTIONS, + }); + await client.connect(); + try { + // What an interrupted REINDEX CONCURRENTLY leaves: an index marked + // invalid under the `_ccnew` name. + await client.query( + `CREATE INDEX "measurements_external_id_idx_ccnew" ON "measurements" ("external_id")`, + ); + await client.query( + `UPDATE pg_index SET indisvalid = false WHERE indexrelid = '"measurements_external_id_idx_ccnew"'::regclass`, + ); + const summary = await runMeasurementMaintenance(client, { + vacuum: false, + reindex: true, + }); + expect(summary.outcome).toBe("completed"); + expect(summary.steps[0]).toMatchObject({ + step: "drop_invalid", + target: "measurements_external_id_idx_ccnew", + ok: true, + }); + const { rows } = await client.query( + `SELECT 1 FROM pg_class WHERE relname = 'measurements_external_id_idx_ccnew'`, + ); + expect(rows).toHaveLength(0); + } finally { + await client + .query(`DROP INDEX IF EXISTS "measurements_external_id_idx_ccnew"`) + .catch(() => {}); + await client.end(); + } + }, 120_000); +}); diff --git a/tests/integration/consolidate-daily-mean.test.ts b/tests/integration/consolidate-daily-mean.test.ts index 317e68a45..0fe63cef8 100644 --- a/tests/integration/consolidate-daily-mean.test.ts +++ b/tests/integration/consolidate-daily-mean.test.ts @@ -33,7 +33,7 @@ beforeEach(async () => { }); describe("consolidateDailyMean (real Postgres)", () => { - it("collapses per-sample rows into one daily MEAN and soft-deletes the originals", async () => { + it("collapses per-sample rows into one daily MEAN and deletes the originals", async () => { const prisma = getPrismaClient(); await prisma.measurement.createMany({ data: [ @@ -85,14 +85,12 @@ describe("consolidateDailyMean (real Postgres)", () => { expect(live[0].value).toBeCloseTo(1.2, 6); expect(live[0].externalId).toBe(SPEED_STATS_ID); - const tombstoned = await prisma.measurement.findMany({ - where: { - userId: TEST_USER_ID, - type: "WALKING_SPEED", - deletedAt: { not: null }, - }, + // v1.42 — the folded per-sample rows are deleted outright, not + // tombstoned: nothing of them is left in the table. + const all = await prisma.measurement.findMany({ + where: { userId: TEST_USER_ID, type: "WALKING_SPEED" }, }); - expect(tombstoned).toHaveLength(3); + expect(all).toHaveLength(1); }); it("leaves manual / Withings (non-APPLE_HEALTH) rows untouched", async () => { diff --git a/tests/integration/dense-intraday-retention.test.ts b/tests/integration/dense-intraday-retention.test.ts index 329d129dd..db955362e 100644 --- a/tests/integration/dense-intraday-retention.test.ts +++ b/tests/integration/dense-intraday-retention.test.ts @@ -39,7 +39,7 @@ beforeEach(async () => { }); describe("runDenseIntradayRetention (real Postgres)", () => { - it("folds out-of-window HRV per-sample rows into hourly MEANs and soft-deletes them", async () => { + it("folds out-of-window HRV per-sample rows into hourly MEANs and deletes them", async () => { const prisma = getPrismaClient(); await prisma.measurement.createMany({ data: [ @@ -103,6 +103,12 @@ describe("runDenseIntradayRetention (real Postgres)", () => { expect(live[1].externalId).toBe(`stats:${HRV_HK}:${DAY_KEY}T11`); expect(live[1].value).toBeCloseTo(70, 6); expect(live[1].measuredAt.toISOString()).toBe("2026-05-01T09:30:00.000Z"); + // v1.42 — the raw rows are gone, not tombstoned. + expect( + await prisma.measurement.count({ + where: { userId: TEST_USER_ID, type: "HEART_RATE_VARIABILITY" }, + }), + ).toBe(2); }); it("retires a pre-hourly DAILY stats row in the same pass (late-sync path, no P2002, no double count)", async () => { diff --git a/tests/integration/sync-changes.test.ts b/tests/integration/sync-changes.test.ts index 606933304..671161f3f 100644 --- a/tests/integration/sync-changes.test.ts +++ b/tests/integration/sync-changes.test.ts @@ -399,7 +399,12 @@ describe("GET /api/sync/changes (real Postgres)", () => { // pruned; the recent reachable tombstone survives. `drained` says the // batched walk reached the end of the backlog rather than stopping at // its per-run cap. - expect(pruned).toEqual({ deleted: 1, drained: true, deferredAccounts: 0 }); + expect(pruned).toEqual({ + deleted: 1, + drained: true, + deferredAccounts: 0, + deferredUserIds: [], + }); const survivors = await prisma.measurement.findMany({ where: { userId: TEST_USER_ID }, select: { externalId: true }, diff --git a/tests/integration/tombstone-cleanup-restore-lock.test.ts b/tests/integration/tombstone-cleanup-restore-lock.test.ts index bf49eee7e..a639df410 100644 --- a/tests/integration/tombstone-cleanup-restore-lock.test.ts +++ b/tests/integration/tombstone-cleanup-restore-lock.test.ts @@ -110,8 +110,14 @@ describe("tombstone purge against a running restore (#1031)", () => { deleted: 3, drained: true, deferredAccounts: 1, + deferredUserIds: [RESTORING], + }); + expect(moods).toEqual({ + deleted: 1, + drained: true, + deferredAccounts: 1, + deferredUserIds: [RESTORING], }); - expect(moods).toEqual({ deleted: 1, drained: true, deferredAccounts: 1 }); // The stand-in rolled back, so the restoring account still has its // tombstones; the next run purges them. expect( @@ -123,7 +129,12 @@ describe("tombstone purge against a running restore (#1031)", () => { expect(await prisma.moodEntry.count({ where: { userId: OTHER } })).toBe(0); const next = await cleanupExpiredMeasurementTombstones(prisma); - expect(next).toEqual({ deleted: 3, drained: true, deferredAccounts: 0 }); + expect(next).toEqual({ + deleted: 3, + drained: true, + deferredAccounts: 0, + deferredUserIds: [], + }); }, 60_000); it("a restore that starts during the purge waits only for the account's batch", async () => {