perf: stop keeping folded samples and re-reading duplicates

The nightly folds soft-deleted every raw Apple Health sample they folded
into an hourly or daily stats row and kept it for 75 days. Nothing ever
read those rows as deletions; they only held a re-upload of the sample
off, and they were most of the measurements table and its indexes.

Ingest now refuses such a re-upload directly: a raw sample of a dense or
mean type, old enough to have been folded, whose hour or day a live
stats row covers, is a duplicate with reason folded_window. The batch
route and the Apple Health export import apply the same rule. With that
in place the dense retention and the daily-mean consolidation delete the
raw rows outright, and a new compaction-tombstone-purge job removes the
backlog: 5,000 rows a batch, one account at a time under the restore
lock, a short pause between batches, resumable because the predicate is
idempotent, queued at every boot and continuing itself while it makes
progress. A person's own deletions are never in its predicate. The sync
feed leaves compaction tombstones out and a restore does not write them
back; all three classify through the same function.

The batch route answers an exact re-sent sample without a transaction:
one read over the batch's sample ids, live or deleted, decides what the
reconciler would have decided, at the same point in the loop, so every
status, reason, counter and the cross-source merge stay as they were. A
batch of duplicates only no longer queues PR detection, a reminder sweep
or the detection audit row. Caches are evicted only for a foreground or
manual sync or a batch that wrote a hand-entered reading, and marked
stale otherwise; the targets cache follows the same split.

Also:
- POST /api/admin/maintenance/measurements queues VACUUM (ANALYZE) and a
  REINDEX INDEX CONCURRENTLY per index, largest first, on a connection of
  its own; it refuses while the purge runs, drops invalid leftover copies
  and stops at the first failed rebuild. Runbook in docs/ops.
- The pure volume queues keep their finished jobs for 96 hours instead
  of seven days, never less than the failure window plus a day.
- work_mem is set per app connection from DATABASE_WORK_MEM (16MB), also
  when the statement timeout is disabled.
- The PR-detection fallback tick re-scans only accounts whose readings
  or workouts changed since the previous tick.
- Type discovery for the status refill walks the live-rows index as a
  loose index scan instead of grouping every live row.
- The host metrics chart polls every five minutes, like its sampler.
- The batch route records per-type HealthKit arrivals.
This commit is contained in:
Marc-André Bombeck committed 2026-10-08 08:41:25 +02:00
1 parent e63be67b5a
commit 900a5c4f1f
48 files changed
+3619 -235

No files matched your search

@@ -3,12 +3,90 @@
* `VACUUM (ANALYZE)` / `REINDEX INDEX CONCURRENTLY` pass over `measurements`
* (v1.42). Cookie-only admin: `requireAdmin()` refuses every Bearer.
*
* Contract stub: authenticates as the finished route will, then answers 501.
* Nothing runs in the request: the pass runs in the background worker
* (`measurement-maintenance.ts`), one statement at a time, and refuses to
* start while the compaction-tombstone purge is still working. At most one
* run is queued or running; a second press answers `enqueued: false`.
* Runbook: `docs/ops/measurement-maintenance.md`.
*/
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { notImplemented } from "@/lib/http/not-implemented";
import { NextRequest } from "next/server";
import { z } from "zod/v4";
export const POST = apiHandler(async () => {
await requireAdmin();
return notImplemented();
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import {
apiError,
apiSuccess,
getClientIp,
returnAllZodIssues,
} from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import {
MEASUREMENT_MAINTENANCE_QUEUE,
MEASUREMENT_MAINTENANCE_SEND_OPTIONS,
type MeasurementMaintenancePayload,
} from "@/lib/jobs/measurement-maintenance";
import { annotate } from "@/lib/logging/context";
/** Two booleans; anything longer is not a request for this route. */
const MAX_BODY_BYTES = 1024;
const maintenanceRequestSchema = z
.object({
vacuum: z.boolean().default(true),
reindex: z.boolean().default(true),
})
.strict();
export const POST = apiHandler(async (request: NextRequest) => {
const { user } = await requireAdmin();
// The body is optional: an empty one runs both steps.
const text = await request.text();
if (text.length > MAX_BODY_BYTES) {
return apiError("Request body too large", 413);
}
let body: unknown = {};
if (text.trim().length > 0) {
try {
body = JSON.parse(text);
} catch {
return apiError("Invalid JSON", 400);
}
}
const parsed = maintenanceRequestSchema.safeParse(body);
if (!parsed.success) return returnAllZodIssues(parsed.error);
const payload: MeasurementMaintenancePayload = {
vacuum: parsed.data.vacuum,
reindex: parsed.data.reindex,
};
if (!payload.vacuum && !payload.reindex) {
return apiError("Choose at least one step", 422);
}
annotate({
action: { name: "admin.maintenance.measurements.requested" },
meta: { vacuum: payload.vacuum, reindex: payload.reindex },
});
const boss = getGlobalBoss();
if (!boss) {
return apiError("Background worker is not available", 503);
}
const jobId = await boss.send(
MEASUREMENT_MAINTENANCE_QUEUE,
payload,
MEASUREMENT_MAINTENANCE_SEND_OPTIONS,
);
const enqueued = jobId !== null;
await auditLog("admin.maintenance.measurements.requested", {
userId: user.id,
ipAddress: getClientIp(request),
details: { ...payload, enqueued },
});
annotate({ meta: { maintenance_enqueued: enqueued } });
return apiSuccess({ enqueued }, 202);
});
@@ -19,6 +19,8 @@ vi.mock("@/lib/db", () => ({
prisma: {
user: {
update: vi.fn(),
// The `folded_window` guard reads the account's zone for old samples.
findUnique: vi.fn().mockResolvedValue({ timezone: null }),
},
measurement: {
findMany: vi.fn(),
@@ -366,7 +368,14 @@ describe("POST /api/measurements/batch — aggregated HR wire contract (iOS #34)
it("keeps the per-sample uuid HR path immutable (duplicate, not overwrite)", async () => {
const SAMPLE_ID = "B3A1C0DE-0000-4000-8000-000000000000";
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
{ type: "PULSE", source: "APPLE_HEALTH", externalId: SAMPLE_ID },
{
type: "PULSE",
source: "APPLE_HEALTH",
externalId: SAMPLE_ID,
// Not the posted instant, so the exact-duplicate prefilter leaves
// the row to the reconciler this suite stubs.
measuredAt: new Date(0),
},
] as never);
const res = await POST(
@@ -461,7 +470,14 @@ describe("POST /api/measurements/batch — stats tombstone resurrection", () =>
// sample stays a checkpointing `duplicate` — no update runs at all.
const SAMPLE_ID = "D00DAD00-0000-4000-8000-000000000000";
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
{ type: "PULSE", source: "APPLE_HEALTH", externalId: SAMPLE_ID },
{
type: "PULSE",
source: "APPLE_HEALTH",
externalId: SAMPLE_ID,
// Not the posted instant, so the exact-duplicate prefilter leaves
// the row to the reconciler this suite stubs.
measuredAt: new Date(0),
},
] as never);
const res = await POST(
@@ -0,0 +1,616 @@
/**
* v1.42 — the exact-duplicate prefilter, the `folded_window` guard, and what
* a batch of duplicates only no longer sets off.
*
* The route runs against an in-memory `measurements` table and a reconciler
* stand-in that answers from it the way `reconcileExternalMeasurement` does
* for samples (external and natural identity, live or tombstoned). The golden
* comparison runs every batch twice: once as shipped, once with the
* prefilter's read answering nothing, which is the pre-v1.42 path where every
* row went through the reconciler. Statuses, reasons and counters must match
* exactly; only the transaction count may differ.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { NextRequest } from "next/server";
interface StoredRow {
id: string;
userId: string;
type: string;
source: string;
externalId: string | null;
measuredAt: Date;
sleepStage: string | null;
value: number;
deletedAt: Date | null;
}
const store = vi.hoisted(() => ({
rows: [] as StoredRow[],
/** Golden mode: the prefilter's read answers nothing. */
prefilterBlind: false,
nextId: 1,
}));
function sameInstant(a: Date, b: Date) {
return a.getTime() === b.getTime();
}
vi.mock("@/lib/db", () => {
const prisma = {
user: {
update: vi.fn().mockResolvedValue({}),
findUnique: vi.fn().mockResolvedValue({ timezone: "Europe/Berlin" }),
},
measurement: {
findMany: vi.fn(
async (args: {
where: {
userId: string;
OR?: unknown;
deletedAt?: null;
type?: { in: string[] };
source?: string;
externalId?: { in: string[] };
};
}) => {
const where = args.where;
// Cross-source merge probe: these suites post no MANUAL twins.
if (where.OR) return [];
const ids = where.externalId?.in ?? [];
// The `folded_window` lookup: live `stats:` rows by id.
if (where.deletedAt === null) {
return store.rows.filter(
(r) =>
r.userId === where.userId &&
r.deletedAt === null &&
r.source === where.source &&
(where.type?.in ?? []).includes(r.type) &&
r.externalId !== null &&
ids.includes(r.externalId),
);
}
// The prefilter: every row, live or not, carrying a posted id.
if (store.prefilterBlind) return [];
return store.rows.filter(
(r) =>
r.userId === where.userId &&
r.externalId !== null &&
ids.includes(r.externalId),
);
},
),
},
$transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn({})),
};
return { prisma };
});
vi.mock("@/lib/measurements/reconcile-external-measurement", () => ({
reconcileExternalMeasurement: async (
_tx: unknown,
desired: {
userId: string;
type: string;
source: string;
externalId: string;
measuredAt: Date;
sleepStage?: string | null;
value: number;
},
options: { exactExternalMatch?: "update" | "duplicate" } = {},
) => {
const scope = store.rows.filter(
(r) =>
r.userId === desired.userId &&
r.type === desired.type &&
r.source === desired.source,
);
const externalHit = scope.find((r) => r.externalId === desired.externalId);
const naturalHit = scope.find(
(r) =>
sameInstant(r.measuredAt, desired.measuredAt) &&
r.sleepStage === (desired.sleepStage ?? null),
);
if (!externalHit && !naturalHit) {
const row: StoredRow = {
id: `m${store.nextId++}`,
userId: desired.userId,
type: desired.type,
source: desired.source,
externalId: desired.externalId,
measuredAt: desired.measuredAt,
sleepStage: desired.sleepStage ?? null,
value: desired.value,
deletedAt: null,
};
store.rows.push(row);
return { status: "inserted", row };
}
const canonical = (externalHit ?? naturalHit)!;
const redundant =
externalHit && naturalHit && externalHit !== naturalHit
? naturalHit
: undefined;
if (
options.exactExternalMatch === "duplicate" &&
externalHit === naturalHit
) {
return { status: "duplicate", row: canonical };
}
if (redundant) {
redundant.deletedAt = new Date();
redundant.measuredAt = new Date(0);
}
const wasDeleted = canonical.deletedAt !== null;
canonical.externalId = desired.externalId;
canonical.measuredAt = desired.measuredAt;
canonical.sleepStage = desired.sleepStage ?? null;
canonical.value = desired.value;
canonical.deletedAt = null;
return {
status: wasDeleted ? "resurrected" : "updated",
row: canonical,
dirtyIdentities: [],
};
},
}));
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
vi.mock("@/lib/auth/audit", () => ({
auditLog: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() }));
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/rate-limit", () => ({ checkRateLimit: vi.fn() }));
vi.mock("@/lib/jobs/pr-detection", () => ({
enqueuePrDetection: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/jobs/reminder-satisfy", () => ({
enqueueReminderSatisfy: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/cache/invalidate", () => ({
invalidateUserMeasurements: vi.fn(),
}));
vi.mock("@/lib/rollups/after-measurement-mutation", () => ({
afterMeasurementMutation: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/arrivals/emit-shared", () => ({
emitDataArrival: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/daily/morning-refresh-trigger", () => ({
maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("next/headers", () => ({
headers: vi.fn(async () => ({ get: () => null })),
cookies: vi.fn(async () => ({
get: () => undefined,
set: () => {},
delete: () => {},
})),
}));
import { POST } from "../route";
import { prisma } from "@/lib/db";
import { getSession } from "@/lib/auth/session";
import { auditLog } from "@/lib/auth/audit";
import { checkRateLimit } from "@/lib/rate-limit";
import { enqueuePrDetection } from "@/lib/jobs/pr-detection";
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
const USER = "user-1";
const NOW = Date.now();
const HOUR = 3_600_000;
const DAY = 24 * HOUR;
interface Entry {
hkIdentifier: string;
value: number;
unit: string;
startDate: string;
endDate: string;
externalId: string;
sleepStage?: number;
source?: "APPLE_HEALTH" | "MANUAL";
}
function weight(externalId: string, at: Date, value = 80): Entry {
return {
hkIdentifier: "HKQuantityTypeIdentifierBodyMass",
value,
unit: "kg",
startDate: at.toISOString(),
endDate: at.toISOString(),
externalId,
};
}
function pulse(externalId: string, at: Date, value = 60): Entry {
return {
hkIdentifier: "HKQuantityTypeIdentifierHeartRate",
value,
unit: "count/min",
startDate: at.toISOString(),
endDate: at.toISOString(),
externalId,
};
}
function seed(row: Partial<StoredRow> & { externalId: string; type: string }) {
store.rows.push({
id: `seed${store.nextId++}`,
userId: USER,
source: "APPLE_HEALTH",
measuredAt: new Date(NOW - DAY),
sleepStage: null,
value: 80,
deletedAt: null,
...row,
});
}
function request(entries: Entry[], syncTrigger?: string) {
return new NextRequest("http://localhost/api/measurements/batch", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ entries, ...(syncTrigger ? { syncTrigger } : {}) }),
});
}
type Body = {
data: {
processed: number;
inserted: number;
updated: number;
duplicates: number;
failed: number;
entries: Array<{ index: number; status: string; reason?: string }>;
};
};
async function post(entries: Entry[], syncTrigger?: string) {
const res = await POST(request(entries, syncTrigger));
expect(res.status).toBe(200);
return ((await res.json()) as Body).data;
}
/** Run a batch on a copy of the store as shipped and on a copy without the prefilter. */
async function golden(setup: () => void, entries: Entry[]) {
store.rows = [];
store.nextId = 1;
setup();
store.prefilterBlind = true;
vi.mocked(prisma.$transaction).mockClear();
const old = await post(entries);
const oldTransactions = vi.mocked(prisma.$transaction).mock.calls.length;
const oldRows = JSON.stringify(store.rows);
store.rows = [];
store.nextId = 1;
setup();
store.prefilterBlind = false;
vi.mocked(prisma.$transaction).mockClear();
const next = await post(entries);
const transactions = vi.mocked(prisma.$transaction).mock.calls.length;
expect(next).toEqual(old);
expect(JSON.stringify(store.rows)).toBe(oldRows);
return { old, next, oldTransactions, transactions };
}
beforeEach(() => {
vi.clearAllMocks();
store.rows = [];
store.prefilterBlind = false;
vi.mocked(getSession).mockResolvedValue({
session: { id: "s", expiresAt: new Date(NOW + HOUR) },
user: { id: USER, username: "u", role: "USER" as const },
} as never);
vi.mocked(checkRateLimit).mockResolvedValue({
allowed: true,
limit: 60,
remaining: 60,
resetAt: NOW + 60_000,
});
});
describe("exact-duplicate prefilter (v1.42)", () => {
it("1: 500 exact live duplicates open no transaction and answer as before", async () => {
const entries = Array.from({ length: 500 }, (_, i) =>
weight(`uuid-${i}`, new Date(NOW - DAY - i * 60_000)),
);
const setup = () =>
entries.forEach((e) =>
seed({
type: "WEIGHT",
externalId: e.externalId,
measuredAt: new Date(e.endDate),
}),
);
const { next, oldTransactions, transactions } = await golden(
setup,
entries,
);
expect(next.duplicates).toBe(500);
expect(oldTransactions).toBe(500);
expect(transactions).toBe(0);
});
it("2: an exact hit on a tombstone stays a duplicate and the row stays deleted", async () => {
const at = new Date(NOW - DAY);
const setup = () =>
seed({
type: "WEIGHT",
externalId: "uuid-gone",
measuredAt: at,
deletedAt: new Date(NOW - HOUR),
});
const { next, transactions } = await golden(setup, [
weight("uuid-gone", at),
]);
expect(next.entries[0]).toEqual({ index: 0, status: "duplicate" });
expect(transactions).toBe(0);
expect(store.rows[0].deletedAt).not.toBeNull();
});
it("3: the same id at another instant still reconciles to `updated`", async () => {
const setup = () =>
seed({
type: "WEIGHT",
externalId: "uuid-move",
measuredAt: new Date(NOW - 2 * DAY),
});
const { next, transactions } = await golden(setup, [
weight("uuid-move", new Date(NOW - DAY)),
]);
expect(next.entries[0].status).toBe("updated");
expect(transactions).toBe(1);
});
it("4: the same id with another sleep stage still reconciles", async () => {
const at = new Date(NOW - DAY);
const setup = () =>
seed({
type: "SLEEP_DURATION",
externalId: "uuid-sleep",
measuredAt: at,
sleepStage: "DEEP",
value: 30,
});
const entry: Entry = {
hkIdentifier: "HKCategoryTypeIdentifierSleepAnalysis",
value: 30,
unit: "min",
startDate: new Date(at.getTime() - 30 * 60_000).toISOString(),
endDate: at.toISOString(),
externalId: "uuid-sleep",
sleepStage: 3,
};
const { transactions } = await golden(setup, [entry]);
expect(transactions).toBe(1);
});
it("5: `stats:` rows are never prefiltered; a second snapshot is superseded", async () => {
const day = new Date(NOW - DAY).toISOString().slice(0, 10);
const id = `stats:HKQuantityTypeIdentifierStepCount:${day}`;
const steps = (value: number): Entry => ({
hkIdentifier: "HKQuantityTypeIdentifierStepCount",
value,
unit: "count",
startDate: `${day}T00:00:00.000Z`,
endDate: `${day}T18:00:00.000Z`,
externalId: id,
});
const setup = () =>
seed({
type: "ACTIVITY_STEPS",
externalId: id,
measuredAt: new Date(`${day}T18:00:00.000Z`),
value: 1000,
});
const { next, transactions } = await golden(setup, [
steps(2000),
steps(3000),
]);
expect(next.entries).toEqual([
{ index: 0, status: "duplicate", reason: "superseded_in_batch" },
{ index: 1, status: "updated" },
]);
expect(transactions).toBe(1);
});
it("6: an id twice in the batch and not stored inserts once, then duplicates", async () => {
const at = new Date(NOW - DAY);
const { next } = await golden(() => {}, [
weight("uuid-twice", at),
weight("uuid-twice", at),
]);
expect(next.entries.map((e) => e.status)).toEqual([
"inserted",
"duplicate",
]);
});
it("6b: a stored id posted twice is left to the reconciler", async () => {
const at = new Date(NOW - DAY);
const setup = () =>
seed({ type: "WEIGHT", externalId: "uuid-twice", measuredAt: at });
const { transactions } = await golden(setup, [
weight("uuid-twice", at),
weight("uuid-twice", at),
]);
expect(transactions).toBe(2);
});
it("9: the same id held by another account is not a duplicate", async () => {
const at = new Date(NOW - DAY);
const setup = () =>
seed({
type: "WEIGHT",
externalId: "uuid-shared",
measuredAt: at,
userId: "user-2",
});
const { next } = await golden(setup, [weight("uuid-shared", at)]);
expect(next.entries[0].status).toBe("inserted");
});
it("10: an all-duplicate batch still sets the sync checkpoint", async () => {
const at = new Date(NOW - DAY);
seed({ type: "WEIGHT", externalId: "uuid-dup", measuredAt: at });
await post([weight("uuid-dup", at)], "background");
expect(prisma.user.update).toHaveBeenCalledWith({
where: { id: USER },
data: {
lastSyncedAt: expect.any(Date),
healthKitLastSyncedAt: expect.any(Date),
healthKitLastSyncTrigger: "background",
healthKitLastBackgroundSyncAt: expect.any(Date),
},
});
});
});
describe("a batch of duplicates only (v1.42)", () => {
it("enqueues no PR detection, no reminder sweep and writes no detection audit row", async () => {
const at = new Date(NOW - DAY);
seed({ type: "WEIGHT", externalId: "uuid-dup", measuredAt: at });
await post([weight("uuid-dup", at)]);
expect(enqueuePrDetection).not.toHaveBeenCalled();
expect(enqueueReminderSatisfy).not.toHaveBeenCalled();
expect(
vi
.mocked(auditLog)
.mock.calls.some(
([action]) => action === "personal_records.detection_enqueued",
),
).toBe(false);
});
it("still enqueues all three when a row was written", async () => {
await post([weight("uuid-new", new Date(NOW - DAY))]);
expect(enqueuePrDetection).toHaveBeenCalledTimes(1);
expect(enqueueReminderSatisfy).toHaveBeenCalledTimes(1);
expect(
vi
.mocked(auditLog)
.mock.calls.some(
([action]) => action === "personal_records.detection_enqueued",
),
).toBe(true);
});
});
describe("cache invalidation by origin (v1.42)", () => {
it.each([
["foreground", { evict: true }],
["manual", { evict: true }],
["background", undefined],
["push", undefined],
[undefined, undefined],
])("syncTrigger %s → %o", async (trigger, expected) => {
await post([weight(`uuid-${trigger}`, new Date(NOW - DAY))], trigger);
expect(invalidateUserMeasurements).toHaveBeenCalledWith(USER, expected);
});
it("evicts for a background batch that wrote a hand-entered reading", async () => {
await post(
[
{
...weight("manual-1", new Date(NOW - DAY)),
source: "MANUAL",
},
],
"background",
);
expect(invalidateUserMeasurements).toHaveBeenCalledWith(USER, {
evict: true,
});
});
});
describe("folded_window guard (v1.42)", () => {
// Europe/Berlin is UTC+1 in winter and UTC+2 in summer; the store's
// account zone is Berlin. 120 days back is past the 90-day dense window.
const old = new Date(NOW - 120 * DAY);
const localHourId = () => {
const parts = new Intl.DateTimeFormat("en-CA", {
timeZone: "Europe/Berlin",
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
hour12: false,
}).formatToParts(old);
const get = (t: string) => parts.find((p) => p.type === t)!.value;
const hour = get("hour") === "24" ? "00" : get("hour");
return `stats:HKQuantityTypeIdentifierHeartRate:${get("year")}-${get("month")}-${get("day")}T${hour}`;
};
it("12a: an old raw sample under a live hourly `stats:` row is a folded duplicate", async () => {
seed({
type: "PULSE",
externalId: localHourId(),
measuredAt: new Date(old.getTime() + 1),
});
const data = await post([pulse("uuid-old-hr", old)]);
expect(data.entries[0]).toEqual({
index: 0,
status: "duplicate",
reason: "folded_window",
});
expect(store.rows.some((r) => r.externalId === "uuid-old-hr")).toBe(false);
expect(prisma.$transaction).not.toHaveBeenCalled();
});
it("12b: without a covering `stats:` row it is inserted", async () => {
const data = await post([pulse("uuid-old-hr", old)]);
expect(data.entries[0].status).toBe("inserted");
});
it("12b': a tombstoned `stats:` row does not cover", async () => {
seed({
type: "PULSE",
externalId: localHourId(),
measuredAt: new Date(old.getTime() + 1),
deletedAt: new Date(),
});
const data = await post([pulse("uuid-old-hr", old)]);
expect(data.entries[0].status).toBe("inserted");
});
it("12c: a mean-type sample younger than 36 h is inserted even under a daily row", async () => {
const recent = new Date(NOW - 2 * HOUR);
const day = new Intl.DateTimeFormat("en-CA", {
timeZone: "Europe/Berlin",
}).format(recent);
seed({
type: "RESPIRATORY_RATE",
externalId: `stats:HKQuantityTypeIdentifierRespiratoryRate:${day}`,
measuredAt: new Date(recent.getTime() + 1),
});
const data = await post([
{
hkIdentifier: "HKQuantityTypeIdentifierRespiratoryRate",
value: 14,
unit: "count/min",
startDate: recent.toISOString(),
endDate: recent.toISOString(),
externalId: "uuid-rr",
},
]);
expect(data.entries[0].status).toBe("inserted");
});
it("does not touch a MANUAL reading of the same instant", async () => {
seed({
type: "PULSE",
externalId: localHourId(),
measuredAt: new Date(old.getTime() + 1),
});
const data = await post([{ ...pulse("manual-hr", old), source: "MANUAL" }]);
expect(data.entries[0].status).toBe("inserted");
});
});
@@ -21,6 +21,8 @@ vi.mock("@/lib/db", () => ({
prisma: {
user: {
update: vi.fn(),
// The `folded_window` guard reads the account's zone for old samples.
findUnique: vi.fn().mockResolvedValue({ timezone: null }),
},
measurement: {
findMany: vi.fn(),
@@ -25,7 +25,11 @@ const mocks = vi.hoisted(() => ({ annotate: vi.fn() }));
vi.mock("@/lib/db", () => ({
prisma: {
user: { update: vi.fn() },
user: {
update: vi.fn(),
// The `folded_window` guard reads the account's zone for old samples.
findUnique: vi.fn().mockResolvedValue({ timezone: null }),
},
measurement: {
findMany: vi.fn(),
createManyAndReturn: vi.fn(),
+230 -18
View File
@@ -65,6 +65,15 @@ import {
type MergeCandidate,
} from "@/lib/measurements/cross-source-merge";
import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement";
import { classifyRangeRejection } from "@/lib/measurements/plausibility-gate";
import {
FOLDED_WINDOW_REASON,
findFoldedWindowDuplicates,
} from "@/lib/measurements/folded-window";
import {
recordHealthKitTypeArrivals,
type HealthKitTypeArrival,
} from "@/lib/integrations/healthkit-type-sync";
import { isPlausibleEntryInstant } from "@/lib/validations/entry-instant";
import {
classifyExternalId,
@@ -86,7 +95,11 @@ import { emitDataArrival } from "@/lib/arrivals/emit-shared";
import { groupRowsByArrivalKind } from "@/lib/arrivals/measurement-kind";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation";
import { Prisma, type MeasurementType } from "@/generated/prisma/client";
import {
Prisma,
type MeasurementSource,
type MeasurementType,
} from "@/generated/prisma/client";
// v1.4.25 W16c — historical-backfill threshold for PR push
// suppression. A batch larger than this fires the detection job with
@@ -530,6 +543,8 @@ async function postBatch(request: NextRequest): Promise<Response> {
// composite-key duplicate). Surfaced as a dedicated wide-event count so
// an operator can see how often the standalone-pair mirror collapses.
let crossSourceMergedCount = 0;
// v1.42 — subset of `duplicateCount` refused by the `folded_window` guard.
let foldedWindowCount = 0;
// Only rows the reconciler actually inserted drive arrival side effects.
const insertedPrepared: Prepared[] = [];
const writtenIdentities: Array<{
@@ -621,6 +636,128 @@ async function postBatch(request: NextRequest): Promise<Response> {
);
}
// v1.42 — exact-duplicate prefilter. A re-sent sample used to cost one
// interactive transaction of about ten round-trips (savepoint, three
// advisory locks, two candidate reads, a row lock) only for the
// reconciler to answer `duplicate` without writing. A backfill after an
// anchor reset re-sends whole 500-entry batches of exactly that.
//
// The reconciler answers `duplicate` for a non-`stats:` row exactly when
// an existing row, live OR tombstoned, has the same
// `(type, source, externalId)` AND the same `measuredAt` and
// `sleepStage`: the natural-identity index is unique NULLS NOT DISTINCT,
// so that row is both its external and its natural match, and with no
// aggregation provenance on a sample nothing else can win. One read over
// the batch's sample ids, without a `deletedAt` filter, answers that for
// every row at once.
//
// The verdict is taken at the very point the reconciler would have been
// called, after `superseded_in_batch` and the cross-source merge, so
// statuses, reasons, counters and the in-batch merge candidates are the
// same as before. A row is only prefiltered when neither its external
// key nor its natural identity occurs a second time in the batch: an
// earlier entry sharing either could move or retire the matched row
// before this one is reached, and then the reconciler has to decide.
// `stats:` rows are never prefiltered (overwrite semantics), and neither
// is a value the reconciler's own plausibility gate would refuse.
const externalKeyCounts = new Map<string, number>();
const naturalKeyCounts = new Map<string, number>();
const naturalKeyOf = (
type: unknown,
source: unknown,
measuredAt: Date,
sleepStage: unknown,
) => `${type}::${source}::${measuredAt.getTime()}::${sleepStage ?? "null"}`;
for (const p of prepared) {
const externalKey = `${p.row.type}::${p.row.source}::${p.row.externalId}`;
externalKeyCounts.set(
externalKey,
(externalKeyCounts.get(externalKey) ?? 0) + 1,
);
const naturalKey = naturalKeyOf(
p.row.type,
p.row.source,
p.row.measuredAt as Date,
p.row.sleepStage,
);
naturalKeyCounts.set(
naturalKey,
(naturalKeyCounts.get(naturalKey) ?? 0) + 1,
);
}
const sampleExternalIds = [
...new Set(
prepared
.map((p) => p.row.externalId as string)
.filter((externalId) => !isStatsExternalId(externalId)),
),
];
const existingByExternalKey = new Map<
string,
{ measuredAt: Date; sleepStage: unknown }
>();
if (sampleExternalIds.length > 0) {
const existing = await prisma.measurement.findMany({
where: { userId: user.id, externalId: { in: sampleExternalIds } },
select: {
type: true,
source: true,
externalId: true,
measuredAt: true,
sleepStage: true,
},
});
for (const row of existing) {
existingByExternalKey.set(
`${row.type}::${row.source}::${row.externalId}`,
{ measuredAt: row.measuredAt, sleepStage: row.sleepStage },
);
}
}
function isExactStoredDuplicate(p: Prepared): boolean {
const externalKey = `${p.row.type}::${p.row.source}::${p.row.externalId}`;
const stored = existingByExternalKey.get(externalKey);
if (!stored) return false;
const measuredAt = p.row.measuredAt as Date;
const naturalKey = naturalKeyOf(
p.row.type,
p.row.source,
measuredAt,
p.row.sleepStage,
);
return (
stored.measuredAt.getTime() === measuredAt.getTime() &&
(stored.sleepStage ?? null) === (p.row.sleepStage ?? null) &&
externalKeyCounts.get(externalKey) === 1 &&
naturalKeyCounts.get(naturalKey) === 1 &&
classifyRangeRejection(
p.row.type as MeasurementType,
p.row.value as number,
) === null
);
}
// v1.42 — the `folded_window` guard (`folded-window.ts`). A raw Apple
// Health sample of a dense or mean type, old enough to have been folded,
// whose hour or day a live `stats:` row already covers, is a duplicate:
// the fold no longer leaves the raw rows behind as tombstones, so this is
// what keeps a re-upload after an anchor reset from standing beside the
// mean that already accounts for it. One read for the batch, and none at
// all when no row is old enough.
const foldedIndexes = await findFoldedWindowDuplicates(
prisma,
user.id,
prepared.map((p) => ({
type: p.row.type as MeasurementType,
source: p.row.source as MeasurementSource,
externalId: p.row.externalId as string,
measuredAt: p.row.measuredAt as Date,
})),
);
const foldedPrepared = new Set(
[...foldedIndexes].map((index) => prepared[index]),
);
for (const p of prepared) {
const externalId = p.row.externalId as string;
const statsRow = isStatsExternalId(externalId);
@@ -646,6 +783,32 @@ async function postBatch(request: NextRequest): Promise<Response> {
continue;
}
// Both verdicts below are the reconciler's `duplicate`, reached without
// its transaction; like every duplicate they fall through to the
// in-batch merge candidates.
const prefiltered = !statsRow && isExactStoredDuplicate(p);
const folded = !prefiltered && !statsRow && foldedPrepared.has(p);
if (prefiltered || folded) {
results[p.index] = folded
? {
index: p.index,
status: "duplicate",
reason: FOLDED_WINDOW_REASON,
}
: { index: p.index, status: "duplicate" };
duplicateCount++;
if (folded) foldedWindowCount++;
if (isMergeableSource(p.row.source as string)) {
inBatchCandidates.push({
type: p.row.type as MeasurementType,
source: p.row.source as string,
value: p.row.value as number,
measuredAt: p.row.measuredAt as Date,
});
}
continue;
}
const verdict = await prisma.$transaction((tx) =>
reconcileExternalMeasurement(
tx,
@@ -776,6 +939,34 @@ async function postBatch(request: NextRequest): Promise<Response> {
const syncedAt = new Date();
const deliveredWithoutTheApp =
syncTrigger === "background" || syncTrigger === "push";
// v1.42 (#1173) — what this sync brought per type, for the per-type
// freshness list on the Apple Health card. Apple Health rows only, and
// under the same conditions as the checkpoint beside it.
const arrivals = new Map<MeasurementType, HealthKitTypeArrival>();
for (const p of prepared) {
if (p.row.source !== "APPLE_HEALTH") continue;
const status = results[p.index]?.status;
if (
status !== "inserted" &&
status !== "updated" &&
status !== "duplicate"
) {
continue;
}
const type = p.row.type as MeasurementType;
const entry = arrivals.get(type) ?? { accepted: 0, newSamples: 0 };
entry.accepted += 1;
if (status !== "duplicate") entry.newSamples += 1;
arrivals.set(type, entry);
}
if (arrivals.size > 0) {
await recordHealthKitTypeArrivals(prisma, {
userId: user.id,
counts: arrivals,
trigger: syncTrigger ?? null,
at: syncedAt,
});
}
await prisma.user.update({
where: { id: user.id },
data: {
@@ -920,15 +1111,18 @@ async function postBatch(request: NextRequest): Promise<Response> {
});
}
// v1.4.25 W16c — kick off PR detection for this user. We always
// enqueue when at least one row was written (or the batch had any
// measurements to consider) so a single off-day reading still gets
// evaluated; the warm-up gate inside the detector decides whether
// it's a record. Suppress push notifications for historical
// backfills above the silent threshold. Updated `stats:*` rows also
// count — a per-day-total overwrite can flip the day's value past a
// personal record.
if (insertedCount > 0 || updatedCount > 0 || duplicateCount > 0) {
// v1.4.25 W16c — kick off PR detection for this user when at least one
// row was written, so a single off-day reading still gets evaluated; the
// warm-up gate inside the detector decides whether it's a record. Suppress
// push notifications for historical backfills above the silent threshold.
// Updated `stats:*` rows also count — a per-day-total overwrite can flip
// the day's value past a personal record.
//
// v1.42 — a batch of duplicates only no longer enqueues. It changed
// nothing a record or a reminder could read, and an anchor-reset backfill
// sends hundreds of them, each of which used to queue a detection pass, a
// reminder sweep and an audit row for no new reading.
if (insertedCount > 0 || updatedCount > 0) {
const silent = entries.length > PR_DETECTION_SILENT_THRESHOLD;
// v1.18.1 — eventful Vorsorge satisfaction. A matching reading just
// landed; resolve the user's reminders now rather than waiting on the
@@ -970,9 +1164,13 @@ async function postBatch(request: NextRequest): Promise<Response> {
skipped_by_reason: skippedByReason,
failed: failedCount,
// v1.32.8 (iOS #66) — diagnostic sync-trigger tag (foreground /
// background / push), or null when the client did not send one.
// Observability only; it never influenced any count above.
// background / push / manual), or null when the client did not send
// one. It never influenced any count above; since v1.42 it decides
// how hard the caches below are invalidated.
syncTrigger: syncTrigger ?? null,
// v1.42 — duplicates refused because a live `stats:` row already
// covers the sample's hour or day (`folded-window.ts`).
folded_window: foldedWindowCount,
},
});
@@ -984,12 +1182,26 @@ async function postBatch(request: NextRequest): Promise<Response> {
// and must invalidate every consumer that reads through this user's
// measurements.
if (insertedCount > 0 || updatedCount > 0) {
// v1.18.9 (#38) — hard-evict so a focus-refetch after a background
// iOS / Apple-Health batch sync returns post-sync data. A mark-stale
// would let the `cachedSwr` snapshot serve the pre-batch body on the
// very next read, leaving the dashboard up to ~180 s stale; the manual
// measurement routes already pass `{ evict: true }` for the same reason.
invalidateUserMeasurements(user.id, { evict: true });
// v1.42 — evict only when someone is looking. v1.18.9 (#38) evicted on
// every batch so a focus-refetch after a sync showed post-sync data, but
// a phone syncing in the background posts a batch every few minutes all
// day, and each eviction sent the next dashboard read into a cold
// rebuild. Every other background sync marks stale, which `cachedSwr`
// serves once while one rebuild runs; this route now does the same
// unless the app is open (`foreground`), the person pressed sync
// (`manual`), or the batch wrote one of their own hand-entered readings
// (`MANUAL`), whose entry has to show on the very next read.
const wroteManualRow = prepared.some(
(p) =>
p.row.source === "MANUAL" &&
(results[p.index]?.status === "inserted" ||
results[p.index]?.status === "updated"),
);
const evict =
syncTrigger === "foreground" ||
syncTrigger === "manual" ||
wroteManualRow;
invalidateUserMeasurements(user.id, evict ? { evict: true } : undefined);
// v1.37.19 (C2-F1) — shared post-mutation tail for every distinct
// (type, day) the batch touched. Updates participate as well as
+35
View File
@@ -51,6 +51,11 @@ import type { NextRequest } from "next/server";
import { z } from "zod/v4";
import { prisma } from "@/lib/db";
import {
findCompactionTombstones,
isFoldedWindowCandidate,
loadFoldTimezone,
} from "@/lib/measurements/folded-window";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import {
apiError,
@@ -416,7 +421,37 @@ export const GET = apiHandler(async (request: NextRequest) => {
const mPage = mHasMore ? measurementRows.slice(0, limit) : measurementRows;
const measurementUpserts: MeasurementUpsert[] = [];
const measurementTombstones: MeasurementTombstone[] = [];
// v1.42 — compaction tombstones are not deletions anyone made. The folds
// used to soft-delete the raw Apple Health samples they folded into an
// hourly or daily `stats:` row, and those rows rode this feed as
// tombstones for 75 days. They are left out now, by the same classifier the
// backlog purge uses (`folded-window.ts`); the cursor still advances past
// them, so the keyset walk is unchanged. A person's own deletions are
// reported exactly as before.
const compaction = new Set<string>();
const tombstonesOnPage = mPage.filter((row) => row.deletedAt !== null);
if (tombstonesOnPage.length > 0) {
const candidates = tombstonesOnPage.map((row) => ({
type: row.type,
source: row.source,
externalId: row.externalId,
measuredAt: row.measuredAt,
deletedAt: row.deletedAt,
}));
if (
candidates.some((row) => isFoldedWindowCandidate(row, row.deletedAt!))
) {
const classA = await findCompactionTombstones(
prisma,
user.id,
await loadFoldTimezone(prisma, user.id),
candidates,
);
for (const index of classA) compaction.add(tombstonesOnPage[index].id);
}
}
for (const row of mPage) {
if (compaction.has(row.id)) continue;
if (row.deletedAt) {
measurementTombstones.push({
id: row.id,