mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 21:10:59 +00:00
Merge pull request #469 from MBombeck/release/v1.28.25
release/v1.28.25 — every integration held to the same standard
This commit is contained in:
79 files changed
+3221
-564
No files matched your search
@@ -2,6 +2,44 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.28.25] — 2026-07-11 — Every integration held to the same standard
|
||||
|
||||
A platform-wide hardening pass: the failure classes found live this week were
|
||||
hunted down across every integration, not just where they first surfaced.
|
||||
|
||||
Fitbit carried the same silent wedge fixed for Google Health — a soft-deleted
|
||||
reading permanently blocked its own re-import, hidden behind a comment
|
||||
describing a database index that never existed. Fixed the same way: a re-import
|
||||
revives the deleted row in place.
|
||||
|
||||
Sleep segments now keep stable identities across re-scoring for Withings,
|
||||
WHOOP, Polar and Oura — previously a source refining a night could duplicate or
|
||||
orphan its stage rows, quietly inflating the night's total (the class fixed for
|
||||
Google earlier this week). Each sync now also sweeps rows a re-score left
|
||||
behind, so already-affected nights heal themselves as they are re-read.
|
||||
|
||||
One platform rule for deleted rows: a reading owned by a connected source comes
|
||||
back when the source reports it again — across Withings, WHOOP, Oura, Polar,
|
||||
Nightscout, the mood webhook, CSV re-imports and Apple Health day totals.
|
||||
Deleting an Apple Health sample in the Health app still sticks, as the paired
|
||||
client expects. Import counts and per-entry statuses now tell the truth in
|
||||
every one of these paths.
|
||||
|
||||
Google Health lifecycle: an expired connection no longer lets the hourly sync
|
||||
stamp success while importing nothing (it parks cleanly for reconnect instead
|
||||
of retrying against a dead token forever); history backfills and the sleep
|
||||
repair only mark themselves done after a clean pass, so a transient error now
|
||||
retries instead of silently leaving a gap; and a failed database write holds
|
||||
the sync watermark so the affected window is re-fetched.
|
||||
|
||||
Scale: the doctor report reads only the columns and sections it renders;
|
||||
exports and backups read the measurement table in pages instead of one giant
|
||||
query; long-window charts for sensor-dense metrics (glucose, pulse) aggregate
|
||||
per day in the database beyond 90 days instead of shipping every raw sample;
|
||||
blood-pressure pairing, achievement tallies and several "which types exist"
|
||||
scans moved from in-memory loops to the database. A year of continuous sensor
|
||||
data can no longer stall or crash a request.
|
||||
|
||||
## [1.28.24] — 2026-07-11 — A deleted Google reading no longer blocks its re-import
|
||||
|
||||
A Google Health reading that had been soft-deleted could never be imported
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: HealthLog API
|
||||
version: 1.28.24
|
||||
version: 1.28.25
|
||||
description: >-
|
||||
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "healthlog",
|
||||
"version": "1.28.24",
|
||||
"version": "1.28.25",
|
||||
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
"homepage": "https://healthlog.dev",
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ try {
|
||||
// v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild.
|
||||
const CACHE_VERSION =
|
||||
(typeof self !== "undefined" && self.__APP_VERSION__) ||
|
||||
/* @sw-version-fallback */ "v1.28.24";
|
||||
/* @sw-version-fallback */ "v1.28.25";
|
||||
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
|
||||
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
|
||||
// v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`
|
||||
|
||||
@@ -17,6 +17,9 @@ import { NextRequest } from "next/server";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
// v1.28.25 — the achievements vitals read is a raw (day, hour, type)
|
||||
// bucket aggregation.
|
||||
$queryRaw: vi.fn(),
|
||||
measurement: { findMany: vi.fn(), count: vi.fn() },
|
||||
medication: { findMany: vi.fn() },
|
||||
medicationIntakeEvent: { findMany: vi.fn(), count: vi.fn() },
|
||||
@@ -31,6 +34,14 @@ vi.mock("@/lib/db", () => ({
|
||||
createMany: vi.fn(),
|
||||
},
|
||||
auditEvent: { findMany: vi.fn() },
|
||||
// v1.28.25 — the achievements aggregation previously aborted mid-flight
|
||||
// on this mock (no `auditLog` model), which the old assertion masked
|
||||
// because the vitals findMany fired before the abort. With the vitals
|
||||
// read on `$queryRaw`, the sleep findMany the assertion now pins runs
|
||||
// AFTER these models — mock them so the aggregation completes.
|
||||
auditLog: { findMany: vi.fn() },
|
||||
userHealthProfile: { findUnique: vi.fn() },
|
||||
illnessEpisode: { findMany: vi.fn() },
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -95,6 +106,7 @@ beforeEach(() => {
|
||||
remaining: 9,
|
||||
resetAt: Date.now() + 3_600_000,
|
||||
});
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(0);
|
||||
vi.mocked(prisma.medication.findMany).mockResolvedValue([] as never);
|
||||
@@ -107,6 +119,12 @@ beforeEach(() => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({
|
||||
heightCm: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.passkey.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.auditLog.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.userHealthProfile.findUnique).mockResolvedValue(
|
||||
null as never,
|
||||
);
|
||||
vi.mocked(prisma.illnessEpisode.findMany).mockResolvedValue([] as never);
|
||||
});
|
||||
|
||||
describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
|
||||
@@ -154,5 +172,14 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => {
|
||||
where: expect.objectContaining({ deletedAt: null }),
|
||||
});
|
||||
}
|
||||
// v1.28.25 — the vitals read moved to a raw (day, hour, type) bucket
|
||||
// aggregation; the soft-delete scope must survive on that path too.
|
||||
const rawCalls = vi.mocked(prisma.$queryRaw).mock.calls;
|
||||
expect(rawCalls.length).toBeGreaterThan(0);
|
||||
for (const [strings] of rawCalls) {
|
||||
expect((strings as unknown as readonly string[]).join("?")).toContain(
|
||||
`"deleted_at" IS NULL`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -23,6 +23,7 @@ import { auditLog } from "@/lib/auth/audit";
|
||||
import { apiError, getClientIp } from "@/lib/api-response";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import { toCSV, formatMeasurementsForExport } from "@/lib/export";
|
||||
import { findMeasurementsPaged } from "@/lib/export/paged-measurements";
|
||||
import { shapeMeasurementNotes } from "@/lib/crypto/note-cipher";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
@@ -49,9 +50,24 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
|
||||
const [measurements, userTz, sourcePriorityJson, profile] = await Promise.all(
|
||||
[
|
||||
prisma.measurement.findMany({
|
||||
where,
|
||||
orderBy: { measuredAt: "desc" },
|
||||
// v1.28.25 — keyset-paginated read with a narrow select. The route
|
||||
// keeps its contract (an absent `since`/`until` means the full
|
||||
// history), but the read is now chunked so an unbounded window on a
|
||||
// CGM / per-sample-HR account never materialises the whole table in
|
||||
// one query. The select is exactly the formatter's `ExportMeasurement`
|
||||
// shape + note decryption + the `id` cursor key.
|
||||
findMeasurementsPaged(prisma, where, {
|
||||
id: true,
|
||||
type: true,
|
||||
value: true,
|
||||
unit: true,
|
||||
measuredAt: true,
|
||||
source: true,
|
||||
notes: true,
|
||||
notesEncrypted: true,
|
||||
glucoseContext: true,
|
||||
sleepStage: true,
|
||||
deviceType: true,
|
||||
}),
|
||||
resolveUserTimezone(user.id),
|
||||
loadUserSourcePriority(user.id),
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
formatMoodEntriesForExport,
|
||||
} from "@/lib/export";
|
||||
import { shapeMeasurementNotes, shapeMoodNote } from "@/lib/crypto/note-cipher";
|
||||
import { findMeasurementsPaged } from "@/lib/export/paged-measurements";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
@@ -59,13 +60,33 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
const data: Record<string, unknown> = {};
|
||||
|
||||
if (type === "measurements" || type === "all") {
|
||||
const measurements = await prisma.measurement.findMany({
|
||||
// v1.28.25 — keyset-paginated read with a narrow select. The legacy
|
||||
// export has no date window, so a CGM / per-sample-HR account holds a
|
||||
// six-figure row set; one unbounded full-width `findMany` here was the
|
||||
// scale hazard. The select carries exactly the fields the formatter +
|
||||
// note decryption consume (`ExportMeasurement` + `shapeMeasurementNotes`
|
||||
// + the `id` cursor key) — no Bytes column beyond the note ciphertext
|
||||
// the export must decrypt.
|
||||
const measurements = await findMeasurementsPaged(
|
||||
prisma,
|
||||
// v1.4.41 W-DELETED-2 — exclude soft-deleted measurements from
|
||||
// the legacy /api/export endpoint so CSV + JSON downloads stay
|
||||
// consistent with the live measurement reads.
|
||||
where: { userId, deletedAt: null },
|
||||
orderBy: { measuredAt: "desc" },
|
||||
});
|
||||
{ userId, deletedAt: null },
|
||||
{
|
||||
id: true,
|
||||
type: true,
|
||||
value: true,
|
||||
unit: true,
|
||||
measuredAt: true,
|
||||
source: true,
|
||||
notes: true,
|
||||
notesEncrypted: true,
|
||||
glucoseContext: true,
|
||||
sleepStage: true,
|
||||
deviceType: true,
|
||||
},
|
||||
);
|
||||
// v1.11.5 — sleep collapses to one row per night by default (the
|
||||
// formatter's `night` granularity), matching the per-type CSV route.
|
||||
// Thread the user's source-priority so a multi-source night dedups to
|
||||
|
||||
@@ -3,6 +3,9 @@ import { NextRequest } from "next/server";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
// v1.28.25 — the achievements vitals read is a raw (day, hour, type)
|
||||
// bucket aggregation.
|
||||
$queryRaw: vi.fn(),
|
||||
measurement: { findMany: vi.fn() },
|
||||
medicationIntakeEvent: { findMany: vi.fn() },
|
||||
medication: { findMany: vi.fn() },
|
||||
@@ -84,6 +87,7 @@ beforeEach(() => {
|
||||
// v1.4.34 IW-G — reset achievement LRU between tests so each case
|
||||
// observes a cold cache.
|
||||
__resetAllCachesForTests();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue(
|
||||
[] as never,
|
||||
|
||||
@@ -3,6 +3,9 @@ import { NextRequest } from "next/server";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
// v1.28.25 — the achievements vitals read is a raw (day, hour, type)
|
||||
// bucket aggregation.
|
||||
$queryRaw: vi.fn(),
|
||||
measurement: { findMany: vi.fn() },
|
||||
medicationIntakeEvent: { findMany: vi.fn() },
|
||||
medication: { findMany: vi.fn() },
|
||||
@@ -70,6 +73,7 @@ beforeEach(() => {
|
||||
// v1.4.34 IW-G — reset achievement LRU between tests so each case
|
||||
// observes a cold cache.
|
||||
__resetAllCachesForTests();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue(
|
||||
[] as never,
|
||||
|
||||
@@ -16,6 +16,9 @@ import { NextRequest } from "next/server";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
// v1.28.25 — the achievements vitals read is a raw (day, hour, type)
|
||||
// bucket aggregation.
|
||||
$queryRaw: vi.fn(),
|
||||
measurement: { findMany: vi.fn() },
|
||||
medicationIntakeEvent: { findMany: vi.fn() },
|
||||
medication: { findMany: vi.fn() },
|
||||
@@ -76,6 +79,7 @@ const SESSION_OK = {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
__resetAllCachesForTests();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue(
|
||||
[] as never,
|
||||
@@ -123,6 +127,7 @@ describe("GET /api/gamification/achievements — achievements module gate", () =
|
||||
expect(body.meta.module).toBe("achievements");
|
||||
|
||||
// The disappearance is total: no badge aggregation ran.
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(prisma.measurement.findMany).not.toHaveBeenCalled();
|
||||
expect(prisma.userAchievement.findMany).not.toHaveBeenCalled();
|
||||
expect(prisma.userAchievement.createMany).not.toHaveBeenCalled();
|
||||
|
||||
@@ -55,6 +55,6 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
}
|
||||
}
|
||||
|
||||
const imported = await syncUserGoogleHealth(user.id, { fullSync });
|
||||
const { imported } = await syncUserGoogleHealth(user.id, { fullSync });
|
||||
return apiSuccess({ imported, fullSync });
|
||||
});
|
||||
@@ -103,7 +103,11 @@ beforeEach(() => {
|
||||
updateMany: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
measurement.findMany.mockResolvedValue([]);
|
||||
measurement.createMany.mockResolvedValue({ count: 0 });
|
||||
// Echo the attempted chunk size, matching Postgres semantics when no row
|
||||
// collides — the route now reconciles `inserted` against this count.
|
||||
measurement.createMany.mockImplementation(
|
||||
async (arg: { data: unknown[] }) => ({ count: arg.data.length }),
|
||||
);
|
||||
measurement.updateMany.mockResolvedValue({ count: 0 });
|
||||
(
|
||||
prisma.$transaction as unknown as {
|
||||
@@ -226,6 +230,29 @@ describe("POST /api/import/csv — batched write + per-row envelope", () => {
|
||||
expect(mMeasurement().createMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resurrects a tombstoned externalId row on re-import (deletedAt: null in update)", async () => {
|
||||
// The ext-probe is deliberately deletedAt-less, so a tombstoned IMPORT
|
||||
// row matches like a live one; the update must carry the resurrection
|
||||
// (IMPORT rows are re-importable by design).
|
||||
mMeasurement().findMany.mockResolvedValue([
|
||||
{ type: "WEIGHT", externalId: "ext-tomb" },
|
||||
]);
|
||||
|
||||
const res = await POST(
|
||||
csvRequest(
|
||||
[HEADER, "WEIGHT,81.0,kg,2026-05-01T08:00:00Z,,,ext-tomb"].join("\n"),
|
||||
),
|
||||
);
|
||||
const body = (await res.json()) as CsvEnvelope;
|
||||
expect(body.data?.updated).toBe(1);
|
||||
expect(body.data?.rows[0].status).toBe("updated");
|
||||
const updateArg = mMeasurement().updateMany.mock.calls[0][0] as {
|
||||
data: { value: number; deletedAt: Date | null };
|
||||
};
|
||||
expect(updateArg.data.value).toBe(81.0);
|
||||
expect(updateArg.data.deletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("inserts an externalId row when it does not exist yet", async () => {
|
||||
mMeasurement().findMany.mockResolvedValue([]);
|
||||
const res = await POST(
|
||||
@@ -274,4 +301,33 @@ describe("POST /api/import/csv — batched write + per-row envelope", () => {
|
||||
// Only the first survivor reaches the bulk insert.
|
||||
expect(mMeasurement().createMany.mock.calls[0][0].data).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("reconciles `inserted` against the createMany count when skipDuplicates absorbs a race", async () => {
|
||||
// Two fresh rows attempted, but a concurrent double-submit already
|
||||
// landed one of them — `skipDuplicates` absorbs the conflict and the
|
||||
// count comes back short. The envelope must sum to the DB truth: one
|
||||
// inserted, one downgraded to skipped/duplicate.
|
||||
mMeasurement().createMany.mockImplementation(
|
||||
async (arg: { data: unknown[] }) => ({ count: arg.data.length - 1 }),
|
||||
);
|
||||
|
||||
const res = await POST(
|
||||
csvRequest(
|
||||
[
|
||||
HEADER,
|
||||
"WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,",
|
||||
"WEIGHT,81.5,kg,2026-05-01T09:00:00Z,,,",
|
||||
].join("\n"),
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as CsvEnvelope;
|
||||
expect(body.data?.inserted).toBe(1);
|
||||
expect(body.data?.skipped).toBe(1);
|
||||
const statuses = body.data?.rows.map((r) => r.status).sort();
|
||||
expect(statuses).toEqual(["inserted", "skipped"]);
|
||||
expect(body.data?.rows.find((r) => r.status === "skipped")?.reason).toBe(
|
||||
"duplicate",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -239,12 +239,14 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
...plainInserts.map(buildCreateData),
|
||||
];
|
||||
const CREATE_CHUNK = 200;
|
||||
let createdCount = 0;
|
||||
await prisma.$transaction(async (tx) => {
|
||||
for (let i = 0; i < toCreate.length; i += CREATE_CHUNK) {
|
||||
await tx.measurement.createMany({
|
||||
const res = await tx.measurement.createMany({
|
||||
data: toCreate.slice(i, i + CREATE_CHUNK),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
createdCount += res.count;
|
||||
}
|
||||
for (const m of extUpdates.values()) {
|
||||
await tx.measurement.updateMany({
|
||||
@@ -262,11 +264,37 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
notesEncrypted: encryptNote(m.notes ?? null),
|
||||
glucoseContext:
|
||||
(m.glucoseContext as GlucoseContext | undefined) ?? null,
|
||||
// No-op on a live row, a deliberate RESURRECTION on a
|
||||
// tombstoned one — IMPORT rows are re-importable by design, so
|
||||
// a re-imported externalId brings the row back (mirrors the
|
||||
// source-owned sync resurrect rule).
|
||||
deletedAt: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Reconcile `inserted` against what `createMany` actually wrote
|
||||
// (mirrors the batch route's raced-duplicate downgrade). Under
|
||||
// `skipDuplicates` a conflicting row is silently absorbed — the key is
|
||||
// present in the table either way, so we cannot identify the SPECIFIC
|
||||
// raced lines; we only need the counters and per-line statuses to sum
|
||||
// to the truth. Downgrade enough `inserted` lines to `duplicate` so
|
||||
// the envelope matches the DB write count.
|
||||
const racedDuplicates = toCreate.length - createdCount;
|
||||
if (racedDuplicates > 0) {
|
||||
let downgraded = 0;
|
||||
for (const [line, outcome] of writeOutcome) {
|
||||
if (downgraded >= racedDuplicates) break;
|
||||
if (outcome === "inserted") {
|
||||
writeOutcome.set(line, "duplicate");
|
||||
inserted--;
|
||||
skipped++;
|
||||
downgraded++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One bounded rollup re-fold per touched (type, day) — a 10 000-row CSV
|
||||
// pays at most ~N (type, day) recomputes, not 10 000 per-row hooks.
|
||||
// Best-effort: a populator hiccup never fails the importer.
|
||||
|
||||
@@ -104,6 +104,10 @@ vi.mock("@/lib/jobs/insight-pregenerate-shared", () => ({
|
||||
// enqueue pipeline itself is covered by the comprehensive-generate tests.
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
enqueueStatusRefillForUser: vi.fn(async () => 7),
|
||||
// v1.28.25 — the route shares the lib's comparison-snapshot builder
|
||||
// (its private near-copy was deleted). Null = comparison toggle off,
|
||||
// matching the pre-existing fixtures (no dashboardWidgetsJson set).
|
||||
buildComparisonSnapshotForUser: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
|
||||
@@ -30,10 +30,7 @@ import {
|
||||
buildDerivedBriefingPrompt,
|
||||
} from "@/lib/insights/derived-briefing";
|
||||
import { getAgeFromDateOfBirth } from "@/lib/analytics/pulse-targets";
|
||||
import {
|
||||
buildUserPrompt,
|
||||
type ComparisonSnapshot,
|
||||
} from "@/lib/ai/prompts/insight-system-prompt";
|
||||
import { buildUserPrompt } from "@/lib/ai/prompts/insight-system-prompt";
|
||||
import {
|
||||
buildSystemPromptWithReferences,
|
||||
buildBriefingPersonalisationBlock,
|
||||
@@ -43,18 +40,7 @@ import {
|
||||
getSelfContextTextForUser,
|
||||
} from "@/lib/ai/coach/about-me";
|
||||
import { metricsFromPresentSections } from "@/lib/ai/medical-references";
|
||||
import { summarize, type DataPoint } from "@/lib/analytics/trends";
|
||||
import {
|
||||
reconstructSleepNights,
|
||||
type SleepStageRow,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
import {
|
||||
resolveDashboardLayout,
|
||||
type ComparisonBaseline,
|
||||
} from "@/lib/dashboard-layout";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import {
|
||||
insightResultSchema,
|
||||
singleUserTurn,
|
||||
@@ -88,7 +74,13 @@ import { resolveServerLocale } from "@/lib/i18n/server-locale";
|
||||
import { hasUsableStatusProvider } from "@/lib/insights/status-provider";
|
||||
import { hashInsightSnapshot } from "@/lib/insights/snapshot-hash";
|
||||
import { enqueueForceWarm } from "@/lib/jobs/insight-pregenerate-shared";
|
||||
import { enqueueStatusRefillForUser } from "@/lib/insights/comprehensive-generate";
|
||||
// v1.28.25 — the comparison-snapshot builder was a private near-copy of
|
||||
// the lib export (drifted only in comments); the route now shares the
|
||||
// one implementation the nightly warm pass uses.
|
||||
import {
|
||||
buildComparisonSnapshotForUser,
|
||||
enqueueStatusRefillForUser,
|
||||
} from "@/lib/insights/comprehensive-generate";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -120,153 +112,6 @@ export function resolveInsightsRateLimit(): number {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.4.16 phase B8 — fetch the user's persisted comparison toggle and
|
||||
* build a `ComparisonSnapshot` for the prompt builder. Returns null
|
||||
* when comparison is off (most users), so the prompt builder skips
|
||||
* the context block entirely.
|
||||
*
|
||||
* Snapshot construction reuses the analytics `summarize()` helper
|
||||
* (which now emits `avg30LastMonth` + `avg30LastYear`) so the numbers
|
||||
* the LLM sees match the numbers the dashboard tile renders — no
|
||||
* second source of truth for the comparison delta.
|
||||
*/
|
||||
async function buildComparisonSnapshotForUser(
|
||||
userId: string,
|
||||
): Promise<ComparisonSnapshot | null> {
|
||||
// The features payload doesn't carry prior-period values — pulling
|
||||
// a fresh `summarize()` per metric below keeps the snapshot
|
||||
// self-contained without retrofitting features.ts.
|
||||
const row = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { dashboardWidgetsJson: true },
|
||||
});
|
||||
const layout = resolveDashboardLayout(row?.dashboardWidgetsJson);
|
||||
const baseline: ComparisonBaseline = layout.comparisonBaseline ?? "none";
|
||||
if (baseline === "none") return null;
|
||||
|
||||
// Pull every measurement type once and run summarize() so the
|
||||
// prior-period averages line up with what the dashboard analytics
|
||||
// computes. Only include rows where the LLM can sensibly narrate
|
||||
// a prior comparison (i.e. the metric exists in our snapshot type
|
||||
// mapping below).
|
||||
const typeToSnapshotKey: Record<string, string> = {
|
||||
WEIGHT: "weight",
|
||||
BLOOD_PRESSURE_SYS: "bloodPressureSys",
|
||||
BLOOD_PRESSURE_DIA: "bloodPressureDia",
|
||||
PULSE: "pulse",
|
||||
BODY_FAT: "bodyFat",
|
||||
SLEEP_DURATION: "sleep",
|
||||
ACTIVITY_STEPS: "steps",
|
||||
};
|
||||
const typeUnits: Record<string, string> = {
|
||||
WEIGHT: "kg",
|
||||
BLOOD_PRESSURE_SYS: "mmHg",
|
||||
BLOOD_PRESSURE_DIA: "mmHg",
|
||||
PULSE: "bpm",
|
||||
BODY_FAT: "%",
|
||||
SLEEP_DURATION: "h",
|
||||
ACTIVITY_STEPS: "",
|
||||
};
|
||||
const types = Object.keys(typeToSnapshotKey) as MeasurementType[];
|
||||
// The snapshot only feeds summarize()'s avg30 / avg30LastMonth /
|
||||
// avg30LastYear. The widest of those reaches back into the [365, 395)-day
|
||||
// window (ageMs < 395 * DAY in meanOfWindow), so nothing older than 395
|
||||
// days can change a result. Bound the read at 400 days — a 5-day floor over
|
||||
// the strict-less-than boundary — instead of scanning the full history,
|
||||
// which on multi-year accounts is tens of thousands of rows per type on the
|
||||
// page-blocking generate path and the nightly pregenerate cron.
|
||||
const sinceMeasuredAt = new Date(Date.now() - 400 * 86_400_000);
|
||||
const rows = await Promise.all(
|
||||
types.map(async (type) => {
|
||||
// SLEEP_DURATION is stored ONE ROW PER STAGE per night; summarising the
|
||||
// raw stage rows mislabels a per-stage average as a night total (and
|
||||
// double-counts a bare ASLEEP aggregate against its granular twin). Route
|
||||
// the comparison through the per-night dedup reconstruction so the
|
||||
// current/baseline averages are per-night TIME-ASLEEP totals in minutes.
|
||||
let dataPoints: DataPoint[];
|
||||
if (type === "SLEEP_DURATION") {
|
||||
const [sleepRows, sleepTz, sleepPriority] = await Promise.all([
|
||||
prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type,
|
||||
deletedAt: null,
|
||||
measuredAt: { gte: sinceMeasuredAt },
|
||||
},
|
||||
orderBy: { measuredAt: "asc" },
|
||||
select: {
|
||||
measuredAt: true,
|
||||
value: true,
|
||||
sleepStage: true,
|
||||
source: true,
|
||||
// Writer-level collapse: two HealthKit apps behind one
|
||||
// source (watch stages vs phone in-bed) must not blend.
|
||||
deviceType: true,
|
||||
},
|
||||
}),
|
||||
resolveUserTimezone(userId),
|
||||
loadUserSourcePriority(userId),
|
||||
]);
|
||||
dataPoints = reconstructSleepNights(
|
||||
sleepRows as SleepStageRow[],
|
||||
sleepTz,
|
||||
sleepPriority,
|
||||
)
|
||||
.filter((n) => n.asleepMinutes > 0)
|
||||
.map((n) => ({ date: n.measuredAt, value: n.asleepMinutes }));
|
||||
} else {
|
||||
const measurements = await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type,
|
||||
deletedAt: null,
|
||||
measuredAt: { gte: sinceMeasuredAt },
|
||||
},
|
||||
orderBy: { measuredAt: "asc" },
|
||||
select: { measuredAt: true, value: true },
|
||||
});
|
||||
dataPoints = measurements.map((m): DataPoint => ({
|
||||
date: m.measuredAt,
|
||||
value: m.value,
|
||||
}));
|
||||
}
|
||||
const summary = summarize(dataPoints);
|
||||
const baselineAvg =
|
||||
baseline === "lastMonth"
|
||||
? (summary.avg30LastMonth ?? null)
|
||||
: (summary.avg30LastYear ?? null);
|
||||
const currentAvg = summary.avg30 ?? null;
|
||||
const delta =
|
||||
currentAvg !== null && baselineAvg !== null
|
||||
? Math.round((currentAvg - baselineAvg) * 100) / 100
|
||||
: null;
|
||||
const deltaPercent =
|
||||
delta !== null && baselineAvg !== null && baselineAvg !== 0
|
||||
? Math.round((delta / Math.abs(baselineAvg)) * 100 * 10) / 10
|
||||
: null;
|
||||
return {
|
||||
type: typeToSnapshotKey[type] ?? type,
|
||||
currentAvg,
|
||||
baselineAvg,
|
||||
delta,
|
||||
deltaPercent,
|
||||
unit: typeUnits[type] ?? "",
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
// Drop fully-empty rows (current AND baseline missing) so the prompt
|
||||
// doesn't pad with noise. Keep partial rows (one side null) — the
|
||||
// block renderer flags them with "no prior-period data available"
|
||||
// which is useful context for the model.
|
||||
const metrics = rows.filter(
|
||||
(row) => row.currentAvg !== null || row.baselineAvg !== null,
|
||||
);
|
||||
|
||||
return { baseline, metrics };
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/insights/generate — read-only advisor read.
|
||||
*
|
||||
|
||||
@@ -145,6 +145,9 @@ beforeEach(() => {
|
||||
(
|
||||
prisma.moodEntryRollup.findMany as ReturnType<typeof vi.fn>
|
||||
).mockResolvedValue([]);
|
||||
// v1.28.25 — the latest-ever-per-type read is a raw `DISTINCT ON`
|
||||
// (Prisma's `distinct` dedups client-side after pulling every row).
|
||||
(prisma.$queryRawUnsafe as ReturnType<typeof vi.fn>).mockResolvedValue([]);
|
||||
});
|
||||
|
||||
describe("GET /api/insights/targets — mood-rollup tier swap", () => {
|
||||
@@ -291,36 +294,33 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => {
|
||||
expect(moodTarget).toBeUndefined();
|
||||
});
|
||||
|
||||
it("floors the `findMany distinct` latest-ever-per-type read at one year", async () => {
|
||||
// Audit High finding 3: the `findMany({orderBy: measuredAt desc,
|
||||
// distinct: ["type"]})` for "absolute latest measurement per type"
|
||||
// used to run WITHOUT a `measuredAt >=` floor. On a 347 k-row
|
||||
// tenant Postgres still has to sort the full set before applying
|
||||
// DISTINCT ON because Prisma's `distinct` does not compile to PG's
|
||||
// `DISTINCT ON` — it dedups in the driver. v1.4.40 adds a
|
||||
// 365-day floor so the planner-side walk is bounded.
|
||||
(prisma.measurement.findMany as ReturnType<typeof vi.fn>).mockResolvedValue(
|
||||
[],
|
||||
);
|
||||
|
||||
it("reads latest-ever-per-type via raw DISTINCT ON with the one-year floor", async () => {
|
||||
// Audit High finding 3 established the 365-day floor; v1.28.25 moves
|
||||
// the read onto a real Postgres `DISTINCT ON` (Prisma's `distinct`
|
||||
// dedups in the driver AFTER pulling every row in the window, which
|
||||
// on a dense-type tenant shipped a year of raw rows to Node just to
|
||||
// keep seven). Pin the raw query: DISTINCT ON, per-type descending
|
||||
// walk, user id + floored date as bound parameters.
|
||||
await callGet(makeReq());
|
||||
|
||||
// Find the `distinct: ["type"]` call.
|
||||
const calls = (
|
||||
prisma.measurement.findMany as ReturnType<typeof vi.fn>
|
||||
).mock.calls.map((c) => c[0]);
|
||||
const distinctCall = calls.find(
|
||||
(c) =>
|
||||
c &&
|
||||
Array.isArray(c.distinct) &&
|
||||
c.distinct[0] === "type" &&
|
||||
c.orderBy?.measuredAt === "desc",
|
||||
const rawCalls = (prisma.$queryRawUnsafe as ReturnType<typeof vi.fn>).mock
|
||||
.calls;
|
||||
const distinctCall = rawCalls.find(
|
||||
(c) => typeof c[0] === "string" && c[0].includes("DISTINCT ON"),
|
||||
);
|
||||
expect(distinctCall).toBeDefined();
|
||||
// The `measuredAt >= oneYearAgo` floor must be present.
|
||||
expect(distinctCall?.where?.measuredAt).toBeDefined();
|
||||
expect(distinctCall?.where?.measuredAt?.gte).toBeInstanceOf(Date);
|
||||
const flooredAt = distinctCall?.where?.measuredAt?.gte as Date;
|
||||
const [sql, boundUserId, flooredAt] = distinctCall as [
|
||||
string,
|
||||
string,
|
||||
Date,
|
||||
];
|
||||
expect(sql).toContain(`DISTINCT ON (m."type")`);
|
||||
expect(sql).toContain(`ORDER BY m."type" ASC, m."measured_at" DESC`);
|
||||
// Parameter-bound, never spliced: user id + date ride $1/$2.
|
||||
expect(sql).toContain("$1");
|
||||
expect(sql).toContain("$2");
|
||||
expect(boundUserId).toBe("user-targets-1");
|
||||
expect(flooredAt).toBeInstanceOf(Date);
|
||||
const oneYearMs = 365 * 24 * 60 * 60 * 1000;
|
||||
const ageMs = Date.now() - flooredAt.getTime();
|
||||
// Floor is somewhere around one year ago — give a ±1-hour window
|
||||
|
||||
@@ -133,6 +133,11 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
source: "MOODLOG",
|
||||
date: entry.date,
|
||||
moodLoggedAt,
|
||||
// No-op on a live row, a deliberate RESURRECTION on a tombstoned
|
||||
// one — moodLog is the source of truth for the rows it minted; a
|
||||
// created/updated event means the entry exists upstream (deletion
|
||||
// arrives as its own `mood.deleted` event).
|
||||
deletedAt: null,
|
||||
},
|
||||
create: {
|
||||
userId: user.id,
|
||||
@@ -160,6 +165,8 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
score: entry.score,
|
||||
tags: entry.tags ? JSON.stringify(entry.tags) : null,
|
||||
source: "MOODLOG",
|
||||
// Same resurrection rule as the externalId-keyed upsert above.
|
||||
deletedAt: null,
|
||||
},
|
||||
create: {
|
||||
userId: user.id,
|
||||
|
||||
@@ -308,3 +308,69 @@ describe("POST /api/measurements/batch — hourly HR wire contract (iOS #34)", (
|
||||
expect(data.entries[0].status).toBe("updated");
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/measurements/batch — stats tombstone resurrection", () => {
|
||||
it("resurrects a tombstoned stats: day-total on re-post (deletedAt: null, status updated)", async () => {
|
||||
const STEP_STATS_ID = "stats:HKQuantityTypeIdentifierStepCount:2026-06-21";
|
||||
// The existence probe is deliberately deletedAt-less, so a tombstoned
|
||||
// day-total row matches exactly like a live one and routes into the
|
||||
// overwrite branch.
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{
|
||||
type: "ACTIVITY_STEPS",
|
||||
source: "APPLE_HEALTH",
|
||||
externalId: STEP_STATS_ID,
|
||||
},
|
||||
] as never);
|
||||
|
||||
const res = await POST(
|
||||
makeRequest({
|
||||
entries: [
|
||||
{
|
||||
hkIdentifier: "HKQuantityTypeIdentifierStepCount",
|
||||
value: 9001,
|
||||
unit: "count",
|
||||
startDate: "2026-06-21T00:00:00.000Z",
|
||||
endDate: "2026-06-21T23:59:59.000Z",
|
||||
externalId: STEP_STATS_ID,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const { data } = await readJson(res);
|
||||
expect(data.updated).toBe(1);
|
||||
expect(data.entries[0].status).toBe("updated");
|
||||
|
||||
// The overwrite carries the resurrection: the observer re-posts the
|
||||
// day's canonical total, so the update data pins `deletedAt: null`.
|
||||
expect(prisma.measurement.updateMany).toHaveBeenCalledTimes(1);
|
||||
const updateArg = vi.mocked(prisma.measurement.updateMany).mock.calls[0][0];
|
||||
const updateData = (
|
||||
updateArg as { data: { value: number; deletedAt: Date | null } }
|
||||
).data;
|
||||
expect(updateData.value).toBe(9001);
|
||||
expect(updateData.deletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps a tombstoned SAMPLE-grain row suppressed (duplicate, no resurrect write)", async () => {
|
||||
// Apple LWW contract: the iOS reconciler propagates HealthKit
|
||||
// deletions for sample-grain rows, so a re-post of a tombstoned
|
||||
// sample stays a checkpointing `duplicate` — no update runs at all.
|
||||
const SAMPLE_ID = "D00DAD00-0000-4000-8000-000000000000";
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{ type: "PULSE", source: "APPLE_HEALTH", externalId: SAMPLE_ID },
|
||||
] as never);
|
||||
|
||||
const res = await POST(
|
||||
makeRequest({ entries: [hrBucketEntry(SAMPLE_ID, 61)] }),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const { data } = await readJson(res);
|
||||
expect(data.duplicates).toBe(1);
|
||||
expect(data.updated).toBe(0);
|
||||
expect(data.entries[0].status).toBe("duplicate");
|
||||
expect(prisma.measurement.updateMany).not.toHaveBeenCalled();
|
||||
expect(prisma.measurement.createMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -545,6 +545,14 @@ async function postBatch(request: NextRequest): Promise<Response> {
|
||||
string | null,
|
||||
deviceType: p.row.deviceType as Prepared["row"]["deviceType"],
|
||||
sleepStage: p.row.sleepStage as Prepared["row"]["sleepStage"],
|
||||
// No-op on a live row, a deliberate RESURRECTION on a
|
||||
// tombstoned one — the observer re-posts the day's canonical
|
||||
// total, so a `stats:` day-total row follows the
|
||||
// source-owned resurrect rule (mirrors Google / Fitbit).
|
||||
// SAMPLE-grain rows never reach this branch: their
|
||||
// tombstones stick (the iOS LWW reconciler propagates
|
||||
// HealthKit deletions and reports them as `duplicate`).
|
||||
deletedAt: null,
|
||||
},
|
||||
});
|
||||
results[p.index] = { index: p.index, status: "updated" };
|
||||
|
||||
@@ -6,6 +6,9 @@ vi.mock("@/lib/db", () => ({
|
||||
measurement: { findMany: vi.fn() },
|
||||
user: { findUnique: vi.fn() },
|
||||
auditLog: { create: vi.fn() },
|
||||
// v1.28.25 — dense kinds (glucose / pulse) day-bucket in SQL for
|
||||
// windows beyond the 90-day raw cap.
|
||||
$queryRaw: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -65,6 +68,7 @@ function req(query: string): NextRequest {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never);
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({
|
||||
glucoseUnit: "mg/dL",
|
||||
@@ -122,6 +126,48 @@ describe("GET /api/measurements/series", () => {
|
||||
expect(body.data.stats.count).toBe(1);
|
||||
});
|
||||
|
||||
it("pairs BP via the sorted two-pointer merge exactly like the old scan (v1.28.25)", async () => {
|
||||
// Pins the O(sys + dia) merge against the previous per-systolic full
|
||||
// rescan: nearest diastolic within ±5 min wins, a tie keeps the
|
||||
// EARLIER diastolic row, an out-of-window systolic pairs null.
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
const at = (iso: string) => new Date(iso);
|
||||
const findManyMock = vi.mocked(prisma.measurement.findMany);
|
||||
findManyMock.mockImplementation(((args: unknown) => {
|
||||
const a = args as { where: { type: string } };
|
||||
if (a.where.type === "BLOOD_PRESSURE_SYS") {
|
||||
return Promise.resolve([
|
||||
{ id: "s1", value: 120, measuredAt: at("2026-05-01T10:00:00Z") },
|
||||
// Equidistant (±2 min) between the 10:03 and 10:07 dia rows.
|
||||
{ id: "s2", value: 124, measuredAt: at("2026-05-01T10:05:00Z") },
|
||||
{ id: "s3", value: 130, measuredAt: at("2026-05-01T10:10:00Z") },
|
||||
// > 5 min from every diastolic row — stays unpaired.
|
||||
{ id: "s4", value: 140, measuredAt: at("2026-05-01T11:00:00Z") },
|
||||
]) as never;
|
||||
}
|
||||
if (a.where.type === "BLOOD_PRESSURE_DIA") {
|
||||
return Promise.resolve([
|
||||
{ id: "d1", value: 80, measuredAt: at("2026-05-01T10:01:00Z") },
|
||||
{ id: "d2", value: 82, measuredAt: at("2026-05-01T10:03:00Z") },
|
||||
{ id: "d3", value: 84, measuredAt: at("2026-05-01T10:07:00Z") },
|
||||
{ id: "d4", value: 86, measuredAt: at("2026-05-01T10:11:00Z") },
|
||||
]) as never;
|
||||
}
|
||||
return Promise.resolve([]) as never;
|
||||
}) as never);
|
||||
const res = await GET(req("kind=bloodPressure&days=30"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: { points: Array<{ id: string; secondary: number | null }> };
|
||||
};
|
||||
expect(body.data.points.map((p) => [p.id, p.secondary])).toEqual([
|
||||
["s1", 80], // nearest is d1 (1 min)
|
||||
["s2", 82], // d2/d3 tie at 2 min — earlier row wins
|
||||
["s3", 86], // nearest is d4 (1 min)
|
||||
["s4", null], // nothing within ±5 min
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns single-value series for kind=weight", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
@@ -377,6 +423,184 @@ describe("GET /api/measurements/series", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)", () => {
|
||||
it("day-buckets glucose in SQL beyond the 90-day raw window", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
// 1st query — day buckets.
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
bucket_start: new Date("2025-01-01T00:00:00Z"),
|
||||
mean: 100,
|
||||
min_value: 70,
|
||||
max_value: 180,
|
||||
},
|
||||
{
|
||||
bucket_start: new Date("2025-01-02T00:00:00Z"),
|
||||
mean: 126,
|
||||
min_value: 80,
|
||||
max_value: 200,
|
||||
},
|
||||
] as never)
|
||||
// 2nd query — raw-row aggregate for the stats strip.
|
||||
.mockResolvedValueOnce([
|
||||
{ n: 576, mean: 113.04, min: 70, max: 200, sd: 22.34 },
|
||||
] as never);
|
||||
|
||||
const res = await GET(req("kind=glucose&days=365"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
unit: string;
|
||||
points: Array<{
|
||||
id: string;
|
||||
at: string;
|
||||
value: number;
|
||||
secondary: number | null;
|
||||
}>;
|
||||
stats: {
|
||||
mean: number;
|
||||
min: number;
|
||||
max: number;
|
||||
stdDev: number;
|
||||
count: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
// No raw row walk — the read is SQL-aggregate only.
|
||||
expect(prisma.measurement.findMany).not.toHaveBeenCalled();
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(2);
|
||||
expect(body.data.unit).toBe("mg/dL");
|
||||
expect(body.data.points).toEqual([
|
||||
{
|
||||
id: "day:2025-01-01",
|
||||
at: "2025-01-01T00:00:00.000Z",
|
||||
value: 100,
|
||||
secondary: null,
|
||||
},
|
||||
{
|
||||
id: "day:2025-01-02",
|
||||
at: "2025-01-02T00:00:00.000Z",
|
||||
value: 126,
|
||||
secondary: null,
|
||||
},
|
||||
]);
|
||||
// Stats come from the RAW-row aggregate (count = raw samples, not
|
||||
// day buckets), matching the raw path's mg/dL parity convention.
|
||||
expect(body.data.stats).toEqual({
|
||||
mean: 113,
|
||||
min: 70,
|
||||
max: 200,
|
||||
stdDev: 22,
|
||||
count: 576,
|
||||
});
|
||||
});
|
||||
|
||||
it("converts day-bucketed glucose for a mmol/L-preference user", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({
|
||||
glucoseUnit: "mmol/L",
|
||||
} as never);
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
bucket_start: new Date("2025-01-01T00:00:00Z"),
|
||||
mean: 100,
|
||||
min_value: 70,
|
||||
max_value: 180,
|
||||
},
|
||||
] as never)
|
||||
.mockResolvedValueOnce([
|
||||
{ n: 288, mean: 100, min: 70, max: 180, sd: 18 },
|
||||
] as never);
|
||||
|
||||
const res = await GET(req("kind=glucose&days=3650"));
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
unit: string;
|
||||
points: Array<{ value: number }>;
|
||||
stats: { mean: number };
|
||||
};
|
||||
};
|
||||
expect(body.data.unit).toBe("mmol/L");
|
||||
expect(body.data.points[0].value).toBe(5.5); // 100 mg/dL → 5.5 mmol/L
|
||||
expect(body.data.stats.mean).toBe(5.5);
|
||||
});
|
||||
|
||||
it("day-buckets pulse beyond 90 days with the day's min/max band", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
bucket_start: new Date("2025-06-01T00:00:00Z"),
|
||||
mean: 71.666,
|
||||
min_value: 52,
|
||||
max_value: 143,
|
||||
},
|
||||
] as never)
|
||||
.mockResolvedValueOnce([
|
||||
{ n: 24, mean: 71.666, min: 58, max: 132, sd: 9.876 },
|
||||
] as never);
|
||||
|
||||
const res = await GET(req("kind=pulse&days=365"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
points: Array<{
|
||||
id: string;
|
||||
value: number;
|
||||
valueMin: number | null;
|
||||
valueMax: number | null;
|
||||
}>;
|
||||
stats: { mean: number; stdDev: number; count: number };
|
||||
};
|
||||
};
|
||||
expect(prisma.measurement.findMany).not.toHaveBeenCalled();
|
||||
expect(body.data.points).toEqual([
|
||||
{
|
||||
id: "day:2025-06-01",
|
||||
at: "2025-06-01T00:00:00.000Z",
|
||||
value: 71.67,
|
||||
secondary: null,
|
||||
valueMin: 52,
|
||||
valueMax: 143,
|
||||
},
|
||||
]);
|
||||
expect(body.data.stats.mean).toBe(71.67);
|
||||
expect(body.data.stats.stdDev).toBe(9.88);
|
||||
expect(body.data.stats.count).toBe(24);
|
||||
});
|
||||
|
||||
it("keeps the 90-day window on the raw path (byte-identical short ranges)", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{ id: "g1", value: 100, measuredAt: new Date("2026-06-01T08:00:00Z") },
|
||||
] as never);
|
||||
const res = await GET(req("kind=glucose&days=90"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: { points: Array<{ id: string }> };
|
||||
};
|
||||
// Raw read, individual row ids — the SQL bucket path never fires.
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(body.data.points[0].id).toBe("g1");
|
||||
});
|
||||
|
||||
it("keeps sparse kinds raw even on the ten-year window", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{ id: "w1", value: 78.4, measuredAt: new Date() },
|
||||
] as never);
|
||||
const res = await GET(req("kind=weight&days=3650"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: { points: Array<{ id: string }> };
|
||||
};
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(body.data.points[0].id).toBe("w1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/measurements/series — 422 multi-issue (v1.4.43 W6)", () => {
|
||||
it("surfaces TWO simultaneous validation errors", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
|
||||
@@ -35,6 +35,21 @@ import { convertGlucose, resolveGlucoseUnit } from "@/lib/glucose";
|
||||
*/
|
||||
const SLEEP_SERIES_MAX_DAYS = 365;
|
||||
|
||||
/**
|
||||
* v1.28.25 — sample-dense kinds read raw rows only up to this window.
|
||||
* A CGM account stores ~288 BLOOD_GLUCOSE rows/day (105k+ rows/year) and
|
||||
* per-sample PULSE piles up similarly, so the 365 / 3650-day requests
|
||||
* walked a six-figure row set into JS, mapped it twice, and serialized
|
||||
* everything. Mirrors the sleep precedent above, but instead of silently
|
||||
* clamping the window, requests beyond the cap keep their full range and
|
||||
* are day-bucketed in SQL (one point per day, mean + min/max band) — the
|
||||
* chart still shows the whole requested history, just at daily grain.
|
||||
* The short windows users actually browse (7 / 30 / 90 d) stay raw and
|
||||
* byte-identical. Sparse kinds (weight, BP, …) are untouched.
|
||||
*/
|
||||
const DENSE_SERIES_RAW_WINDOW_DAYS = 90;
|
||||
const DENSE_SERIES_KINDS: ReadonlySet<string> = new Set(["glucose", "pulse"]);
|
||||
|
||||
const kindEnum = z.enum([
|
||||
"weight",
|
||||
"bloodPressure",
|
||||
@@ -204,7 +219,9 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
// v1.16.16 — when glucose stats are computed over RAW mg/dL (parity with the
|
||||
// detail page + FHIR), the branch fills this so the response below skips the
|
||||
// generic over-points path that would double-round mmol/L figures.
|
||||
let glucoseStats: {
|
||||
// v1.28.25 — the dense day-bucket branch also fills it (stats stay
|
||||
// aggregated over the RAW rows in SQL, not over the day-bucket points).
|
||||
let statsOverride: {
|
||||
mean: number;
|
||||
min: number;
|
||||
max: number;
|
||||
@@ -306,24 +323,155 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
}),
|
||||
]);
|
||||
|
||||
// Pair by closest timestamp within ±5 minutes.
|
||||
// Pair by closest timestamp within ±5 minutes. Both reads arrive
|
||||
// sorted ascending, so the nearest-diastolic index is non-decreasing
|
||||
// across the systolic walk — a two-pointer merge finds each pair in
|
||||
// O(sys + dia) instead of the previous full diastolic rescan per
|
||||
// systolic row (O(sys × dia) — quadratic on dense imports). Ties keep
|
||||
// the earlier diastolic row, matching the old scan's strict-`<` keep.
|
||||
const PAIR_WINDOW_MS = 5 * 60_000;
|
||||
let diaIdx = 0;
|
||||
points = sys.map((s) => {
|
||||
let bestDia: { value: number; dt: number } | null = null;
|
||||
for (const d of dia) {
|
||||
const dt = Math.abs(d.measuredAt.getTime() - s.measuredAt.getTime());
|
||||
if (dt > PAIR_WINDOW_MS) continue;
|
||||
if (!bestDia || dt < bestDia.dt) {
|
||||
bestDia = { value: d.value, dt };
|
||||
}
|
||||
const sysMs = s.measuredAt.getTime();
|
||||
while (
|
||||
diaIdx + 1 < dia.length &&
|
||||
Math.abs(dia[diaIdx + 1].measuredAt.getTime() - sysMs) <
|
||||
Math.abs(dia[diaIdx].measuredAt.getTime() - sysMs)
|
||||
) {
|
||||
diaIdx += 1;
|
||||
}
|
||||
const best = dia[diaIdx];
|
||||
const paired =
|
||||
best !== undefined &&
|
||||
Math.abs(best.measuredAt.getTime() - sysMs) <= PAIR_WINDOW_MS;
|
||||
return {
|
||||
id: s.id,
|
||||
at: s.measuredAt.toISOString(),
|
||||
value: s.value,
|
||||
secondary: bestDia?.value ?? null,
|
||||
secondary: paired ? best.value : null,
|
||||
};
|
||||
});
|
||||
} else if (
|
||||
DENSE_SERIES_KINDS.has(kind) &&
|
||||
days > DENSE_SERIES_RAW_WINDOW_DAYS
|
||||
) {
|
||||
// v1.28.25 — long-window read of a sample-dense kind (CGM glucose,
|
||||
// per-sample / hourly pulse). Day-bucket in SQL instead of walking
|
||||
// every raw row into JS: one aggregate pass in Postgres returns at
|
||||
// most `days` rows regardless of sample density. Parameter-bound
|
||||
// tagged-template `$queryRaw`, mirroring the rollup tier's
|
||||
// `date_trunc(... ) GROUP BY` aggregate (measurement-rollups.ts).
|
||||
const type = KIND_TO_TYPE[kind];
|
||||
const bucketRows = await prisma.$queryRaw<
|
||||
Array<{
|
||||
bucket_start: Date;
|
||||
mean: number;
|
||||
min_value: number;
|
||||
max_value: number;
|
||||
}>
|
||||
>`
|
||||
SELECT
|
||||
date_trunc('day', m."measured_at") AS bucket_start,
|
||||
AVG(m."value")::double precision AS mean,
|
||||
MIN(COALESCE(m."value_min", m."value"))::double precision AS min_value,
|
||||
MAX(COALESCE(m."value_max", m."value"))::double precision AS max_value
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${user.id}
|
||||
AND m."type" = ${type}::"measurement_type"
|
||||
AND m."measured_at" >= ${since}
|
||||
AND m."deleted_at" IS NULL
|
||||
GROUP BY date_trunc('day', m."measured_at")
|
||||
ORDER BY bucket_start ASC
|
||||
`;
|
||||
// Stats stay aggregated over the RAW rows (not the day buckets) so the
|
||||
// strip's mean/min/max/stdDev/count are the same figures the raw path
|
||||
// computed: `AVG`/`MIN`/`MAX` over `value`, `STDDEV_POP` matching the
|
||||
// JS population-variance helper, exact row count. Aggregate-only —
|
||||
// no row transfer.
|
||||
const [rawAgg] = await prisma.$queryRaw<
|
||||
Array<{
|
||||
n: number;
|
||||
mean: number | null;
|
||||
min: number | null;
|
||||
max: number | null;
|
||||
sd: number | null;
|
||||
}>
|
||||
>`
|
||||
SELECT
|
||||
COUNT(*)::int AS n,
|
||||
AVG(m."value")::double precision AS mean,
|
||||
MIN(m."value")::double precision AS min,
|
||||
MAX(m."value")::double precision AS max,
|
||||
STDDEV_POP(m."value")::double precision AS sd
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${user.id}
|
||||
AND m."type" = ${type}::"measurement_type"
|
||||
AND m."measured_at" >= ${since}
|
||||
AND m."deleted_at" IS NULL
|
||||
`;
|
||||
const round2 = (v: number) => Math.round(v * 100) / 100;
|
||||
const dayId = (d: Date) => `day:${d.toISOString().slice(0, 10)}`;
|
||||
if (kind === "glucose") {
|
||||
// Same unit engine as the raw glucose branch: canonical mg/dL in
|
||||
// the DB, converted ONCE at serialization to the user's preference.
|
||||
const profile = await prisma.user.findUnique({
|
||||
where: { id: user.id },
|
||||
select: { glucoseUnit: true },
|
||||
});
|
||||
const glucoseUnit = resolveGlucoseUnit(profile?.glucoseUnit ?? null);
|
||||
unit = glucoseUnit;
|
||||
points = bucketRows.map((r) => ({
|
||||
id: dayId(r.bucket_start),
|
||||
at: r.bucket_start.toISOString(),
|
||||
value: convertGlucose(r.mean, glucoseUnit),
|
||||
secondary: null,
|
||||
}));
|
||||
if (
|
||||
rawAgg !== undefined &&
|
||||
rawAgg.n > 0 &&
|
||||
rawAgg.mean !== null &&
|
||||
rawAgg.min !== null &&
|
||||
rawAgg.max !== null
|
||||
) {
|
||||
// Mirrors the raw branch's v1.16.16 parity: round the mg/dL
|
||||
// aggregate to 2 decimals (summarize()'s convention), then
|
||||
// convert each figure once.
|
||||
statsOverride = {
|
||||
mean: convertGlucose(round2(rawAgg.mean), glucoseUnit),
|
||||
min: convertGlucose(rawAgg.min, glucoseUnit),
|
||||
max: convertGlucose(rawAgg.max, glucoseUnit),
|
||||
stdDev: convertGlucose(round2(rawAgg.sd ?? 0), glucoseUnit),
|
||||
count: rawAgg.n,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// pulse — `value` is the day's average; valueMin/valueMax carry the
|
||||
// day's low/high band (folding each hourly bucket's own spread via
|
||||
// the COALESCE above), same band semantics as the per-hour shape.
|
||||
points = bucketRows.map((r) => ({
|
||||
id: dayId(r.bucket_start),
|
||||
at: r.bucket_start.toISOString(),
|
||||
value: round2(r.mean),
|
||||
secondary: null,
|
||||
valueMin: r.min_value,
|
||||
valueMax: r.max_value,
|
||||
}));
|
||||
if (
|
||||
rawAgg !== undefined &&
|
||||
rawAgg.n > 0 &&
|
||||
rawAgg.mean !== null &&
|
||||
rawAgg.min !== null &&
|
||||
rawAgg.max !== null
|
||||
) {
|
||||
statsOverride = {
|
||||
mean: round2(rawAgg.mean),
|
||||
min: rawAgg.min,
|
||||
max: rawAgg.max,
|
||||
stdDev: round2(rawAgg.sd ?? 0),
|
||||
count: rawAgg.n,
|
||||
};
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const type = KIND_TO_TYPE[kind];
|
||||
// v1.19.2 (iOS #34 extension) — pull the per-bucket spread for the
|
||||
@@ -371,7 +519,7 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
// already-converted+rounded points double-rounds the mean and stdDev
|
||||
// for mmol/L users, drifting the last decimal off the detail page.
|
||||
// Single-reading sets stay identical (100 → 5.5). Hand back stats here
|
||||
// so the shared block below leaves a populated `glucoseStats` alone.
|
||||
// so the shared block below leaves a populated `statsOverride` alone.
|
||||
const rawDataPoints: DataPoint[] = rows.map((r) => ({
|
||||
date: r.measuredAt,
|
||||
value: r.value,
|
||||
@@ -379,7 +527,7 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
if (rawDataPoints.length > 0) {
|
||||
const rawSummary = summarize(rawDataPoints);
|
||||
const rawValues = rawDataPoints.map((p) => p.value);
|
||||
glucoseStats = {
|
||||
statsOverride = {
|
||||
mean:
|
||||
rawSummary.mean === null
|
||||
? 0
|
||||
@@ -443,7 +591,7 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
unit,
|
||||
points,
|
||||
stats:
|
||||
glucoseStats ??
|
||||
statsOverride ??
|
||||
(summary
|
||||
? {
|
||||
mean: summary.mean,
|
||||
|
||||
@@ -413,3 +413,71 @@ describe("POST /api/mood-entries/bulk — structured tagKeys (v1.12.0)", () => {
|
||||
expect(json.data.entries[1].status).toBe("inserted");
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/mood-entries/bulk — tombstone suppression", () => {
|
||||
it("keeps a tombstoned match deleted: true no-op reported as duplicate", async () => {
|
||||
// A soft-deleted row under the probe key: the sync feed already
|
||||
// surfaced the deletion to paired clients, so a stale offline re-post
|
||||
// must NOT resurrect it — and must not churn the hidden row either.
|
||||
vi.mocked(prisma.moodEntry.findUnique).mockResolvedValue({
|
||||
id: "entry-tomb",
|
||||
deletedAt: new Date("2026-05-01T00:00:00.000Z"),
|
||||
} as never);
|
||||
const res = await POST(
|
||||
postReq({
|
||||
entries: [
|
||||
{
|
||||
mood: "GUT",
|
||||
moodLoggedAt: "2026-05-16T08:00:00.000Z",
|
||||
externalId: "ios-uuid-tomb",
|
||||
tagKeys: ["movies"],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const json = (await res.json()) as {
|
||||
data: {
|
||||
inserted: number;
|
||||
duplicates: number;
|
||||
entries: Array<{ status: string; id?: string; externalId?: string }>;
|
||||
};
|
||||
};
|
||||
expect(json.data.inserted).toBe(0);
|
||||
expect(json.data.duplicates).toBe(1);
|
||||
expect(json.data.entries[0].status).toBe("duplicate");
|
||||
expect(json.data.entries[0].id).toBe("entry-tomb");
|
||||
expect(json.data.entries[0].externalId).toBe("ios-uuid-tomb");
|
||||
// No write at all on the hidden row: no upsert (value churn /
|
||||
// updatedAt bump), no tag-link insert.
|
||||
expect(prisma.moodEntry.upsert).not.toHaveBeenCalled();
|
||||
expect(createTagLinks).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still refreshes a LIVE match in place (duplicate + upsert runs)", async () => {
|
||||
vi.mocked(prisma.moodEntry.findUnique).mockResolvedValue({
|
||||
id: "entry-live",
|
||||
deletedAt: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.moodEntry.upsert).mockResolvedValue({
|
||||
id: "entry-live",
|
||||
} as never);
|
||||
const res = await POST(
|
||||
postReq({
|
||||
entries: [
|
||||
{
|
||||
mood: "OKAY",
|
||||
moodLoggedAt: "2026-05-16T08:00:00.000Z",
|
||||
externalId: "ios-uuid-live",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const json = (await res.json()) as {
|
||||
data: { duplicates: number };
|
||||
};
|
||||
expect(json.data.duplicates).toBe(1);
|
||||
expect(prisma.moodEntry.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -227,9 +227,28 @@ async function postBulk(request: NextRequest): Promise<Response> {
|
||||
// grows.
|
||||
const existing = await prisma.moodEntry.findUnique({
|
||||
where: probeWhere,
|
||||
select: { id: true },
|
||||
select: { id: true, deletedAt: true },
|
||||
});
|
||||
|
||||
// Tombstone suppression: a soft-deleted match stays deleted. The
|
||||
// user-facing DELETE route flips `deletedAt` so the `/api/sync/changes`
|
||||
// feed surfaces the deletion to paired clients offline at delete time
|
||||
// (see the MoodEntry schema note) — an offline client's later re-post
|
||||
// of the same entry is stale state, not a recreation, and must not
|
||||
// resurrect the row. True no-op: no value churn, no `updatedAt` bump,
|
||||
// no tag-link writes on the hidden row; report `duplicate` so the
|
||||
// client checkpoints past it exactly like a live-row match.
|
||||
if (existing?.deletedAt) {
|
||||
duplicates += 1;
|
||||
results.push({
|
||||
index: i,
|
||||
status: "duplicate",
|
||||
id: existing.id,
|
||||
...(entry.externalId ? { externalId: entry.externalId } : {}),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await prisma.moodEntry.upsert({
|
||||
where: probeWhere,
|
||||
create: {
|
||||
|
||||
@@ -866,6 +866,40 @@ export async function collectDoctorReportData(
|
||||
const recoveryEnabled = moduleMap.recovery !== false;
|
||||
const workoutsEnabled = moduleMap.workouts !== false;
|
||||
|
||||
// v1.28.25 — push the section / module gates INTO the measurement query.
|
||||
// `filterMeasurementKeys` below strips gated types from the returned
|
||||
// payload anyway, so rows of a disabled section / module were fetched only
|
||||
// to be dropped — pure dead weight, and the heaviest series (BLOOD_GLUCOSE
|
||||
// minute-grain CGM, gated by the glucose module) is exactly the one that
|
||||
// runs to six figures over a 730-day window. Excluding them at the query
|
||||
// is output-identical by construction.
|
||||
//
|
||||
// WEIGHT is deliberately NEVER excluded: the BMI figure (gated by
|
||||
// `sections.bmi`, not `sections.weight`) and the GLP-1 weight-delta both
|
||||
// read the PRE-filter `byType` / `stats` maps, so weight rows must be
|
||||
// present even when the weight section itself is toggled off. No other
|
||||
// gated type is read before the filter (each remaining pre-filter read —
|
||||
// sleep nights, glucose panel, recovery summary — sits behind the same
|
||||
// gate that drives its exclusion here).
|
||||
//
|
||||
// NOTE a closed include-list is NOT possible here: every ungated type in
|
||||
// the window lands in `stats` / `measurements`, which the clinician share
|
||||
// view and the MCP doctor-visit summary iterate wholesale
|
||||
// (`Object.entries(data.stats)`). Dense ENABLED types (BLOOD_GLUCOSE
|
||||
// minute-grain for a glucose-module user, hourly PULSE) therefore remain a
|
||||
// known load — the per-day bucket tier is the follow-up for that.
|
||||
const excludedMeasurementTypes: MeasurementType[] = [];
|
||||
if (sections.bp === false) {
|
||||
excludedMeasurementTypes.push("BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA");
|
||||
}
|
||||
if (sections.pulse === false) excludedMeasurementTypes.push("PULSE");
|
||||
if (sections.sleep === false) excludedMeasurementTypes.push("SLEEP_DURATION");
|
||||
for (const [type, moduleKey] of Object.entries(MEASUREMENT_TYPE_MODULE)) {
|
||||
if (moduleMap[moduleKey] === false) {
|
||||
excludedMeasurementTypes.push(type as MeasurementType);
|
||||
}
|
||||
}
|
||||
|
||||
const [measurements, medications, intakeEvents, moodEntries, userProfile] =
|
||||
await Promise.all([
|
||||
prisma.measurement.findMany({
|
||||
@@ -876,8 +910,27 @@ export async function collectDoctorReportData(
|
||||
userId,
|
||||
measuredAt: { gte: start, lte: end },
|
||||
deletedAt: null,
|
||||
...(excludedMeasurementTypes.length > 0
|
||||
? { type: { notIn: excludedMeasurementTypes } }
|
||||
: {}),
|
||||
},
|
||||
orderBy: { measuredAt: "asc" },
|
||||
// v1.28.25 — narrow select. The collector reads exactly these seven
|
||||
// fields (canonical-source collapse: type / value / measuredAt /
|
||||
// source / deviceType; sleep-night reconstruction adds sleepStage;
|
||||
// the glucose panel adds glucoseContext). The full-width read pulled
|
||||
// every column — including the notesEncrypted Bytes — across up to
|
||||
// 730 days of rows, which on a CGM + per-sample-HR account is the
|
||||
// v1.28.2x six-figure-row incident class.
|
||||
select: {
|
||||
type: true,
|
||||
value: true,
|
||||
measuredAt: true,
|
||||
source: true,
|
||||
deviceType: true,
|
||||
sleepStage: true,
|
||||
glucoseContext: true,
|
||||
},
|
||||
}),
|
||||
prisma.medication.findMany({
|
||||
where: { userId, active: true },
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { readNote } from "@/lib/crypto/note-cipher";
|
||||
import { findMeasurementsPaged } from "@/lib/export/paged-measurements";
|
||||
import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup";
|
||||
import { buildCycleBackupSection } from "@/lib/cycle/backup";
|
||||
|
||||
@@ -39,10 +40,28 @@ export async function buildFullBackupPayload(
|
||||
): Promise<FullBackupResult> {
|
||||
const [measurements, medications, intakeEvents, moodEntries, cycle] =
|
||||
await Promise.all([
|
||||
prisma.measurement.findMany({
|
||||
where: { userId, deletedAt: null },
|
||||
orderBy: { measuredAt: "desc" },
|
||||
}),
|
||||
// v1.28.25 — keyset-paginated read with a narrow select. The backup
|
||||
// is inherently whole-history, so on a CGM / per-sample-HR account the
|
||||
// previous single full-width `findMany` materialised a six-figure row
|
||||
// set in one shot. The chunked read accumulates the same rows in the
|
||||
// same `measuredAt desc` order; the payload FORMAT below is untouched
|
||||
// (restore compatibility). The select is exactly the fields the
|
||||
// serializer reads: type / value / unit / measuredAt / source + the
|
||||
// note columns `readNote` decrypts + the `id` cursor key.
|
||||
findMeasurementsPaged(
|
||||
prisma,
|
||||
{ userId, deletedAt: null },
|
||||
{
|
||||
id: true,
|
||||
type: true,
|
||||
value: true,
|
||||
unit: true,
|
||||
measuredAt: true,
|
||||
source: true,
|
||||
notes: true,
|
||||
notesEncrypted: true,
|
||||
},
|
||||
),
|
||||
prisma.medication.findMany({
|
||||
where: { userId },
|
||||
select: {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Keyset-paginated measurement reader for the export surfaces.
|
||||
*
|
||||
* The legacy exports (`GET /api/export`, `GET /api/export/measurements`, the
|
||||
* full-backup builder) read the user's ENTIRE measurement table in one
|
||||
* `findMany`. On a CGM + per-sample-HR account that is a six-figure row set
|
||||
* materialised by a single query — the v1.28.2x incident class. This helper
|
||||
* keeps the output row set (and its `measuredAt desc` order) identical while
|
||||
* bounding each database round-trip to `MEASUREMENT_EXPORT_PAGE_SIZE` rows:
|
||||
* the driver never buffers one giant result, and the event loop yields
|
||||
* between chunks.
|
||||
*
|
||||
* Ordering: `measuredAt desc, id desc`. The exports always sorted
|
||||
* `measuredAt desc`; the `id` tie-breaker makes the keyset cursor total (two
|
||||
* rows can share a timestamp) — for equal-timestamp rows the previous order
|
||||
* was unspecified heap order, so pinning it is not a behaviour change any
|
||||
* consumer could have relied on.
|
||||
*
|
||||
* The `select` MUST include `id` (the cursor key); the generic constraint
|
||||
* enforces it at compile time.
|
||||
*/
|
||||
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
|
||||
|
||||
export const MEASUREMENT_EXPORT_PAGE_SIZE = 10_000;
|
||||
|
||||
export async function findMeasurementsPaged<
|
||||
S extends Prisma.MeasurementSelect & { id: true },
|
||||
>(
|
||||
db: PrismaClient,
|
||||
where: Prisma.MeasurementWhereInput,
|
||||
select: S,
|
||||
pageSize: number = MEASUREMENT_EXPORT_PAGE_SIZE,
|
||||
): Promise<Array<Prisma.MeasurementGetPayload<{ select: S }>>> {
|
||||
const rows: Array<Prisma.MeasurementGetPayload<{ select: S }>> = [];
|
||||
let cursorId: string | null = null;
|
||||
for (;;) {
|
||||
const batch: Array<Prisma.MeasurementGetPayload<{ select: S }>> =
|
||||
(await db.measurement.findMany({
|
||||
where,
|
||||
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
|
||||
take: pageSize,
|
||||
...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}),
|
||||
select,
|
||||
})) as Array<Prisma.MeasurementGetPayload<{ select: S }>>;
|
||||
// Loop, not `push(...batch)` — a 10k-element spread is fine today, but
|
||||
// this file exists because of stack-overflow-by-spread regressions on
|
||||
// large arrays (v1.28.22 class); keep the pattern boring.
|
||||
for (const row of batch) rows.push(row);
|
||||
if (batch.length < pageSize) break;
|
||||
cursorId = (batch[batch.length - 1] as { id: string }).id;
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
@@ -538,8 +538,11 @@ describe("sleep mapping (1.2 levels.data)", () => {
|
||||
const sessions = readSleepSessions({
|
||||
sleep: [
|
||||
{
|
||||
// No logId → the anchor falls back to the END instant, which must
|
||||
// also be tz-resolved so the externalId is stable across syncs.
|
||||
// No logId → the anchor falls back to the START instant (the
|
||||
// session's identity; the end moves when Fitbit re-scores a night),
|
||||
// which must also be tz-resolved so the externalId is stable across
|
||||
// syncs. endTime is present to prove start wins the fallback.
|
||||
startTime: "2026-05-10T23:00:00.000",
|
||||
endTime: "2026-05-11T00:30:00.000",
|
||||
levels: {
|
||||
data: [
|
||||
@@ -556,8 +559,8 @@ describe("sleep mapping (1.2 levels.data)", () => {
|
||||
|
||||
const out = mapSleepSession(sessions[0]!, "Europe/Berlin");
|
||||
expect(out).toHaveLength(1);
|
||||
// anchor = end instant ISO (UTC) → fieldTag starts with that instant.
|
||||
expect(out[0]!.fieldTag).toBe("2026-05-10T22:30:00.000Z:sleep_rem:0");
|
||||
// anchor = start instant ISO (UTC) → fieldTag starts with that instant.
|
||||
expect(out[0]!.fieldTag).toBe("2026-05-10T21:00:00.000Z:sleep_rem:0");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Pins the tombstone-resurrect contract of `upsertFitbitMeasurements`.
|
||||
*
|
||||
* The measurements unique index on `(userId, type, source, externalId)` is
|
||||
* FULL — it covers soft-deleted rows — so a tombstoned row permanently owns
|
||||
* its key. The probe must therefore match tombstoned rows too and route them
|
||||
* into the UPDATE branch with `deletedAt: null`: treating them as absent plans
|
||||
* an insert that `skipDuplicates` drops SILENTLY against the tombstone's key,
|
||||
* wedging the key forever (the same wedge shipped live on the Google
|
||||
* transport).
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
|
||||
findManyMock: vi.fn(async () => [] as unknown[]),
|
||||
createManyMock: vi.fn(async () => ({ count: 0 })),
|
||||
updateMock: vi.fn(async () => ({})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findMany: findManyMock,
|
||||
createMany: createManyMock,
|
||||
update: updateMock,
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn(async () => false),
|
||||
recordSyncFailure: vi.fn(async () => {}),
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
recomputeBucketsForMeasurement: vi.fn(async () => {}),
|
||||
recomputeUserRollups: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/integrations/oauth-refresh", () => ({
|
||||
persistRotatedToken: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserFitbitCredentials: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../client", () => ({ refreshAccessToken: vi.fn() }));
|
||||
|
||||
import { upsertFitbitMeasurements } from "../sync";
|
||||
|
||||
const STEPS_READING = {
|
||||
type: "ACTIVITY_STEPS",
|
||||
value: 8123,
|
||||
unit: "steps",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
externalId: "stats:steps:2026-07-08",
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
findManyMock.mockReset().mockResolvedValue([]);
|
||||
createManyMock.mockReset().mockResolvedValue({ count: 0 });
|
||||
updateMock.mockReset().mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe("upsertFitbitMeasurements — tombstones resurrect", () => {
|
||||
it("probes WITHOUT a deletedAt filter (a tombstoned row must be matched)", async () => {
|
||||
await upsertFitbitMeasurements("user-1", [STEPS_READING], {
|
||||
deferRollup: true,
|
||||
});
|
||||
const arg = (findManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
where: Record<string, unknown>;
|
||||
};
|
||||
expect(arg.where).toEqual({
|
||||
userId: "user-1",
|
||||
source: "FITBIT",
|
||||
externalId: { in: ["stats:steps:2026-07-08"] },
|
||||
});
|
||||
expect(arg.where).not.toHaveProperty("deletedAt");
|
||||
});
|
||||
|
||||
it("routes a matched (tombstoned) row into an update that clears deletedAt", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "ACTIVITY_STEPS",
|
||||
externalId: "stats:steps:2026-07-08",
|
||||
},
|
||||
]);
|
||||
|
||||
const { imported } = await upsertFitbitMeasurements(
|
||||
"user-1",
|
||||
[STEPS_READING],
|
||||
{ deferRollup: true },
|
||||
);
|
||||
|
||||
// No insert is planned for an owned key — the update resurrects in place.
|
||||
expect(createManyMock).not.toHaveBeenCalled();
|
||||
expect(updateMock).toHaveBeenCalledTimes(1);
|
||||
const upd = (updateMock.mock.calls[0] as unknown[])[0] as {
|
||||
where: { id: string };
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(upd.where).toEqual({ id: "row-1" });
|
||||
expect(upd.data.deletedAt).toBeNull();
|
||||
expect(upd.data.value).toBe(8123);
|
||||
expect(upd.data.syncVersion).toEqual({ increment: 1 });
|
||||
expect(imported).toBe(1);
|
||||
});
|
||||
|
||||
it("still creates a genuinely fresh key", async () => {
|
||||
createManyMock.mockResolvedValue({ count: 1 });
|
||||
const { imported } = await upsertFitbitMeasurements(
|
||||
"user-1",
|
||||
[STEPS_READING],
|
||||
{ deferRollup: true },
|
||||
);
|
||||
expect(updateMock).not.toHaveBeenCalled();
|
||||
expect(createManyMock).toHaveBeenCalledTimes(1);
|
||||
expect(imported).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -237,7 +237,7 @@ describe("getValidToken — rotating refresh", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => {
|
||||
describe("upsertFitbitMeasurements — batched write, tombstones resurrect", () => {
|
||||
it("inserts a fresh reading via createMany with field-by-field data, source pinned to FITBIT", async () => {
|
||||
const measuredAt = new Date("2026-05-10T07:00:00.000Z");
|
||||
prismaMock.measurement.findMany.mockResolvedValue([]); // nothing live yet
|
||||
@@ -255,16 +255,17 @@ describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => {
|
||||
expect(imported).toBe(1);
|
||||
expect(prismaMock.measurement.update).not.toHaveBeenCalled();
|
||||
|
||||
// The probe filters to LIVE FITBIT rows for the batch's externalIds.
|
||||
// The probe matches EVERY FITBIT row for the batch's externalIds — live
|
||||
// AND tombstoned (the full unique index owns the key either way).
|
||||
const probeArg = prismaMock.measurement.findMany.mock.calls[0]![0] as {
|
||||
where: Record<string, unknown>;
|
||||
};
|
||||
expect(probeArg.where).toMatchObject({
|
||||
userId: "user1",
|
||||
source: "FITBIT",
|
||||
deletedAt: null,
|
||||
externalId: { in: ["2026-05-10T07:00:00.000Z:weight"] },
|
||||
});
|
||||
expect(probeArg.where).not.toHaveProperty("deletedAt");
|
||||
|
||||
const createArg = prismaMock.measurement.createMany.mock.calls[0]![0] as {
|
||||
data: Record<string, unknown>[];
|
||||
@@ -310,11 +311,14 @@ describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => {
|
||||
expect(updateArg.data.syncVersion).toEqual({ increment: 1 });
|
||||
});
|
||||
|
||||
it("does NOT resurrect a soft-deleted row — a tombstone is absent from the live probe, so a fresh insert is created", async () => {
|
||||
it("RESURRECTS a soft-deleted row — the probe matches the tombstone and the update clears deletedAt", async () => {
|
||||
const measuredAt = new Date("2026-05-10T07:00:00.000Z");
|
||||
// The user deleted this Fitbit reading: the tombstone (deletedAt set) is
|
||||
// excluded from the live probe, so findMany returns NOTHING for the key.
|
||||
prismaMock.measurement.findMany.mockResolvedValue([]);
|
||||
// The tombstone still owns its key under the FULL unique index, so the
|
||||
// probe returns it; planning an insert instead would be dropped silently
|
||||
// by `skipDuplicates` and wedge the key forever.
|
||||
prismaMock.measurement.findMany.mockResolvedValue([
|
||||
{ id: "m-dead", type: "WEIGHT", externalId: "stats:weight:2026-05-10" },
|
||||
]);
|
||||
|
||||
const { imported } = await upsertFitbitMeasurements("user1", [
|
||||
{
|
||||
@@ -326,12 +330,14 @@ describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => {
|
||||
},
|
||||
]);
|
||||
|
||||
// The deleted row is NOT updated/resurrected — a fresh insert is attempted
|
||||
// (the partial unique index `WHERE deleted_at IS NULL` keeps it from
|
||||
// colliding with the dead row, and skipDuplicates guards a live race).
|
||||
expect(imported).toBe(1);
|
||||
expect(prismaMock.measurement.update).not.toHaveBeenCalled();
|
||||
expect(prismaMock.measurement.createMany).toHaveBeenCalledTimes(1);
|
||||
expect(prismaMock.measurement.createMany).not.toHaveBeenCalled();
|
||||
const updateArg = prismaMock.measurement.update.mock.calls[0]![0] as {
|
||||
where: { id: string };
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(updateArg.where).toEqual({ id: "m-dead" });
|
||||
expect(updateArg.data.deletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("splits a mixed batch: matched-live → update, unmatched → createMany", async () => {
|
||||
|
||||
@@ -922,14 +922,19 @@ interface FitbitSleepSession {
|
||||
}
|
||||
|
||||
/**
|
||||
* The stable session anchor for a sleep externalId — the logId, else the end (or
|
||||
* start) instant. A re-scored night reuses the same logId → overwrites in place.
|
||||
* The stable session anchor for a sleep externalId — the logId, else the start
|
||||
* (or end) instant. A re-scored night reuses the same logId → overwrites in
|
||||
* place. The fallback prefers `startTime` because it is the session's identity;
|
||||
* `endTime` moves when Fitbit revises a night's scoring, which would mint a new
|
||||
* externalId for the same session. Pre-existing rows keyed on the old endTime
|
||||
* fallback are rare (logId is normally present); a changed key simply creates
|
||||
* one fresh row.
|
||||
*/
|
||||
function sleepSessionAnchor(s: FitbitSleepSession, tz?: string): string {
|
||||
if (typeof s.logId === "number" && Number.isFinite(s.logId)) {
|
||||
return String(s.logId);
|
||||
}
|
||||
for (const t of [s.endTime, s.startTime]) {
|
||||
for (const t of [s.startTime, s.endTime]) {
|
||||
if (typeof t === "string") {
|
||||
const d = parseLocalInstant(t, tz);
|
||||
if (d) return d.toISOString();
|
||||
|
||||
+42
-21
@@ -382,16 +382,23 @@ const FITBIT_CREATE_CHUNK = 500;
|
||||
* (mirrors the WHOOP / Withings sync tail). Returns the count of rows written
|
||||
* plus the distinct `(type, day)` keys the write touched.
|
||||
*
|
||||
* TOMBSTONE-SAFE: the live-row probe filters `deletedAt: null`, so a row the
|
||||
* user soft-deleted (iOS LWW reconciler) is NOT matched — a fresh insert is
|
||||
* created instead of resurrecting the tombstone on the next hourly sync. The
|
||||
* partial unique index (`Measurement … WHERE deleted_at IS NULL`) keeps the
|
||||
* fresh insert from colliding with the live set; the tombstone sits outside it.
|
||||
* TOMBSTONES RESURRECT: the measurements unique index on
|
||||
* `(userId, type, source, externalId)` is FULL — it covers soft-deleted rows
|
||||
* too, so exactly ONE row can ever exist per key, live or tombstoned. An
|
||||
* earlier revision assumed a partial (live-only) index and skipped tombstoned
|
||||
* rows in the probe, expecting a fresh insert; in reality `createMany` with
|
||||
* `skipDuplicates` then hit the tombstoned row's key and dropped the insert
|
||||
* SILENTLY — once a Fitbit row was soft-deleted, that key could never import
|
||||
* again (the same wedge shipped live on the Google transport). The probe now
|
||||
* matches tombstoned rows as well and the update branch clears `deletedAt`:
|
||||
* Fitbit remains the source of truth for its own rows, so a re-import
|
||||
* deliberately revives a deleted one. An operator who wants Fitbit data gone
|
||||
* permanently disconnects the integration rather than deleting rows.
|
||||
*
|
||||
* OVERWRITE CONTRACT preserved: a re-fetched daily summary (`stats:`-keyed and
|
||||
* every other stable per-point anchor) maps to the SAME live `externalId`, so
|
||||
* the probe finds the existing live row and takes the in-place update branch
|
||||
* (bumping `syncVersion` for the iOS LWW reconciler) — never a duplicate.
|
||||
* every other stable per-point anchor) maps to the SAME `externalId`, so the
|
||||
* probe finds the existing row and takes the in-place update branch (bumping
|
||||
* `syncVersion` for the iOS LWW reconciler) — never a duplicate.
|
||||
*
|
||||
* BATCHING: the existence probe is a single `findMany` over the batch's
|
||||
* externalIds; fresh rows go through chunked `createMany` (collapsing the N+1
|
||||
@@ -411,22 +418,24 @@ export async function upsertFitbitMeasurements(
|
||||
}> {
|
||||
if (readings.length === 0) return { imported: 0, touched: [] };
|
||||
|
||||
// Probe the LIVE rows (deletedAt: null) for every externalId in the batch in
|
||||
// a single query. A tombstoned row deliberately does NOT appear here, so it is
|
||||
// treated as absent → a fresh insert, not a resurrecting update.
|
||||
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
|
||||
// in a single query. The full unique index guarantees at most one row per
|
||||
// `(type, externalId)` key, and a tombstoned match must take the UPDATE
|
||||
// branch (which clears `deletedAt`) — treating it as absent would plan an
|
||||
// insert that `skipDuplicates` silently drops against the tombstone's key,
|
||||
// permanently wedging the key (see TOMBSTONES RESURRECT above).
|
||||
const externalIds = readings.map((r) => r.externalId);
|
||||
let liveByKey = new Map<string, { id: string }>();
|
||||
let rowByKey = new Map<string, { id: string }>();
|
||||
try {
|
||||
const existing = await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
source: "FITBIT",
|
||||
deletedAt: null,
|
||||
externalId: { in: externalIds },
|
||||
},
|
||||
select: { id: true, type: true, externalId: true },
|
||||
});
|
||||
liveByKey = new Map(
|
||||
rowByKey = new Map(
|
||||
existing
|
||||
.filter((e) => e.externalId !== null)
|
||||
.map((e) => [
|
||||
@@ -436,9 +445,9 @@ export async function upsertFitbitMeasurements(
|
||||
);
|
||||
} catch (err) {
|
||||
// A probe failure must not strand the whole batch; fall back to treating
|
||||
// every row as fresh (the partial unique index still rejects a genuine
|
||||
// live-row collision, which the per-create catch swallows).
|
||||
getEvent()?.addWarning(`Fitbit: live-row probe failed: ${err}`);
|
||||
// every row as fresh (`skipDuplicates` absorbs the collision on the full
|
||||
// unique index — those rows simply retry on the next sync tick).
|
||||
getEvent()?.addWarning(`Fitbit: row probe failed: ${err}`);
|
||||
}
|
||||
|
||||
const toCreate: Array<{
|
||||
@@ -458,7 +467,7 @@ export async function upsertFitbitMeasurements(
|
||||
for (const r of readings) {
|
||||
const type = r.type as MeasurementType;
|
||||
const key = `${type}${DEDUP_KEY_DELIM}${r.externalId}`;
|
||||
const live = liveByKey.get(key);
|
||||
const live = rowByKey.get(key);
|
||||
if (live) {
|
||||
toUpdate.push({ id: live.id, r });
|
||||
} else if (!plannedCreateKeys.has(key)) {
|
||||
@@ -520,8 +529,11 @@ export async function upsertFitbitMeasurements(
|
||||
}
|
||||
}
|
||||
|
||||
// Live-row overwrites: per-row update (differing values) on the live id, so
|
||||
// the re-fetched daily summary overwrites in place and bumps `syncVersion`.
|
||||
// Existing-row overwrites: per-row update (differing values) on the matched
|
||||
// id, so the re-fetched daily summary overwrites in place and bumps
|
||||
// `syncVersion`. `deletedAt: null` rides along unconditionally — a no-op on
|
||||
// a live row, a deliberate RESURRECTION on a tombstoned one (Fitbit is the
|
||||
// source of truth for its own rows; see TOMBSTONES RESURRECT above).
|
||||
for (const { id, r } of toUpdate) {
|
||||
try {
|
||||
await prisma.measurement.update({
|
||||
@@ -531,6 +543,7 @@ export async function upsertFitbitMeasurements(
|
||||
unit: r.unit,
|
||||
measuredAt: r.measuredAt,
|
||||
sleepStage: r.sleepStage ?? null,
|
||||
deletedAt: null,
|
||||
// Surface the server-side mutation to the iOS LWW reconciler.
|
||||
syncVersion: { increment: 1 },
|
||||
},
|
||||
@@ -551,7 +564,15 @@ export async function upsertFitbitMeasurements(
|
||||
// path keeps the inline per-day hook here (small touched set, warm next read).
|
||||
if (opts.deferRollup) {
|
||||
const tracker = rollupDeferStorage.getStore();
|
||||
if (tracker) tracker.keys.push(...touched);
|
||||
// Plain loop, never `push(...touched)`: `touched` carries ONE entry PER
|
||||
// READING, and a full-history batch on a sample-dense account (a
|
||||
// multi-year heart-rate series is six figures of rows) blows the call
|
||||
// stack when spread into a single call — the engine caps argument counts.
|
||||
// The RangeError aborted the whole metrics collection on exactly the
|
||||
// accounts with the most data.
|
||||
if (tracker) {
|
||||
for (const t of touched) tracker.keys.push(t);
|
||||
}
|
||||
return { imported, touched };
|
||||
}
|
||||
|
||||
|
||||
@@ -72,10 +72,29 @@ type MedicationScheduleRecord = {
|
||||
daysOfWeek: string | null;
|
||||
};
|
||||
|
||||
type MeasurementRecord = {
|
||||
/**
|
||||
* v1.28.25 — one SQL-aggregated vitals bucket: every non-deleted, non-import
|
||||
* row of `type` inside the Berlin-local hour `hour` of day `dayKey`, folded
|
||||
* to a row count + value sum by the database. The vitals window is
|
||||
* rollout-date → now and grows forever, and PULSE can be per-sample heart
|
||||
* rate (six figures on a long-tenured wearable account) — reading raw rows
|
||||
* here was the scale hazard. Every consumer's semantics survive the fold:
|
||||
* green-day classification needs the per-day per-type MEAN (= Σsum / Σcount
|
||||
* across the day's hour buckets — identical to the raw mean up to
|
||||
* floating-point summation order), the count badges need row counts, the
|
||||
* engagement / consistency series need day presence, and the hidden
|
||||
* night-owl / early-bird / leap-day tallies need per-hour row counts.
|
||||
*/
|
||||
type VitalsHourBucket = {
|
||||
type: "WEIGHT" | "BLOOD_PRESSURE_SYS" | "BLOOD_PRESSURE_DIA" | "PULSE";
|
||||
value: number;
|
||||
measuredAt: Date;
|
||||
/** Berlin-local YYYY-MM-DD. */
|
||||
dayKey: string;
|
||||
/** Berlin-local hour 0–23. */
|
||||
hour: number;
|
||||
/** Number of raw rows folded into this bucket. */
|
||||
rowCount: number;
|
||||
/** Sum of the raw rows' values (feeds the per-day mean). */
|
||||
valueSum: number;
|
||||
};
|
||||
|
||||
function maxDate(a: Date, b: Date): Date {
|
||||
@@ -123,11 +142,6 @@ function dayKeyToDate(dayKey: string): Date {
|
||||
return new Date(Date.UTC(year, month - 1, day, 12, 0, 0));
|
||||
}
|
||||
|
||||
function mean(values: number[]): number {
|
||||
if (values.length === 0) return 0;
|
||||
return values.reduce((total, value) => total + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function findCountCompletionDate(dates: Date[], target: number): Date | null {
|
||||
if (dates.length < target) return null;
|
||||
const sorted = [...dates].sort((a, b) => a.getTime() - b.getTime());
|
||||
@@ -195,66 +209,64 @@ function getEventDaySeries(dates: Date[]): {
|
||||
};
|
||||
}
|
||||
|
||||
// v1.28.25 — consumes the SQL hour buckets instead of raw rows. The per-day
|
||||
// per-type mean is Σ(valueSum) / Σ(rowCount) over the day's buckets — the
|
||||
// same mean the raw-row version computed (up to floating-point summation
|
||||
// order), feeding the identical classify → green-day → streak pipeline.
|
||||
function getHealthGreenDaySeries(
|
||||
measurements: MeasurementRecord[],
|
||||
buckets: VitalsHourBucket[],
|
||||
heightCm: number | null,
|
||||
// v1.18.11 (W5 perf) — Berlin day-keys precomputed once per vitals row by
|
||||
// the caller and passed in parallel to `measurements`. Falls back to a
|
||||
// per-row `toBerlinDayKey` when absent so the standalone callers / tests
|
||||
// keep working; the result is byte-identical either way.
|
||||
dayKeys?: string[],
|
||||
) {
|
||||
type Acc = { sum: number; count: number };
|
||||
const dayStats = new Map<
|
||||
string,
|
||||
{
|
||||
weight: number[];
|
||||
bpSys: number[];
|
||||
bpDia: number[];
|
||||
pulse: number[];
|
||||
}
|
||||
{ weight: Acc; bpSys: Acc; bpDia: Acc; pulse: Acc }
|
||||
>();
|
||||
|
||||
for (let i = 0; i < measurements.length; i++) {
|
||||
const measurement = measurements[i];
|
||||
const dayKey = dayKeys?.[i] ?? toBerlinDayKey(measurement.measuredAt);
|
||||
const bucket = dayStats.get(dayKey) ?? {
|
||||
weight: [],
|
||||
bpSys: [],
|
||||
bpDia: [],
|
||||
pulse: [],
|
||||
for (const b of buckets) {
|
||||
const slot = dayStats.get(b.dayKey) ?? {
|
||||
weight: { sum: 0, count: 0 },
|
||||
bpSys: { sum: 0, count: 0 },
|
||||
bpDia: { sum: 0, count: 0 },
|
||||
pulse: { sum: 0, count: 0 },
|
||||
};
|
||||
|
||||
if (measurement.type === "WEIGHT") bucket.weight.push(measurement.value);
|
||||
if (measurement.type === "BLOOD_PRESSURE_SYS")
|
||||
bucket.bpSys.push(measurement.value);
|
||||
if (measurement.type === "BLOOD_PRESSURE_DIA")
|
||||
bucket.bpDia.push(measurement.value);
|
||||
if (measurement.type === "PULSE") bucket.pulse.push(measurement.value);
|
||||
|
||||
dayStats.set(dayKey, bucket);
|
||||
const acc =
|
||||
b.type === "WEIGHT"
|
||||
? slot.weight
|
||||
: b.type === "BLOOD_PRESSURE_SYS"
|
||||
? slot.bpSys
|
||||
: b.type === "BLOOD_PRESSURE_DIA"
|
||||
? slot.bpDia
|
||||
: slot.pulse;
|
||||
acc.sum += b.valueSum;
|
||||
acc.count += b.rowCount;
|
||||
dayStats.set(b.dayKey, slot);
|
||||
}
|
||||
|
||||
const bmiGreenDays: string[] = [];
|
||||
const bpGreenDays: string[] = [];
|
||||
const pulseGreenDays: string[] = [];
|
||||
|
||||
for (const [dayKey, bucket] of dayStats.entries()) {
|
||||
if (heightCm && bucket.weight.length > 0) {
|
||||
const bmi = mean(bucket.weight) / (heightCm / 100) ** 2;
|
||||
for (const [dayKey, slot] of dayStats.entries()) {
|
||||
if (heightCm && slot.weight.count > 0) {
|
||||
const bmi = slot.weight.sum / slot.weight.count / (heightCm / 100) ** 2;
|
||||
if (classifyBMI(bmi).severity === "normal") {
|
||||
bmiGreenDays.push(dayKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (bucket.bpSys.length > 0 && bucket.bpDia.length > 0) {
|
||||
const bpClass = classifyBP(mean(bucket.bpSys), mean(bucket.bpDia));
|
||||
if (slot.bpSys.count > 0 && slot.bpDia.count > 0) {
|
||||
const bpClass = classifyBP(
|
||||
slot.bpSys.sum / slot.bpSys.count,
|
||||
slot.bpDia.sum / slot.bpDia.count,
|
||||
);
|
||||
if (bpClass.severity === "normal") {
|
||||
bpGreenDays.push(dayKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (bucket.pulse.length > 0) {
|
||||
const pulseClass = classifyPulse(mean(bucket.pulse));
|
||||
if (slot.pulse.count > 0) {
|
||||
const pulseClass = classifyPulse(slot.pulse.sum / slot.pulse.count);
|
||||
if (pulseClass.severity === "normal") {
|
||||
pulseGreenDays.push(dayKey);
|
||||
}
|
||||
@@ -528,7 +540,7 @@ export async function buildAchievementsResult(
|
||||
const startDate = maxDate(user.createdAt, GAMIFICATION_ROLLOUT_AT);
|
||||
|
||||
const [
|
||||
measurements,
|
||||
vitalsBuckets,
|
||||
intakeEvents,
|
||||
medications,
|
||||
passkeys,
|
||||
@@ -538,25 +550,42 @@ export async function buildAchievementsResult(
|
||||
healthProfile,
|
||||
illnessEpisodes,
|
||||
] = await Promise.all([
|
||||
prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
measuredAt: { gte: startDate, lte: now },
|
||||
source: { not: "IMPORT" },
|
||||
type: {
|
||||
in: ["WEIGHT", "BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA", "PULSE"],
|
||||
},
|
||||
// v1.4.41 W-DELETED-2 — soft-deleted measurements are excluded
|
||||
// from achievement progress so deleting a row immediately rolls
|
||||
// back any streak / count badge it earned.
|
||||
deletedAt: null,
|
||||
},
|
||||
select: {
|
||||
type: true,
|
||||
value: true,
|
||||
measuredAt: true,
|
||||
},
|
||||
}),
|
||||
// v1.28.25 — vitals folded to (type, Berlin day, Berlin hour) buckets in
|
||||
// SQL instead of one row per sample. The window (rollout → now) grows
|
||||
// forever and PULSE can be per-sample heart rate, so the raw read ran to
|
||||
// six figures on long-tenured wearable accounts; the bucket set is
|
||||
// bounded by 4 types × 24 hours × window days and every downstream
|
||||
// metric derives exactly from (count, sum) per bucket — see the
|
||||
// `VitalsHourBucket` doc for the semantics argument. Filters mirror the
|
||||
// previous findMany exactly: the type allowlist is a closed literal set
|
||||
// (never user input), source ≠ IMPORT, and soft-deleted rows stay
|
||||
// excluded (v1.4.41 W-DELETED-2) so deleting a row immediately rolls
|
||||
// back any streak / count badge it earned. `AT TIME ZONE 'UTC' AT TIME
|
||||
// ZONE 'Europe/Berlin'` re-tags the naive UTC `measured_at` and converts
|
||||
// it to the same Berlin wall clock `toBerlinDayKey` / `berlinHour`
|
||||
// derive via Intl. Ordering pins day → hour → type for determinism.
|
||||
prisma.$queryRaw<VitalsHourBucket[]>`
|
||||
SELECT
|
||||
m."type"::text AS "type",
|
||||
to_char(
|
||||
(m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE 'Europe/Berlin',
|
||||
'YYYY-MM-DD'
|
||||
) AS "dayKey",
|
||||
EXTRACT(
|
||||
HOUR FROM (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE 'Europe/Berlin'
|
||||
)::int AS "hour",
|
||||
COUNT(*)::int AS "rowCount",
|
||||
SUM(m."value")::double precision AS "valueSum"
|
||||
FROM "measurements" m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."measured_at" >= ${startDate}
|
||||
AND m."measured_at" <= ${now}
|
||||
AND m."source"::text <> 'IMPORT'
|
||||
AND m."type"::text IN ('WEIGHT', 'BLOOD_PRESSURE_SYS', 'BLOOD_PRESSURE_DIA', 'PULSE')
|
||||
AND m."deleted_at" IS NULL
|
||||
GROUP BY 1, 2, 3
|
||||
ORDER BY 2, 3, 1
|
||||
`,
|
||||
prisma.medicationIntakeEvent.findMany({
|
||||
where: {
|
||||
userId,
|
||||
@@ -711,21 +740,25 @@ export async function buildAchievementsResult(
|
||||
...passwordLoginDates,
|
||||
]);
|
||||
|
||||
// v1.18.11 (W5 perf) — the vitals array runs through several independent
|
||||
// full-array passes, each previously re-deriving a Berlin day-key per row
|
||||
// via `Intl.DateTimeFormat.formatToParts` (~16.8k rows × ~3 passes on a
|
||||
// power-user account dominated the cold build). Compute each row's day-key
|
||||
// ONCE here and thread the parallel array into every consumer; the values
|
||||
// are identical to the per-pass derivation, so the output is byte-identical.
|
||||
const vitalsDayKeys = (measurements as MeasurementRecord[]).map((m) =>
|
||||
toBerlinDayKey(m.measuredAt),
|
||||
);
|
||||
// v1.28.25 — the buckets already carry the Berlin day-key + hour straight
|
||||
// from SQL (superseding the v1.18.11 Intl-once optimisation: no Intl pass
|
||||
// over the vitals remains at all). The parallel arrays thread into every
|
||||
// consumer exactly as the per-row keys did; day-presence consumers dedup
|
||||
// by key, so bucket-level duplicates (several hours / types on one day)
|
||||
// are harmless.
|
||||
const vitalsDayKeys = vitalsBuckets.map((b) => b.dayKey);
|
||||
const vitalsHours = vitalsBuckets.map((b) => b.hour);
|
||||
// Weighted records for the expansion metrics: one record per bucket,
|
||||
// `count` carrying the folded row multiplicity. The `measuredAt` anchor is
|
||||
// never read on this path (day-keys + hours ride the parallel arrays
|
||||
// above) — it only satisfies the record shape.
|
||||
const vitalsRecords = vitalsBuckets.map((b) => ({
|
||||
type: b.type,
|
||||
measuredAt: dayKeyToDate(b.dayKey),
|
||||
count: b.rowCount,
|
||||
}));
|
||||
|
||||
const healthSeries = getHealthGreenDaySeries(
|
||||
measurements as MeasurementRecord[],
|
||||
user.heightCm,
|
||||
vitalsDayKeys,
|
||||
);
|
||||
const healthSeries = getHealthGreenDaySeries(vitalsBuckets, user.heightCm);
|
||||
const onTimeSeries = getOnTimePerfectDaySeries(
|
||||
intakeEvents,
|
||||
schedulesByMedicationId,
|
||||
@@ -746,8 +779,8 @@ export async function buildAchievementsResult(
|
||||
sleepMeasurements.map((m) => m.measuredAt),
|
||||
);
|
||||
const weeklyConsistency = getWeeklyConsistency(
|
||||
// v1.18.11 (W5 perf) — reuse the day-keys computed once above instead of
|
||||
// a fresh per-row `toBerlinDayKey` pass over the full vitals array.
|
||||
// Bucket day-keys — `getWeeklyConsistency` dedups to distinct days, so
|
||||
// the per-hour multiplicity is irrelevant.
|
||||
vitalsDayKeys,
|
||||
MEASUREMENT_CONSISTENCY_MIN_DAYS_PER_WEEK,
|
||||
MEASUREMENT_CONSISTENCY_TARGET_WEEKS,
|
||||
@@ -762,14 +795,15 @@ export async function buildAchievementsResult(
|
||||
: 0;
|
||||
|
||||
const expansionValues = buildExpansionMetricValues({
|
||||
measurements,
|
||||
measurements: vitalsRecords,
|
||||
moodEntries,
|
||||
intakeEvents,
|
||||
auditEvents,
|
||||
// v1.18.11 (W5 perf) — reuse the vitals day-keys computed once above so
|
||||
// the engagement + hidden passes don't re-walk the full vitals array
|
||||
// through `Intl.DateTimeFormat` a second and third time.
|
||||
// The SQL-derived Berlin day-keys + hours ride in parallel to the
|
||||
// weighted bucket records — the expansion passes never touch a bucket's
|
||||
// representative `measuredAt`.
|
||||
measurementDayKeys: vitalsDayKeys,
|
||||
measurementHours: vitalsHours,
|
||||
});
|
||||
const earnability = getEarnabilityFlags({
|
||||
hasMedication: medications.length > 0,
|
||||
|
||||
@@ -47,6 +47,17 @@ export interface MoodEntryRecord {
|
||||
export interface MeasurementRecord {
|
||||
type: string;
|
||||
measuredAt: Date;
|
||||
/**
|
||||
* v1.28.25 — row multiplicity. The achievements builder now reads vitals
|
||||
* as SQL (day, hour, type) buckets instead of one row per sample (a
|
||||
* per-sample PULSE history runs to six figures), so one record here can
|
||||
* stand for N underlying rows. Every count-semantics consumer
|
||||
* (`countMeasurementsByType`, the hidden-metrics tallies) weighs the
|
||||
* record by `count`; day-presence consumers dedup by day key and are
|
||||
* multiplicity-blind. Absent means 1 — raw per-sample callers and the
|
||||
* existing tests are unchanged.
|
||||
*/
|
||||
count?: number;
|
||||
}
|
||||
|
||||
export interface IntakeEventRecord {
|
||||
@@ -79,9 +90,10 @@ export function countMeasurementsByType(
|
||||
let bp = 0;
|
||||
let pulse = 0;
|
||||
for (const m of measurements) {
|
||||
if (m.type === "WEIGHT") weight += 1;
|
||||
else if (m.type === "BLOOD_PRESSURE_SYS") bp += 1;
|
||||
else if (m.type === "PULSE") pulse += 1;
|
||||
const n = m.count ?? 1;
|
||||
if (m.type === "WEIGHT") weight += n;
|
||||
else if (m.type === "BLOOD_PRESSURE_SYS") bp += n;
|
||||
else if (m.type === "PULSE") pulse += n;
|
||||
}
|
||||
return { weightCount: weight, bpCount: bp, pulseCount: pulse };
|
||||
}
|
||||
@@ -289,18 +301,22 @@ export function getHiddenMetrics(input: {
|
||||
let earlyBird = 0;
|
||||
let leapDay = 0;
|
||||
|
||||
const tally = (hour: number, dayKey: string): void => {
|
||||
if (hour >= 2 && hour < 4) nightOwl += 1;
|
||||
if (hour >= 4 && hour < 6) earlyBird += 1;
|
||||
// v1.28.25 — `n` is the row multiplicity (`MeasurementRecord.count`): a
|
||||
// bucketed vitals record stands for N raw rows, and these are COUNT
|
||||
// metrics, so each contributes N. Mood / intake tallies stay per-row (n=1).
|
||||
const tally = (hour: number, dayKey: string, n = 1): void => {
|
||||
if (hour >= 2 && hour < 4) nightOwl += n;
|
||||
if (hour >= 4 && hour < 6) earlyBird += n;
|
||||
// Feb 29 — only valid in leap years.
|
||||
if (dayKey.endsWith("-02-29")) leapDay += 1;
|
||||
if (dayKey.endsWith("-02-29")) leapDay += n;
|
||||
};
|
||||
|
||||
for (let i = 0; i < input.measurements.length; i++) {
|
||||
const ts = input.measurements[i].measuredAt;
|
||||
const m = input.measurements[i];
|
||||
tally(
|
||||
input.measurementHours?.[i] ?? berlinHour(ts),
|
||||
input.measurementDayKeys?.[i] ?? toBerlinDayKey(ts),
|
||||
input.measurementHours?.[i] ?? berlinHour(m.measuredAt),
|
||||
input.measurementDayKeys?.[i] ?? toBerlinDayKey(m.measuredAt),
|
||||
m.count ?? 1,
|
||||
);
|
||||
}
|
||||
for (const e of input.moodEntries) {
|
||||
@@ -387,15 +403,26 @@ export function buildExpansionMetricValues(input: {
|
||||
* hidden passes so the full vitals array is `Intl`-walked once, not ~3×.
|
||||
*/
|
||||
measurementDayKeys?: string[];
|
||||
/**
|
||||
* v1.28.25 — Berlin hours for `measurements`, precomputed by the caller.
|
||||
* The bucketed-vitals path (SQL `(day, hour, type)` buckets) carries the
|
||||
* Berlin hour straight from the query; deriving it here from a bucket's
|
||||
* representative `measuredAt` would be wrong, so the caller MUST supply
|
||||
* this alongside bucketed records. Raw per-row callers omit it and the
|
||||
* hours derive from `measuredAt` exactly as before.
|
||||
*/
|
||||
measurementHours?: number[];
|
||||
}): ExpansionMetrics {
|
||||
const counts = countMeasurementsByType(input.measurements);
|
||||
const mood = getMoodMetrics(input.moodEntries);
|
||||
// The hidden pass also needs the per-row Berlin hour; derive it once here
|
||||
// (only when the caller supplied day-keys, i.e. on the hot achievements
|
||||
// path) so a single hour pass replaces the per-pass re-derivation.
|
||||
const measurementHours = input.measurementDayKeys
|
||||
? input.measurements.map((m) => berlinHour(m.measuredAt))
|
||||
: undefined;
|
||||
const measurementHours =
|
||||
input.measurementHours ??
|
||||
(input.measurementDayKeys
|
||||
? input.measurements.map((m) => berlinHour(m.measuredAt))
|
||||
: undefined);
|
||||
const engagement = getEngagementMetrics({
|
||||
measurements: input.measurements,
|
||||
moodEntries: input.moodEntries,
|
||||
|
||||
@@ -4,19 +4,51 @@
|
||||
* failure and RETURNS — it must not rethrow, because every call site sits in a
|
||||
* per-collection catch and a rethrow would abort the sibling collections
|
||||
* (live: steps 400ing suppressed distance / floors / active-energy entirely).
|
||||
*
|
||||
* Also pins the dead-token verdict: `getValidToken`'s failure paths
|
||||
* (credentials missing, refresh failure) must register on the cycle's
|
||||
* hard-fail ledger, and a cycle whose token is dead must NOT stamp
|
||||
* `markSynced` / `recordSyncSuccess` — otherwise every resource returns 0
|
||||
* without failures, the cycle reads as clean, `recordSyncSuccess` un-parks
|
||||
* `error_reauth`, and the hourly cohort hammers the dead refresh token forever
|
||||
* while `lastSyncedAt` advances past real data.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { recordSyncFailureMock } = vi.hoisted(() => ({
|
||||
const {
|
||||
recordSyncFailureMock,
|
||||
recordSyncSuccessMock,
|
||||
prismaMock,
|
||||
getUserGoogleHealthCredentialsMock,
|
||||
refreshAccessTokenMock,
|
||||
resourceFake,
|
||||
} = vi.hoisted(() => ({
|
||||
recordSyncFailureMock: vi.fn(async () => {}),
|
||||
recordSyncSuccessMock: vi.fn(async () => {}),
|
||||
prismaMock: {
|
||||
googleHealthConnection: {
|
||||
findUnique: vi.fn(),
|
||||
update: vi.fn(async () => ({})),
|
||||
},
|
||||
},
|
||||
getUserGoogleHealthCredentialsMock: vi.fn(async (): Promise<unknown> => null),
|
||||
refreshAccessTokenMock: vi.fn(),
|
||||
// The dead-token cycle test drives the real `syncUserGoogleHealth`; each
|
||||
// resource module is mocked to do exactly what the real ones do first —
|
||||
// resolve the token — so the ledger registration happens inside the cycle's
|
||||
// ALS scope.
|
||||
resourceFake: async (userId: string): Promise<number> => {
|
||||
const { getValidToken } = await import("../sync");
|
||||
return (await getValidToken(userId)) ? 1 : 0;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({ prisma: {} }));
|
||||
vi.mock("@/lib/db", () => ({ prisma: prismaMock }));
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn(async () => false),
|
||||
recordSyncFailure: recordSyncFailureMock,
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
recordSyncSuccess: recordSyncSuccessMock,
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
@@ -27,15 +59,44 @@ vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserGoogleHealthCredentials: vi.fn(async () => null),
|
||||
getUserGoogleHealthCredentials: getUserGoogleHealthCredentialsMock,
|
||||
}));
|
||||
vi.mock("../client", () => ({ refreshAccessToken: vi.fn() }));
|
||||
vi.mock("../client", () => ({ refreshAccessToken: refreshAccessTokenMock }));
|
||||
|
||||
import { handleCollectionFetchError } from "../sync";
|
||||
vi.mock("../sync-metrics", () => ({ syncUserMetrics: resourceFake }));
|
||||
vi.mock("../sync-activity", () => ({ syncUserActivity: resourceFake }));
|
||||
vi.mock("../sync-sleep", () => ({ syncUserSleep: resourceFake }));
|
||||
vi.mock("../sync-workout", () => ({ syncUserWorkout: resourceFake }));
|
||||
|
||||
import {
|
||||
GOOGLE_HEALTH_TOKEN_HARD_FAIL,
|
||||
getValidToken,
|
||||
handleCollectionFetchError,
|
||||
runWithGoogleHealthHardFailLedger,
|
||||
syncUserGoogleHealth,
|
||||
} from "../sync";
|
||||
import { GoogleHealthApiError } from "../response-classifier";
|
||||
|
||||
/** A connection whose access token is inside the 5-min refresh buffer. */
|
||||
const EXPIRED_CONNECTION = {
|
||||
id: "conn-1",
|
||||
userId: "user-1",
|
||||
googleUserId: "g-user-1",
|
||||
accessToken: "enc-access",
|
||||
refreshToken: "enc-refresh",
|
||||
tokenExpiresAt: new Date(Date.now() - 60_000),
|
||||
lastSyncedAt: new Date("2026-07-01T00:00:00.000Z"),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
recordSyncFailureMock.mockClear();
|
||||
recordSyncSuccessMock.mockClear();
|
||||
prismaMock.googleHealthConnection.findUnique.mockReset();
|
||||
prismaMock.googleHealthConnection.update
|
||||
.mockReset()
|
||||
.mockResolvedValue({} as never);
|
||||
getUserGoogleHealthCredentialsMock.mockReset().mockResolvedValue(null);
|
||||
refreshAccessTokenMock.mockReset();
|
||||
});
|
||||
|
||||
describe("handleCollectionFetchError — fail-soft grouping", () => {
|
||||
@@ -89,3 +150,85 @@ describe("handleCollectionFetchError — fail-soft grouping", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getValidToken — dead-token cycle verdict", () => {
|
||||
it("registers the credentials-missing path on the hard-fail ledger", async () => {
|
||||
prismaMock.googleHealthConnection.findUnique.mockResolvedValue(
|
||||
EXPIRED_CONNECTION as never,
|
||||
);
|
||||
|
||||
const { result, failures } = await runWithGoogleHealthHardFailLedger(() =>
|
||||
getValidToken("user-1"),
|
||||
);
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(failures).toEqual([GOOGLE_HEALTH_TOKEN_HARD_FAIL]);
|
||||
expect(recordSyncFailureMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
userId: "user-1",
|
||||
integration: "google-health",
|
||||
kind: "reauth_required",
|
||||
errorCode: "credentials_missing",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("registers a refresh failure on the hard-fail ledger", async () => {
|
||||
prismaMock.googleHealthConnection.findUnique.mockResolvedValue(
|
||||
EXPIRED_CONNECTION as never,
|
||||
);
|
||||
getUserGoogleHealthCredentialsMock.mockResolvedValue({
|
||||
clientId: "id",
|
||||
clientSecret: "secret",
|
||||
});
|
||||
refreshAccessTokenMock.mockRejectedValue(
|
||||
new GoogleHealthApiError({
|
||||
verb: "refreshToken",
|
||||
classification: "reauth_required",
|
||||
httpStatus: 401,
|
||||
reason: "invalid_grant",
|
||||
}),
|
||||
);
|
||||
|
||||
const { result, failures } = await runWithGoogleHealthHardFailLedger(() =>
|
||||
getValidToken("user-1"),
|
||||
);
|
||||
|
||||
expect(result).toBeNull();
|
||||
expect(failures).toEqual([GOOGLE_HEALTH_TOKEN_HARD_FAIL]);
|
||||
expect(recordSyncFailureMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ kind: "reauth_required" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("a dead-token cycle fails the verdict and never stamps markSynced / recordSyncSuccess", async () => {
|
||||
// Every resource resolves the token, the refresh fails each time — before
|
||||
// the ledger registration the cycle read as CLEAN (all resources returned
|
||||
// 0 without failures), stamped the watermark, and un-parked error_reauth.
|
||||
prismaMock.googleHealthConnection.findUnique.mockResolvedValue(
|
||||
EXPIRED_CONNECTION as never,
|
||||
);
|
||||
getUserGoogleHealthCredentialsMock.mockResolvedValue({
|
||||
clientId: "id",
|
||||
clientSecret: "secret",
|
||||
});
|
||||
refreshAccessTokenMock.mockRejectedValue(
|
||||
new GoogleHealthApiError({
|
||||
verb: "refreshToken",
|
||||
classification: "reauth_required",
|
||||
httpStatus: 401,
|
||||
reason: "invalid_grant",
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await syncUserGoogleHealth("user-1");
|
||||
|
||||
expect(res).toEqual({ imported: 0, failed: true });
|
||||
expect(recordSyncSuccessMock).not.toHaveBeenCalled();
|
||||
// No connection write may carry the watermark stamp.
|
||||
for (const call of prismaMock.googleHealthConnection.update.mock.calls) {
|
||||
const arg = (call as unknown[])[0] as { data: Record<string, unknown> };
|
||||
expect(arg.data).not.toHaveProperty("lastSyncedAt");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -118,3 +118,85 @@ describe("upsertGoogleHealthMeasurements — tombstones resurrect", () => {
|
||||
expect(imported).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("upsertGoogleHealthMeasurements — no-op overwrite skip", () => {
|
||||
it("skips the update entirely for an identical LIVE row (no syncVersion churn)", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "ACTIVITY_STEPS",
|
||||
externalId: "stats:steps:2026-07-08",
|
||||
value: 8123,
|
||||
unit: "steps",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
sleepStage: null,
|
||||
deletedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const { imported } = await upsertGoogleHealthMeasurements(
|
||||
"user-1",
|
||||
[STEPS_READING],
|
||||
{ deferRollup: true },
|
||||
);
|
||||
|
||||
// The 24 h overlap re-fetches this row hourly; an unchanged live row must
|
||||
// not be rewritten (the unconditional syncVersion bump churned the DB and
|
||||
// every iOS delta pull).
|
||||
expect(updateMock).not.toHaveBeenCalled();
|
||||
expect(createManyMock).not.toHaveBeenCalled();
|
||||
expect(imported).toBe(0);
|
||||
});
|
||||
|
||||
it("still writes when any payload field differs on a live row", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "ACTIVITY_STEPS",
|
||||
externalId: "stats:steps:2026-07-08",
|
||||
value: 7999, // stale daily total — Google re-rolled the day upward
|
||||
unit: "steps",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
sleepStage: null,
|
||||
deletedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const { imported } = await upsertGoogleHealthMeasurements(
|
||||
"user-1",
|
||||
[STEPS_READING],
|
||||
{ deferRollup: true },
|
||||
);
|
||||
|
||||
expect(updateMock).toHaveBeenCalledTimes(1);
|
||||
expect(imported).toBe(1);
|
||||
});
|
||||
|
||||
it("an identical TOMBSTONED row ALWAYS updates — the write is the resurrection", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "ACTIVITY_STEPS",
|
||||
externalId: "stats:steps:2026-07-08",
|
||||
value: 8123,
|
||||
unit: "steps",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
sleepStage: null,
|
||||
deletedAt: new Date("2026-07-09T00:00:00.000Z"),
|
||||
},
|
||||
]);
|
||||
|
||||
const { imported } = await upsertGoogleHealthMeasurements(
|
||||
"user-1",
|
||||
[STEPS_READING],
|
||||
{ deferRollup: true },
|
||||
);
|
||||
|
||||
expect(updateMock).toHaveBeenCalledTimes(1);
|
||||
const upd = (updateMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(upd.data.deletedAt).toBeNull();
|
||||
expect(imported).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -500,7 +500,8 @@ export const GOOGLE_HEALTH_DATA_TYPES = {
|
||||
timeField: "date",
|
||||
},
|
||||
// `respiratory-rate` does not exist in the catalogue; the daily summary is
|
||||
// `daily-respiratory-rate` (`dailyRespiratoryRateBpm`).
|
||||
// `daily-respiratory-rate`, value leaf `dailyRespiratoryRate.breathsPerMinute`
|
||||
// (the once-assumed `dailyRespiratoryRateBpm` leaf never existed).
|
||||
respiratoryRate: {
|
||||
path: "daily-respiratory-rate",
|
||||
filter: "daily_respiratory_rate",
|
||||
|
||||
@@ -172,9 +172,19 @@ RFC-3339), `sleep.type` (`CLASSIC | STAGES`), `sleep.stages[]`
|
||||
`sleep.summary` (`minutesAsleep`/`minutesAwake`, int64 strings). HealthLog
|
||||
stores one `SLEEP_DURATION` row per stage SEGMENT, `measuredAt = that segment's
|
||||
END instant`, harmonised onto the shared `SleepStage` enum the night-total +
|
||||
hypnogram readers consume. externalId = `<session-anchor>:sleep_<stage>:<i>`
|
||||
(session `interval.endTime` ISO instant + indexed segment), so a re-scored night
|
||||
overwrites in place. Stage map (`SLEEP_STAGE_TYPE` enum, lowercased):
|
||||
hypnogram readers consume. externalId = `<anchor>:sleep:<segment-start-ISO>`
|
||||
(v1.28.18): the anchor prefers the DataPoint's invariant resource `name`,
|
||||
falling back to the session `interval.endTime` (then `startTime`) parsed to a
|
||||
UTC ISO instant, and the per-segment key is the segment's own START instant —
|
||||
NOT a positional index and NOT the stage label, so a re-scored night overwrites
|
||||
in place and a mere re-classification (LIGHT→DEEP on the same block) UPDATES
|
||||
the row rather than orphaning it. After mapping, the sync runs a
|
||||
replace-by-window cleanup (`replaceStaleGoogleHealthSleep`): any LIVE
|
||||
`GOOGLE_HEALTH` `SLEEP_DURATION` row inside the session's
|
||||
`[windowStart, windowEnd]` that this fetch did NOT re-produce is soft-deleted,
|
||||
so segments an earlier scoring left behind (old volatile keys, dropped blocks)
|
||||
never double-count the night total. Stage map (`SLEEP_STAGE_TYPE` enum,
|
||||
lowercased):
|
||||
`LIGHT → CORE` ("light" ↔ Apple "core" shallow-NREM band), `DEEP → DEEP`,
|
||||
`REM → REM`, `AWAKE`/`RESTLESS → AWAKE`, classic `ASLEEP → ASLEEP`.
|
||||
`SLEEP_STAGE_TYPE_UNSPECIFIED` / unknown labels are skipped, not mis-bucketed.
|
||||
@@ -203,9 +213,19 @@ source ladder.
|
||||
`(userId, type, source: "GOOGLE_HEALTH", externalId)` unique. Spot/daily metric
|
||||
rows: `externalId = <anchor>:<fieldTag>`. Daily cumulative activity rows:
|
||||
`externalId = stats:<fieldTag>:<YYYY-MM-DD>` (Apple-Health overwrite shape).
|
||||
Workouts: `(userId, source: "GOOGLE_HEALTH", externalId)` on the `Workout` table.
|
||||
A re-fetch of the same window (daily summaries re-roll after the fact, so the
|
||||
incremental overlap is 24 h) overwrites in place.
|
||||
Sleep rows: `externalId = <anchor>:sleep:<segment-start-ISO>` (see the sleep
|
||||
bundle above). Workouts: `(userId, source: "GOOGLE_HEALTH", externalId)` on the
|
||||
`Workout` table. A re-fetch of the same window (daily summaries re-roll after
|
||||
the fact, so the incremental overlap is 24 h) overwrites in place.
|
||||
|
||||
The measurements unique index is **FULL** — it covers soft-deleted rows too, so
|
||||
exactly one row can ever exist per key, live or tombstoned. The upsert probe
|
||||
therefore matches tombstoned rows as well, and a re-import **RESURRECTS** them
|
||||
(the update branch clears `deletedAt`; v1.28.24) — Google is the source of
|
||||
truth for its own rows, and treating a tombstone as absent would plan an insert
|
||||
that `skipDuplicates` drops silently, wedging the key forever. An unchanged
|
||||
LIVE row is skipped entirely (no write, no `syncVersion` bump) so the 24 h
|
||||
overlap re-fetch doesn't churn the DB or the iOS delta pull.
|
||||
|
||||
## Error surfacing
|
||||
|
||||
|
||||
@@ -128,17 +128,27 @@ export async function syncUserActivity(
|
||||
continue;
|
||||
}
|
||||
|
||||
// The mapper runs INSIDE a per-type catch too (mirrors sync-metrics): a
|
||||
// single malformed point whose `resource.map(point)` throws must not
|
||||
// escape the ROLLUP_RESOURCES loop and abort every sibling type ordered
|
||||
// after it. Route a map throw through the same ledger as a fetch failure —
|
||||
// record it, fail the cycle, and move on to the next type.
|
||||
const readings: GoogleHealthMeasurementUpsert[] = [];
|
||||
for (const point of points) {
|
||||
for (const m of resource.map(point)) {
|
||||
readings.push({
|
||||
type: m.type,
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
measuredAt: m.measuredAt,
|
||||
externalId: externalIdFor(m),
|
||||
});
|
||||
try {
|
||||
for (const point of points) {
|
||||
for (const m of resource.map(point)) {
|
||||
readings.push({
|
||||
type: m.type,
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
measuredAt: m.measuredAt,
|
||||
externalId: externalIdFor(m),
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
imported += await handleCollectionFetchError(resource.verb, userId, err);
|
||||
continue;
|
||||
}
|
||||
if (points.length === 0) {
|
||||
rollupEmptyResponse.push(resource.verb);
|
||||
|
||||
@@ -29,13 +29,14 @@ import {
|
||||
import {
|
||||
getValidToken,
|
||||
handleCollectionFetchError,
|
||||
noteHardFailure,
|
||||
replaceStaleGoogleHealthSleep,
|
||||
upsertGoogleHealthMeasurements,
|
||||
type GoogleHealthMeasurementUpsert,
|
||||
type GoogleHealthResourceSyncOptions,
|
||||
type GoogleHealthSleepReplaceWindow,
|
||||
} from "./sync";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
|
||||
export async function syncUserSleep(
|
||||
@@ -69,7 +70,20 @@ export async function syncUserSleep(
|
||||
const readings: GoogleHealthMeasurementUpsert[] = [];
|
||||
const replaceWindows: GoogleHealthSleepReplaceWindow[] = [];
|
||||
for (const point of points) {
|
||||
const session = mapSleepSessionDetailed(point, tz);
|
||||
// Per-session throw-guard (mapper parity with sync-metrics): a single
|
||||
// malformed session must not abort the sibling sessions in the same page.
|
||||
// The ledger entry still fails the cycle so the watermark holds and the
|
||||
// bad point is re-fetched rather than silently lost past the overlap.
|
||||
let session: ReturnType<typeof mapSleepSessionDetailed>;
|
||||
try {
|
||||
session = mapSleepSessionDetailed(point, tz);
|
||||
} catch (err) {
|
||||
getEvent()?.addWarning(
|
||||
`google-health: sleep session map failed for ${userId}: ${err}`,
|
||||
);
|
||||
noteHardFailure("mapSleepSession");
|
||||
continue;
|
||||
}
|
||||
if (session.rows.length === 0) continue;
|
||||
for (const m of session.rows) {
|
||||
readings.push({
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import {
|
||||
getValidToken,
|
||||
handleCollectionFetchError,
|
||||
noteHardFailure,
|
||||
type GoogleHealthResourceSyncOptions,
|
||||
} from "./sync";
|
||||
import { prisma } from "@/lib/db";
|
||||
@@ -102,6 +103,10 @@ export async function syncUserWorkout(
|
||||
imported++;
|
||||
} catch (err) {
|
||||
getEvent()?.addWarning(`google-health: failed to upsert workout: ${err}`);
|
||||
// A fetched-but-unwritten session is gone once it leaves the overlap
|
||||
// window — fail the cycle's verdict so the watermark holds and the next
|
||||
// tick re-fetches it. No rethrow: sibling sessions keep upserting.
|
||||
noteHardFailure("workout:upsert");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+120
-13
@@ -103,6 +103,12 @@ export async function getValidToken(
|
||||
message: "Google Health credentials missing — token refresh skipped",
|
||||
errorCode: "credentials_missing",
|
||||
});
|
||||
// A dead token means every resource "succeeds" with 0 rows — without a
|
||||
// ledger entry the cycle would read as clean, stamp `markSynced`, and
|
||||
// `recordSyncSuccess` would flip a parked connection back to connected
|
||||
// (an hourly invalid-refresh hammer with an advancing watermark). Fail
|
||||
// the cycle's verdict so the park holds.
|
||||
noteHardFailure(GOOGLE_HEALTH_TOKEN_HARD_FAIL);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -154,6 +160,11 @@ export async function getValidToken(
|
||||
);
|
||||
}
|
||||
await recordGoogleHealthSyncFailure(userId, err);
|
||||
// Same verdict rule as the credentials-missing branch above: a refresh
|
||||
// failure must fail the cycle, or the all-zeros run stamps success,
|
||||
// un-parks `error_reauth`, and the hourly cohort retries the dead
|
||||
// refresh token forever while `lastSyncedAt` advances past real data.
|
||||
noteHardFailure(GOOGLE_HEALTH_TOKEN_HARD_FAIL);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -204,6 +215,41 @@ interface HardFailTracker {
|
||||
}
|
||||
const hardFailStorage = new AsyncLocalStorage<HardFailTracker>();
|
||||
|
||||
/**
|
||||
* Ledger label for a dead-token failure (`getValidToken` returning null on a
|
||||
* missing-credentials or failed-refresh path). Exported so the one-shot jobs
|
||||
* that run a single resource inside their own ledger scope can tell "the token
|
||||
* is dead" (park, don't retry) apart from "a fetch/write hard-failed" (retry).
|
||||
*/
|
||||
export const GOOGLE_HEALTH_TOKEN_HARD_FAIL = "token";
|
||||
|
||||
/**
|
||||
* Record a hard failure on the ambient per-cycle ledger, if one is in scope.
|
||||
* No-op outside a ledger scope (e.g. a direct call from a REPL script). Used by
|
||||
* `getValidToken`'s dead-token paths, the measurement write catches, and the
|
||||
* workout upsert catch — anything that must fail the cycle's verdict without
|
||||
* aborting sibling work.
|
||||
*/
|
||||
export function noteHardFailure(label: string): void {
|
||||
const tracker = hardFailStorage.getStore();
|
||||
if (tracker) tracker.failures.push(label);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `fn` inside a fresh hard-fail ledger scope and surface the collected
|
||||
* failures. `syncUserGoogleHealth` owns its own scope; this export exists for
|
||||
* the one-shot jobs (sleep repair) that drive a single resource sync directly
|
||||
* and must still see a dead token or a hard fetch/write failure before
|
||||
* stamping their completion marker.
|
||||
*/
|
||||
export async function runWithGoogleHealthHardFailLedger<T>(
|
||||
fn: () => Promise<T>,
|
||||
): Promise<{ result: T; failures: string[] }> {
|
||||
const tracker: HardFailTracker = { failures: [] };
|
||||
const result = await hardFailStorage.run(tracker, fn);
|
||||
return { result, failures: tracker.failures };
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-cycle accumulator of the `(type, day)` keys every resource's writes
|
||||
* touched. Only populated on a `fullSync` backfill (when the inline per-day
|
||||
@@ -418,7 +464,15 @@ export async function upsertGoogleHealthMeasurements(
|
||||
// insert that `skipDuplicates` silently drops against the tombstone's key,
|
||||
// permanently wedging the key (see TOMBSTONES RESURRECT above).
|
||||
const externalIds = readings.map((r) => r.externalId);
|
||||
let rowByKey = new Map<string, { id: string }>();
|
||||
interface ProbedRow {
|
||||
id: string;
|
||||
value: number;
|
||||
unit: string;
|
||||
measuredAt: Date;
|
||||
sleepStage: GoogleHealthMeasurementUpsert["sleepStage"];
|
||||
deletedAt: Date | null;
|
||||
}
|
||||
let rowByKey = new Map<string, ProbedRow>();
|
||||
try {
|
||||
const existing = await prisma.measurement.findMany({
|
||||
where: {
|
||||
@@ -426,12 +480,35 @@ export async function upsertGoogleHealthMeasurements(
|
||||
source: "GOOGLE_HEALTH",
|
||||
externalId: { in: externalIds },
|
||||
},
|
||||
select: { id: true, type: true, externalId: true },
|
||||
// The payload fields ride along so the update branch can skip a no-op
|
||||
// overwrite: the 24 h overlap re-fetches every recent row hourly, and an
|
||||
// unconditional write would bump `syncVersion` (iOS delta churn) on rows
|
||||
// whose values did not change.
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
externalId: true,
|
||||
value: true,
|
||||
unit: true,
|
||||
measuredAt: true,
|
||||
sleepStage: true,
|
||||
deletedAt: true,
|
||||
},
|
||||
});
|
||||
rowByKey = new Map(
|
||||
existing
|
||||
.filter((e) => e.externalId !== null)
|
||||
.map((e) => [`${e.type} ${e.externalId}`, { id: e.id }]),
|
||||
.map((e) => [
|
||||
`${e.type} ${e.externalId}`,
|
||||
{
|
||||
id: e.id,
|
||||
value: e.value,
|
||||
unit: e.unit,
|
||||
measuredAt: e.measuredAt,
|
||||
sleepStage: e.sleepStage,
|
||||
deletedAt: e.deletedAt,
|
||||
},
|
||||
]),
|
||||
);
|
||||
} catch (err) {
|
||||
// A probe failure must not strand the whole batch; fall back to treating
|
||||
@@ -457,9 +534,20 @@ export async function upsertGoogleHealthMeasurements(
|
||||
for (const r of readings) {
|
||||
const type = r.type as MeasurementType;
|
||||
const key = `${type} ${r.externalId}`;
|
||||
const live = rowByKey.get(key);
|
||||
if (live) {
|
||||
toUpdate.push({ id: live.id, r });
|
||||
const existingRow = rowByKey.get(key);
|
||||
if (existingRow) {
|
||||
// Skip the no-op overwrite: a LIVE row whose payload matches the
|
||||
// re-fetched reading exactly gains nothing from a write, and the
|
||||
// unconditional `syncVersion` bump churned the DB + every iOS delta pull
|
||||
// hourly for the whole 24 h overlap window. A TOMBSTONED row always
|
||||
// updates — the write is what resurrects it (`deletedAt: null`).
|
||||
const unchanged =
|
||||
existingRow.deletedAt === null &&
|
||||
existingRow.value === r.value &&
|
||||
existingRow.unit === r.unit &&
|
||||
existingRow.measuredAt.getTime() === r.measuredAt.getTime() &&
|
||||
(existingRow.sleepStage ?? null) === (r.sleepStage ?? null);
|
||||
if (!unchanged) toUpdate.push({ id: existingRow.id, r });
|
||||
} else if (!plannedCreateKeys.has(key)) {
|
||||
plannedCreateKeys.add(key);
|
||||
toCreate.push({
|
||||
@@ -518,6 +606,11 @@ export async function upsertGoogleHealthMeasurements(
|
||||
getEvent()?.addWarning(
|
||||
`google-health: failed to create measurements: ${err}`,
|
||||
);
|
||||
// Fetched-but-unwritten rows are lost for good once they age out of the
|
||||
// 24 h overlap — fail the cycle's verdict so the watermark holds and the
|
||||
// next tick re-fetches them. No rethrow: sibling chunks and resources
|
||||
// must keep writing.
|
||||
noteHardFailure("measurements:create");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -549,6 +642,9 @@ export async function upsertGoogleHealthMeasurements(
|
||||
getEvent()?.addWarning(
|
||||
`google-health: failed to update measurement: ${err}`,
|
||||
);
|
||||
// Same rule as the create catch: hold the watermark so the overwrite is
|
||||
// retried next tick rather than silently dropped.
|
||||
noteHardFailure("measurements:update");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -622,16 +718,24 @@ export async function markSynced(userId: string): Promise<void> {
|
||||
* read+stamp would let the first resource move `lastSyncedAt` to now() so later
|
||||
* resources only see the last overlap window — silently dropping the gap after
|
||||
* an outage longer than the overlap.
|
||||
*
|
||||
* VERDICT: returns `{ imported, failed }`. `failed` is exactly the condition
|
||||
* that suppresses `markSynced` — any hard failure (fetch, write, dead token)
|
||||
* or an all-soft-skipped no-op, plus the parked / missing-connection early
|
||||
* returns. The one-shot jobs (backfill) gate their completion markers on it;
|
||||
* stamping a marker after a partial run would freeze the failure in place.
|
||||
*/
|
||||
export async function syncUserGoogleHealth(
|
||||
userId: string,
|
||||
opts: { fullSync?: boolean } = {},
|
||||
): Promise<number> {
|
||||
): Promise<{ imported: number; failed: boolean }> {
|
||||
if (await isReauthRequired(userId, GOOGLE_HEALTH_INTEGRATION_KEY)) {
|
||||
getEvent()?.addWarning(
|
||||
`google-health sync skipped for ${userId}: parked at error_reauth`,
|
||||
);
|
||||
return 0;
|
||||
// Parked is NOT a clean run — a caller gating a completion marker on the
|
||||
// verdict must not stamp over a no-op.
|
||||
return { imported: 0, failed: true };
|
||||
}
|
||||
|
||||
// Snapshot the incremental watermark ONCE for the whole cycle. Every resource
|
||||
@@ -642,7 +746,7 @@ export async function syncUserGoogleHealth(
|
||||
where: { userId },
|
||||
select: { lastSyncedAt: true },
|
||||
});
|
||||
if (!connection) return 0;
|
||||
if (!connection) return { imported: 0, failed: true };
|
||||
const start = incrementalStart(connection.lastSyncedAt, {
|
||||
fullSync: opts.fullSync,
|
||||
});
|
||||
@@ -736,8 +840,9 @@ export async function syncUserGoogleHealth(
|
||||
// 401. Don't stamp success when the whole cycle was soft-skipped and nothing
|
||||
// imported; leave the status as-is so the "looks-healthy" window closes.
|
||||
const allSoftSkipped = tracker.count >= resources.length && total === 0;
|
||||
const failed = anyFailed || allSoftSkipped;
|
||||
|
||||
if (!anyFailed && !allSoftSkipped) {
|
||||
if (!failed) {
|
||||
// Stamp the watermark ONCE for the whole cycle, only on a non-degenerate
|
||||
// run. Every resource already saw the snapshot `start`, so stamping now()
|
||||
// here can't shrink a later resource's window.
|
||||
@@ -746,9 +851,9 @@ export async function syncUserGoogleHealth(
|
||||
}
|
||||
|
||||
annotate({
|
||||
action: { name: "googleHealth.sync", details: { imported: total } },
|
||||
action: { name: "googleHealth.sync", details: { imported: total, failed } },
|
||||
});
|
||||
return total;
|
||||
return { imported: total, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -817,7 +922,9 @@ export async function runGoogleHealthPollCohort(
|
||||
onUserSynced?: (userId: string, imported: number) => void;
|
||||
} = {},
|
||||
): Promise<{ usersSynced: number; measurementsImported: number }> {
|
||||
const sync = opts.sync ?? ((userId: string) => syncUserGoogleHealth(userId));
|
||||
const sync =
|
||||
opts.sync ??
|
||||
((userId: string) => syncUserGoogleHealth(userId).then((r) => r.imported));
|
||||
const limit = pLimit(opts.concurrency ?? GOOGLE_HEALTH_POLL_CONCURRENCY);
|
||||
|
||||
let usersSynced = 0;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const findMany = vi.fn();
|
||||
const queryRaw = vi.fn();
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
@@ -9,6 +10,9 @@ vi.mock("@/lib/db", () => ({
|
||||
// loadUserSourcePriority lazy-loads the source-priority blob; null
|
||||
// (default findUnique) falls back to the default ladders.
|
||||
user: { findUnique: vi.fn() },
|
||||
// v1.28.25 — dense types (BLOOD_GLUCOSE / PULSE) day-bucket the
|
||||
// cold-tier fallback in SQL.
|
||||
$queryRaw: (...args: unknown[]) => queryRaw(...args),
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -52,13 +56,13 @@ afterEach(() => {
|
||||
describe("buildGradedSeriesWithRollups — rollup-miss fallback", () => {
|
||||
const now = new Date("2026-05-31T12:00:00Z");
|
||||
|
||||
it("folds monthly/yearly from the full history when the tier has no coverage", async () => {
|
||||
it("folds monthly/yearly from the bounded fallback when the tier has no coverage", async () => {
|
||||
// The bounded recent read (~90 d) returns only recent points; the
|
||||
// full-history read returns 2+ years so the fold has monthly/yearly.
|
||||
// fallback read returns 2+ years so the fold has monthly/yearly.
|
||||
findMany
|
||||
// 1st call — bounded recent read (gte: since)
|
||||
.mockResolvedValueOnce(dailyRows(80, now))
|
||||
// 2nd call — full-history fallback read (no gte)
|
||||
// 2nd call — bounded fallback read (gte: 1095-day horizon)
|
||||
.mockResolvedValueOnce(dailyRows(800, now));
|
||||
|
||||
// Tier miss: no MONTH / YEAR coverage at all.
|
||||
@@ -68,12 +72,51 @@ describe("buildGradedSeriesWithRollups — rollup-miss fallback", () => {
|
||||
|
||||
// The bug: before the fix monthly/yearly were derived from the
|
||||
// bounded ~90-day read and so were always empty even when years of
|
||||
// raw history existed. After the fix they fold from the full history.
|
||||
// raw history existed. After the fix they fold from the fallback.
|
||||
expect(series.monthly.length).toBeGreaterThan(0);
|
||||
// 800 days of history puts at least one year into the yearly tail.
|
||||
expect(series.yearly.length).toBeGreaterThan(0);
|
||||
// The full-history fallback read must have been issued.
|
||||
// The fallback read must have been issued...
|
||||
expect(findMany).toHaveBeenCalledTimes(2);
|
||||
// ...and bounded (v1.28.25): it must carry a `measuredAt.gte` floor
|
||||
// ~1095 days back, never an unwindowed full-history walk.
|
||||
const fallbackArgs = findMany.mock.calls[1][0] as {
|
||||
where: { measuredAt?: { gte?: Date } };
|
||||
};
|
||||
const gte = fallbackArgs.where.measuredAt?.gte;
|
||||
expect(gte).toBeInstanceOf(Date);
|
||||
expect(gte!.getTime()).toBe(now.getTime() - 1095 * dayMs);
|
||||
});
|
||||
|
||||
it("day-buckets the dense-type fallback in SQL instead of a raw walk (v1.28.25)", async () => {
|
||||
// BLOOD_GLUCOSE on a cold tier (heavy import before the rollup
|
||||
// backfill warms) used to findMany the entire CGM history. The
|
||||
// fallback now runs a single day-bucket aggregate; only the bounded
|
||||
// recent read touches findMany.
|
||||
findMany.mockResolvedValueOnce(dailyRows(80, now)); // bounded recent
|
||||
const bucketRows: Array<{ bucket_start: Date; mean: number }> = [];
|
||||
for (let i = 0; i < 800; i++) {
|
||||
bucketRows.push({
|
||||
bucket_start: new Date(now.getTime() - i * dayMs),
|
||||
mean: 100,
|
||||
});
|
||||
}
|
||||
queryRaw.mockResolvedValueOnce(bucketRows.reverse());
|
||||
|
||||
readBestGranularityRollups.mockResolvedValue(null);
|
||||
|
||||
const series = await buildGradedSeriesWithRollups(
|
||||
"u1",
|
||||
"BLOOD_GLUCOSE",
|
||||
now,
|
||||
);
|
||||
|
||||
expect(series.monthly.length).toBeGreaterThan(0);
|
||||
expect(series.yearly.length).toBeGreaterThan(0);
|
||||
// Raw findMany ran once (the bounded recent read) — the fallback
|
||||
// itself went through the SQL aggregate.
|
||||
expect(findMany).toHaveBeenCalledTimes(1);
|
||||
expect(queryRaw).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads monthly/yearly from the tier when it has coverage and skips the full read", async () => {
|
||||
|
||||
@@ -19,6 +19,7 @@ const auditFindMany = vi.fn();
|
||||
const enqueueStatusGeneration = vi.fn();
|
||||
const userFindUnique = vi.fn();
|
||||
const measurementFindMany = vi.fn();
|
||||
const measurementGroupBy = vi.fn();
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
@@ -27,7 +28,12 @@ vi.mock("@/lib/db", () => ({
|
||||
findMany: (...a: unknown[]) => auditFindMany(...a),
|
||||
},
|
||||
user: { findUnique: (...a: unknown[]) => userFindUnique(...a) },
|
||||
measurement: { findMany: (...a: unknown[]) => measurementFindMany(...a) },
|
||||
measurement: {
|
||||
findMany: (...a: unknown[]) => measurementFindMany(...a),
|
||||
// v1.28.25 — type discovery runs as a server-side GROUP BY, not a
|
||||
// client-deduped `distinct` findMany.
|
||||
groupBy: (...a: unknown[]) => measurementGroupBy(...a),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -49,6 +55,7 @@ beforeEach(() => {
|
||||
enqueueStatusGeneration.mockResolvedValue(undefined);
|
||||
userFindUnique.mockResolvedValue({ locale: "de" });
|
||||
measurementFindMany.mockResolvedValue([]);
|
||||
measurementGroupBy.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
/** Distinct scopes the invalidator enqueued a regenerate for. */
|
||||
@@ -280,7 +287,7 @@ describe("invalidateStatusInsightsForTypes", () => {
|
||||
// ones for free — the hash baseline rows are never deleted.
|
||||
describe("enqueueStatusRefillForUser", () => {
|
||||
it("enqueues the seven specialised scopes plus the user's data-bearing generic scopes", async () => {
|
||||
measurementFindMany.mockResolvedValue([
|
||||
measurementGroupBy.mockResolvedValue([
|
||||
{ type: "WEIGHT" }, // specialised — no generic registry entry
|
||||
{ type: "BLOOD_GLUCOSE" }, // generic card
|
||||
{ type: "ACTIVE_ENERGY_BURNED" }, // generic card under a remapped id
|
||||
@@ -328,7 +335,7 @@ describe("enqueueStatusRefillForUser", () => {
|
||||
});
|
||||
|
||||
it("still refills the specialised scopes when the generic-scope discovery read fails", async () => {
|
||||
measurementFindMany.mockRejectedValue(new Error("pool exhausted"));
|
||||
measurementGroupBy.mockRejectedValue(new Error("pool exhausted"));
|
||||
const count = await enqueueStatusRefillForUser("u1", "de");
|
||||
expect(count).toBe(7);
|
||||
expect(enqueuedScopes()).toEqual([
|
||||
|
||||
@@ -25,6 +25,9 @@ vi.mock("@/lib/db", () => ({
|
||||
auditLog: { findFirst: vi.fn(), create: vi.fn() },
|
||||
measurement: { findMany: vi.fn() },
|
||||
measurementRollup: { findMany: vi.fn() },
|
||||
// v1.28.25 — the graded-series cold-tier fallback day-buckets dense
|
||||
// types (PULSE) via a raw aggregate instead of a full findMany walk.
|
||||
$queryRaw: vi.fn(async () => []),
|
||||
moodEntry: { findMany: vi.fn() },
|
||||
},
|
||||
}));
|
||||
@@ -46,6 +49,7 @@ import { generatePulseStatusForUser } from "../pulse-status";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never);
|
||||
});
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ vi.mock("@/lib/db", () => ({
|
||||
auditLog: { findFirst: vi.fn(), create: vi.fn() },
|
||||
measurement: { findMany: vi.fn() },
|
||||
measurementRollup: { findMany: vi.fn() },
|
||||
// v1.28.25 — the graded-series cold-tier fallback day-buckets dense
|
||||
// types (PULSE) via a raw aggregate instead of a full findMany walk.
|
||||
$queryRaw: vi.fn(async () => []),
|
||||
moodEntry: { findMany: vi.fn() },
|
||||
},
|
||||
}));
|
||||
@@ -47,6 +50,7 @@ function stubCompletion(
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
// Cold rollup tier: the pulse graded series folds monthly/yearly from
|
||||
// the full-history `measurement.findMany` fallback on a tier miss.
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never);
|
||||
@@ -69,6 +73,14 @@ describe("generatePulseStatusForUser — graded payload", () => {
|
||||
} as never);
|
||||
vi.mocked(prisma.auditLog.findFirst).mockResolvedValue(null);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue(records as never);
|
||||
// v1.28.25 — PULSE is a dense type, so the cold-tier fallback reads
|
||||
// a SQL day-bucket aggregate instead of the raw findMany walk. Feed
|
||||
// the same 1000 days as day buckets.
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue(
|
||||
records
|
||||
.map((r) => ({ bucket_start: r.measuredAt, mean: r.value }))
|
||||
.reverse() as never,
|
||||
);
|
||||
vi.mocked(prisma.moodEntry.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.auditLog.create).mockResolvedValue({
|
||||
createdAt: new Date(),
|
||||
|
||||
@@ -608,10 +608,13 @@ export async function enqueueStatusRefillForUser(
|
||||
): Promise<number> {
|
||||
const scopes = new Set<InsightStatusScope>(PER_STATUS_SCOPES);
|
||||
try {
|
||||
const rows = await prisma.measurement.findMany({
|
||||
// `groupBy` compiles to a server-side `GROUP BY` — Prisma's
|
||||
// `distinct` dedups in the client AFTER pulling every live row, which
|
||||
// on a dense multi-year account walks a six-figure row set to answer
|
||||
// "which types exist?" on the request path.
|
||||
const rows = await prisma.measurement.groupBy({
|
||||
by: ["type"],
|
||||
where: { userId, deletedAt: null },
|
||||
distinct: ["type"],
|
||||
select: { type: true },
|
||||
});
|
||||
for (const row of rows) {
|
||||
const metricId = metricIdForMeasurementType(row.type);
|
||||
|
||||
@@ -72,6 +72,29 @@ const WEEKLY_WINDOW_MS = (RECENT_DAYS + WEEKLY_WEEKS * 7) * MS_PER_DAY;
|
||||
// yearly buckets.
|
||||
const MONTHLY_WINDOW_MS = WEEKLY_WINDOW_MS + MONTHLY_MONTHS * 30 * MS_PER_DAY;
|
||||
|
||||
/**
|
||||
* v1.28.25 — bound on the cold-tier fallback read. The coverage-miss
|
||||
* fallback used to `findMany` the FULL raw history with no window or
|
||||
* take; on a CGM / per-sample-pulse account that had just imported years
|
||||
* of data (exactly the state where the rollup tier is still cold) it
|
||||
* pulled six-figure row sets into JS on a request path. 1095 days
|
||||
* matches the yearly rollup router window below, so the cold path now
|
||||
* covers the same horizon the warm path serves.
|
||||
*/
|
||||
const COLD_FALLBACK_WINDOW_DAYS = 1095;
|
||||
|
||||
/**
|
||||
* Sample-dense types whose cold-tier fallback is day-bucketed in SQL
|
||||
* (one AVG row per day) instead of read raw — same dense set the series
|
||||
* route caps (CGM glucose ~288 rows/day, PULSE per-sample / hourly).
|
||||
* The fold's output shape is unchanged; on the cold path only, a dense
|
||||
* type's monthly/yearly min/max/n derive from day means rather than
|
||||
* individual samples — the warm rollup tier restores sample-exact
|
||||
* figures as soon as the backfill lands.
|
||||
*/
|
||||
const DENSE_FALLBACK_TYPES: ReadonlySet<MeasurementType> =
|
||||
new Set<MeasurementType>(["BLOOD_GLUCOSE", "PULSE"]);
|
||||
|
||||
export interface RecentDayBucket {
|
||||
/** Berlin YYYY-MM-DD. */
|
||||
date: string;
|
||||
@@ -326,12 +349,13 @@ function rollupYearly(rows: RollupBucketRow[]): YearlyBucket[] {
|
||||
* caught up on — the bounded ~90-day read can NOT supply the monthly /
|
||||
* yearly slices (every row in it folds into recent / weekly), so naively
|
||||
* reusing it would ship empty monthly / yearly arrays even when years of
|
||||
* raw history exist. On that miss this falls back to a full-history
|
||||
* in-memory fold so the coarse slices are populated from the raw rows
|
||||
* rather than left falsely empty. The full read only happens when the
|
||||
* tier is cold, which is exactly the case the write-amplified tier was
|
||||
* meant to avoid — so the hot path stays bounded and the cold path stays
|
||||
* correct.
|
||||
* raw history exist. On that miss this falls back to an in-memory fold
|
||||
* over a bounded window (1095 days — the yearly router's own horizon;
|
||||
* sample-dense types are additionally day-bucketed in SQL) so the coarse
|
||||
* slices are populated from the raw rows rather than left falsely empty.
|
||||
* The fallback read only happens when the tier is cold, which is exactly
|
||||
* the case the write-amplified tier was meant to avoid — so the hot path
|
||||
* stays bounded and the cold path stays correct.
|
||||
*/
|
||||
export async function buildGradedSeriesWithRollups(
|
||||
userId: string,
|
||||
@@ -396,19 +420,57 @@ export async function buildGradedSeriesWithRollups(
|
||||
}
|
||||
|
||||
// Tier coverage miss for one or both coarse slices: the bounded
|
||||
// ~90-day read can't supply them, so fold the FULL history in memory
|
||||
// and take whichever slices the tier left empty. The full read is the
|
||||
// exception (cold-tier accounts), never the warm-tier norm.
|
||||
// ~90-day read can't supply them, so fold a BOUNDED history window in
|
||||
// memory and take whichever slices the tier left empty. The fallback
|
||||
// read is the exception (cold-tier accounts), never the warm-tier
|
||||
// norm. v1.28.25 — the read is bounded to the same 1095-day horizon
|
||||
// the yearly rollup router serves (it was previously unwindowed), and
|
||||
// the sample-dense types are day-bucketed in SQL so a heavy import
|
||||
// ahead of the rollup backfill can't drag six-figure row sets into JS.
|
||||
if (!monthlyCovered || !yearlyCovered) {
|
||||
const allRows = await prisma.measurement.findMany({
|
||||
where: { userId, type, deletedAt: null },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
select: { measuredAt: true, value: true },
|
||||
});
|
||||
const fullGraded = buildGradedSeriesFromPoints(
|
||||
allRows.map((r) => ({ measuredAt: r.measuredAt, value: r.value })),
|
||||
now,
|
||||
const fallbackSince = new Date(
|
||||
now.getTime() - COLD_FALLBACK_WINDOW_DAYS * MS_PER_DAY,
|
||||
);
|
||||
let fallbackPoints: Point[];
|
||||
if (DENSE_FALLBACK_TYPES.has(type)) {
|
||||
// Parameter-bound day-bucket aggregate, mirroring the rollup tier's
|
||||
// `date_trunc(...) GROUP BY` (measurement-rollups.ts) — at most
|
||||
// ~1095 rows regardless of sample density.
|
||||
const bucketRows = await prisma.$queryRaw<
|
||||
Array<{ bucket_start: Date; mean: number }>
|
||||
>`
|
||||
SELECT
|
||||
date_trunc('day', m."measured_at") AS bucket_start,
|
||||
AVG(m."value")::double precision AS mean
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."type" = ${type}::"measurement_type"
|
||||
AND m."measured_at" >= ${fallbackSince}
|
||||
AND m."deleted_at" IS NULL
|
||||
GROUP BY date_trunc('day', m."measured_at")
|
||||
ORDER BY bucket_start ASC
|
||||
`;
|
||||
fallbackPoints = bucketRows.map((r) => ({
|
||||
measuredAt: r.bucket_start,
|
||||
value: r.mean,
|
||||
}));
|
||||
} else {
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type,
|
||||
deletedAt: null,
|
||||
measuredAt: { gte: fallbackSince },
|
||||
},
|
||||
orderBy: { measuredAt: "asc" },
|
||||
select: { measuredAt: true, value: true },
|
||||
});
|
||||
fallbackPoints = rows.map((r) => ({
|
||||
measuredAt: r.measuredAt,
|
||||
value: r.value,
|
||||
}));
|
||||
}
|
||||
const fullGraded = buildGradedSeriesFromPoints(fallbackPoints, now);
|
||||
if (!monthlyCovered) monthly = fullGraded.monthly;
|
||||
if (!yearlyCovered) yearly = fullGraded.yearly;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* Pins the verdict gate of `runGoogleHealthBackfillForUser`: the
|
||||
* `backfillCompletedAt` marker may only be stamped after a CLEAN full-history
|
||||
* run. `syncUserGoogleHealth` swallows per-resource errors into its verdict —
|
||||
* before the gate, ANY run (partial hard failure, parked no-op) stamped the
|
||||
* marker and the pg-boss `retryLimit: 3` was dead code.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { prismaMock, syncUserGoogleHealthMock, isReauthRequiredMock } =
|
||||
vi.hoisted(() => ({
|
||||
prismaMock: {
|
||||
googleHealthConnection: { findMany: vi.fn(), update: vi.fn() },
|
||||
},
|
||||
syncUserGoogleHealthMock: vi.fn(),
|
||||
isReauthRequiredMock: vi.fn(async () => false),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({ prisma: prismaMock }));
|
||||
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({
|
||||
getGlobalBoss: () => ({ send: vi.fn() }),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
annotate: () => {},
|
||||
getEvent: () => null,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: isReauthRequiredMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/google-health/sync", () => ({
|
||||
GOOGLE_HEALTH_INTEGRATION_KEY: "google-health",
|
||||
syncUserGoogleHealth: (...a: unknown[]) => syncUserGoogleHealthMock(...a),
|
||||
}));
|
||||
|
||||
import { runGoogleHealthBackfillForUser } from "../google-health-backfill";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
isReauthRequiredMock.mockResolvedValue(false);
|
||||
prismaMock.googleHealthConnection.update.mockResolvedValue({});
|
||||
});
|
||||
|
||||
describe("runGoogleHealthBackfillForUser — verdict-gated marker", () => {
|
||||
it("stamps backfillCompletedAt after a CLEAN full-history run", async () => {
|
||||
syncUserGoogleHealthMock.mockResolvedValue({ imported: 42, failed: false });
|
||||
|
||||
const { imported } = await runGoogleHealthBackfillForUser("u1");
|
||||
|
||||
expect(imported).toBe(42);
|
||||
expect(syncUserGoogleHealthMock).toHaveBeenCalledWith("u1", {
|
||||
fullSync: true,
|
||||
});
|
||||
const updateArg = prismaMock.googleHealthConnection.update.mock
|
||||
.calls[0]![0] as {
|
||||
where: Record<string, unknown>;
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(updateArg.where).toEqual({ userId: "u1" });
|
||||
expect(updateArg.data.backfillCompletedAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("a failed verdict THROWS without stamping — pg-boss retries become real", async () => {
|
||||
syncUserGoogleHealthMock.mockResolvedValue({ imported: 7, failed: true });
|
||||
|
||||
await expect(runGoogleHealthBackfillForUser("u1")).rejects.toThrow(
|
||||
/incomplete/,
|
||||
);
|
||||
expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("a connection parked at error_reauth returns WITHOUT running the sync or stamping", async () => {
|
||||
isReauthRequiredMock.mockResolvedValue(true);
|
||||
|
||||
await expect(runGoogleHealthBackfillForUser("u1")).resolves.toEqual({
|
||||
imported: 0,
|
||||
});
|
||||
expect(syncUserGoogleHealthMock).not.toHaveBeenCalled();
|
||||
expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -10,13 +10,15 @@
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { prismaMock, bossSend, syncUserSleep } = vi.hoisted(() => ({
|
||||
prismaMock: {
|
||||
googleHealthConnection: { findMany: vi.fn(), update: vi.fn() },
|
||||
},
|
||||
bossSend: vi.fn(),
|
||||
syncUserSleep: vi.fn(),
|
||||
}));
|
||||
const { prismaMock, bossSend, syncUserSleep, isReauthRequiredMock } =
|
||||
vi.hoisted(() => ({
|
||||
prismaMock: {
|
||||
googleHealthConnection: { findMany: vi.fn(), update: vi.fn() },
|
||||
},
|
||||
bossSend: vi.fn(),
|
||||
syncUserSleep: vi.fn(),
|
||||
isReauthRequiredMock: vi.fn(async () => false),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({ prisma: prismaMock }));
|
||||
|
||||
@@ -33,6 +35,33 @@ vi.mock("@/lib/logging/context", () => ({
|
||||
getEvent: () => null,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: isReauthRequiredMock,
|
||||
recordSyncFailure: vi.fn(async () => {}),
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
// The repair module threads the REAL hard-fail ledger from
|
||||
// `@/lib/google-health/sync`; its heavy transitive deps are mocked the same
|
||||
// way the google-health failsoft suite does.
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
recomputeBucketsForMeasurement: vi.fn(async () => {}),
|
||||
recomputeUserRollups: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/google-health/credentials", () => ({
|
||||
getUserGoogleHealthCredentials: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("@/lib/google-health/client", () => ({ refreshAccessToken: vi.fn() }));
|
||||
|
||||
import {
|
||||
GOOGLE_HEALTH_TOKEN_HARD_FAIL,
|
||||
noteHardFailure,
|
||||
} from "@/lib/google-health/sync";
|
||||
import {
|
||||
enqueueBootTimeGoogleHealthSleepRepair,
|
||||
runGoogleHealthSleepRepairForUser,
|
||||
@@ -40,6 +69,7 @@ import {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
isReauthRequiredMock.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
describe("enqueueBootTimeGoogleHealthSleepRepair — discovery", () => {
|
||||
@@ -139,4 +169,41 @@ describe("runGoogleHealthSleepRepairForUser", () => {
|
||||
);
|
||||
expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("a swallowed hard failure (ledger entry) throws and does NOT stamp — pg-boss retries", async () => {
|
||||
// `syncUserSleep` swallows fetch/write hard failures into the ambient
|
||||
// ledger and still resolves a count; the repair must read the ledger, not
|
||||
// the count, before stamping.
|
||||
syncUserSleep.mockImplementation(async () => {
|
||||
noteHardFailure("fetchSleep");
|
||||
return 5;
|
||||
});
|
||||
|
||||
await expect(runGoogleHealthSleepRepairForUser("u1")).rejects.toThrow(
|
||||
/incomplete/,
|
||||
);
|
||||
expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("a dead token returns WITHOUT stamping and WITHOUT throwing (boot discovery re-enqueues)", async () => {
|
||||
syncUserSleep.mockImplementation(async () => {
|
||||
noteHardFailure(GOOGLE_HEALTH_TOKEN_HARD_FAIL);
|
||||
return 0;
|
||||
});
|
||||
|
||||
await expect(runGoogleHealthSleepRepairForUser("u1")).resolves.toEqual({
|
||||
imported: 0,
|
||||
});
|
||||
expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("a connection parked at error_reauth returns WITHOUT running the sync or stamping", async () => {
|
||||
isReauthRequiredMock.mockResolvedValue(true);
|
||||
|
||||
await expect(runGoogleHealthSleepRepairForUser("u1")).resolves.toEqual({
|
||||
imported: 0,
|
||||
});
|
||||
expect(syncUserSleep).not.toHaveBeenCalled();
|
||||
expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -171,7 +171,12 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{
|
||||
Date.now() - DENSE_INTRADAY_RETENTION_DAYS * 24 * 60 * 60 * 1000,
|
||||
);
|
||||
|
||||
const users = await prisma.measurement.findMany({
|
||||
// `groupBy` compiles to a server-side `GROUP BY` — Prisma's
|
||||
// `distinct` dedups in the client AFTER pulling every matching row,
|
||||
// an instance-wide scan of every out-of-window dense-tier row just
|
||||
// to list user ids.
|
||||
const users = await prisma.measurement.groupBy({
|
||||
by: ["userId"],
|
||||
where: {
|
||||
source: "APPLE_HEALTH",
|
||||
type: { in: types },
|
||||
@@ -179,8 +184,6 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{
|
||||
NOT: { externalId: { startsWith: "stats:" } },
|
||||
measuredAt: { lt: windowStart },
|
||||
},
|
||||
select: { userId: true },
|
||||
distinct: ["userId"],
|
||||
});
|
||||
|
||||
if (users.length === 0) {
|
||||
|
||||
@@ -14,7 +14,11 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { syncUserGoogleHealth } from "@/lib/google-health/sync";
|
||||
import { isReauthRequired } from "@/lib/integrations/status";
|
||||
import {
|
||||
GOOGLE_HEALTH_INTEGRATION_KEY,
|
||||
syncUserGoogleHealth,
|
||||
} from "@/lib/google-health/sync";
|
||||
|
||||
export const GOOGLE_HEALTH_BACKFILL_QUEUE = "google-health-backfill";
|
||||
|
||||
@@ -32,14 +36,44 @@ export interface GoogleHealthBackfillPayload {
|
||||
|
||||
/**
|
||||
* Per-user backfill handler. Runs a full-history sync for one account and
|
||||
* stamps `backfillCompletedAt` so the discovery query drops it. Idempotent: the
|
||||
* per-resource upserts are key-stable, so a re-run (e.g. a reboot mid-walk)
|
||||
* overwrites rather than duplicating. Mirrors `runFitbitBackfillForUser`.
|
||||
* stamps `backfillCompletedAt` ONLY on a clean run, so the discovery query
|
||||
* drops it. Idempotent: the per-resource upserts are key-stable, so a re-run
|
||||
* (e.g. a reboot mid-walk) overwrites rather than duplicating. Mirrors
|
||||
* `runFitbitBackfillForUser`.
|
||||
*
|
||||
* Verdict-gated: a partial hard failure (one collection 400ing, a write
|
||||
* failing) THROWS so pg-boss retries the job — stamping the marker over a
|
||||
* partial walk would freeze the gap in forever. A connection parked at
|
||||
* `error_reauth` is NOT clean either, but throwing against a dead grant would
|
||||
* just burn the retry budget on the same no-op: return WITHOUT stamping — the
|
||||
* boot discovery re-enqueues the account on the next boot, and the stamp lands
|
||||
* once the user reconnects and a clean walk completes.
|
||||
*/
|
||||
export async function runGoogleHealthBackfillForUser(
|
||||
userId: string,
|
||||
): Promise<{ imported: number }> {
|
||||
const imported = await syncUserGoogleHealth(userId, { fullSync: true });
|
||||
if (await isReauthRequired(userId, GOOGLE_HEALTH_INTEGRATION_KEY)) {
|
||||
annotate({
|
||||
action: {
|
||||
name: "google_health.backfill.skipped_reauth",
|
||||
details: { imported: 0 },
|
||||
},
|
||||
});
|
||||
return { imported: 0 };
|
||||
}
|
||||
|
||||
const { imported, failed } = await syncUserGoogleHealth(userId, {
|
||||
fullSync: true,
|
||||
});
|
||||
|
||||
if (failed) {
|
||||
// Surface through the pg-boss retry path (retryLimit 3, backoff). If the
|
||||
// failure parked the connection at error_reauth meanwhile, the next
|
||||
// attempt takes the reauth return above instead of failing again.
|
||||
throw new Error(
|
||||
`google-health backfill incomplete for user ${userId} — marker not stamped`,
|
||||
);
|
||||
}
|
||||
|
||||
await prisma.googleHealthConnection.update({
|
||||
where: { userId },
|
||||
|
||||
@@ -26,6 +26,12 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { isReauthRequired } from "@/lib/integrations/status";
|
||||
import {
|
||||
GOOGLE_HEALTH_INTEGRATION_KEY,
|
||||
GOOGLE_HEALTH_TOKEN_HARD_FAIL,
|
||||
runWithGoogleHealthHardFailLedger,
|
||||
} from "@/lib/google-health/sync";
|
||||
import { syncUserSleep } from "@/lib/google-health/sync-sleep";
|
||||
|
||||
export const GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE = "google-health-sleep-repair";
|
||||
@@ -50,11 +56,51 @@ export interface GoogleHealthSleepRepairPayload {
|
||||
* Idempotent: the per-segment upserts are key-stable and the replace-by-window
|
||||
* clears stale copies, so a re-run (e.g. a reboot mid-walk) converges rather
|
||||
* than duplicating. Does NOT move `lastSyncedAt`.
|
||||
*
|
||||
* Verdict-gated: `syncUserSleep` swallows fetch/write hard failures into the
|
||||
* ambient hard-fail ledger (returning a count either way), so the repair runs
|
||||
* it inside its own ledger scope — the same mechanism `syncUserGoogleHealth`
|
||||
* uses for its cycle verdict — and stamps `sleepRepairedAt` ONLY on a clean
|
||||
* run. A hard fetch/write failure THROWS so pg-boss retries; a dead token
|
||||
* (parked connection, missing credentials, failed refresh) is NOT clean but
|
||||
* does not throw — retrying a dead grant burns the retry budget on the same
|
||||
* no-op. It returns WITHOUT stamping instead: the boot discovery re-enqueues
|
||||
* the account on the next boot, and the stamp lands after a reconnect.
|
||||
*/
|
||||
export async function runGoogleHealthSleepRepairForUser(
|
||||
userId: string,
|
||||
): Promise<{ imported: number }> {
|
||||
const imported = await syncUserSleep(userId, { deferRollup: false });
|
||||
if (await isReauthRequired(userId, GOOGLE_HEALTH_INTEGRATION_KEY)) {
|
||||
annotate({
|
||||
action: {
|
||||
name: "googleHealth.sleepRepair.skipped_reauth",
|
||||
details: { imported: 0 },
|
||||
},
|
||||
});
|
||||
return { imported: 0 };
|
||||
}
|
||||
|
||||
const { result: imported, failures } =
|
||||
await runWithGoogleHealthHardFailLedger(() =>
|
||||
syncUserSleep(userId, { deferRollup: false }),
|
||||
);
|
||||
|
||||
if (failures.length > 0) {
|
||||
if (failures.every((f) => f === GOOGLE_HEALTH_TOKEN_HARD_FAIL)) {
|
||||
// Dead token: nothing was fetched, so nothing to retry until the user
|
||||
// reconnects. No stamp — the next boot's discovery picks it back up.
|
||||
annotate({
|
||||
action: {
|
||||
name: "googleHealth.sleepRepair.skipped_token",
|
||||
details: { imported },
|
||||
},
|
||||
});
|
||||
return { imported };
|
||||
}
|
||||
throw new Error(
|
||||
`google-health sleep repair incomplete for user ${userId} (${failures.join(", ")}) — marker not stamped`,
|
||||
);
|
||||
}
|
||||
|
||||
await prisma.googleHealthConnection.update({
|
||||
where: { userId },
|
||||
|
||||
@@ -388,17 +388,21 @@ async function warmGenericMetricCaches(
|
||||
// rows for. Best-effort — a read failure (or a worker prisma surface
|
||||
// without `measurement`) yields zero metrics rather than aborting the
|
||||
// user's whole pre-generation loop.
|
||||
let rows: Array<{ type: string }>;
|
||||
let typesWithData: ReadonlySet<string>;
|
||||
try {
|
||||
rows = await prisma.measurement.findMany({
|
||||
// `groupBy` compiles to a server-side `GROUP BY` — Prisma's
|
||||
// `distinct` dedups in the client AFTER pulling every live row, which
|
||||
// on a dense multi-year account walks a six-figure row set nightly to
|
||||
// answer "which types exist?".
|
||||
const rows = await prisma.measurement.groupBy({
|
||||
by: ["type"],
|
||||
where: { userId, deletedAt: null },
|
||||
distinct: ["type"],
|
||||
select: { type: true },
|
||||
orderBy: { type: "asc" },
|
||||
});
|
||||
typesWithData = new Set(rows.map((r) => r.type));
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
const typesWithData = new Set(rows.map((r) => r.type));
|
||||
|
||||
const metricsWithData = METRIC_STATUS_IDS.filter(
|
||||
(id: MetricStatusMetricId) => {
|
||||
|
||||
@@ -106,15 +106,19 @@ export async function enqueueBootTimeMeanConsolidation(
|
||||
return { enqueued: 0, skipped: 0, error: null };
|
||||
}
|
||||
|
||||
const users = await prisma.measurement.findMany({
|
||||
// `groupBy` compiles to a server-side `GROUP BY` — Prisma's
|
||||
// `distinct` dedups in the client AFTER pulling every matching row,
|
||||
// an instance-wide unbounded scan just to list user ids. No date
|
||||
// bound on purpose: a fresh import of OLD per-sample rows must still
|
||||
// surface its user here, however old the rows are.
|
||||
const users = await prisma.measurement.groupBy({
|
||||
by: ["userId"],
|
||||
where: {
|
||||
source: "APPLE_HEALTH",
|
||||
type: { in: types },
|
||||
deletedAt: null,
|
||||
NOT: { externalId: { startsWith: "stats:" } },
|
||||
},
|
||||
select: { userId: true },
|
||||
distinct: ["userId"],
|
||||
});
|
||||
|
||||
if (users.length === 0) {
|
||||
|
||||
@@ -116,6 +116,10 @@ describe("syncUserNightscout", () => {
|
||||
const arg = upsertMock.mock.calls[0]![0];
|
||||
// An immutable sample: update must not carry a new value/measuredAt.
|
||||
expect(arg.update).not.toHaveProperty("value");
|
||||
expect(arg.update).not.toHaveProperty("measuredAt");
|
||||
// But it DOES carry the resurrection — Nightscout owns its rows, so a
|
||||
// re-synced reading clears a tombstone (no-op on a live row).
|
||||
expect(arg.update).toEqual({ deletedAt: null });
|
||||
});
|
||||
|
||||
it("records sync success after a clean pass", async () => {
|
||||
|
||||
@@ -167,7 +167,11 @@ export async function upsertNightscoutEntries(
|
||||
},
|
||||
// First-write-wins: a CGM sample is immutable, so a re-sync of the
|
||||
// same reading must not rewrite the stored value or measuredAt.
|
||||
update: {},
|
||||
// `deletedAt: null` is a no-op on a live row, a deliberate
|
||||
// RESURRECTION on a tombstoned one — Nightscout is the source of
|
||||
// truth for its own rows, so a re-synced reading brings the row
|
||||
// back (mirrors Google / Fitbit).
|
||||
update: { deletedAt: null },
|
||||
});
|
||||
touched.push({ type, measuredAt: mapped.measuredAt });
|
||||
imported++;
|
||||
|
||||
@@ -229,10 +229,46 @@ describe("mapSleep", () => {
|
||||
const onset = new Date("2026-06-09T23:00:00.000Z").getTime();
|
||||
expect(mapped[0].measuredAt.getTime()).toBe(onset + 10 * 60_000);
|
||||
expect(mapped[1].measuredAt.getTime()).toBe(onset + 25 * 60_000);
|
||||
// Distinct, record-scoped, segment-indexed externalIds (no collapse).
|
||||
// Distinct, record-scoped externalIds keyed on each run's own START
|
||||
// instant (no collapse). NOT a positional run index: a revised hypnogram
|
||||
// re-segments the night, so an index shifted every following run onto the
|
||||
// wrong row and left phantom tail rows the night-total double-counted.
|
||||
const ids = mapped.map((m) => m.externalId);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
expect(ids[0]).toBe("sleep:rec-T:seg:0");
|
||||
expect(ids[0]).toBe("sleep:rec-T:seg:2026-06-09T23:00:00.000Z");
|
||||
expect(ids[1]).toBe("sleep:rec-T:seg:2026-06-09T23:10:00.000Z");
|
||||
});
|
||||
|
||||
it("keeps IDENTICAL externalIds for unchanged runs across a revised hypnogram", () => {
|
||||
// The same night re-scored: Oura re-classifies the SECOND run's digits,
|
||||
// splitting one run into two — every later run's index would have
|
||||
// shifted (+1) under the retired run-indexed key, minting fresh ids. The
|
||||
// start-keyed ids of the UNCHANGED first and last runs must survive; the
|
||||
// re-segmented middle mints new ids and the record-scoped sweep clears
|
||||
// what the revision orphaned.
|
||||
const base = {
|
||||
id: "rec-T",
|
||||
day: "2026-06-10",
|
||||
bedtime_start: "2026-06-09T23:00:00.000Z",
|
||||
bedtime_end: "2026-06-10T07:00:00.000Z",
|
||||
};
|
||||
const idsOf = (hyp: string) =>
|
||||
mapSleep({ ...base, sleep_phase_5_min: hyp })
|
||||
.filter((m) => m.type === "SLEEP_DURATION")
|
||||
.map((m) => m.externalId);
|
||||
|
||||
const first = idsOf("1122234");
|
||||
const revised = idsOf("1121234"); // run "222" split into "2","1","2"
|
||||
|
||||
// First run (digits 0-1) and the tail runs that keep their start offsets
|
||||
// are identical across the revision.
|
||||
expect(revised[0]).toBe(first[0]);
|
||||
expect(revised.at(-1)).toBe(first.at(-1));
|
||||
// The re-segmented middle mints ids keyed on the new run starts — all
|
||||
// still under the record's `sleep:rec-T:seg:` sweep prefix.
|
||||
expect(revised.every((id) => id!.startsWith("sleep:rec-T:seg:"))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores an empty / malformed hypnogram and uses stage totals", () => {
|
||||
|
||||
@@ -14,6 +14,7 @@ const {
|
||||
fetchResilienceMock,
|
||||
refreshMock,
|
||||
upsertMock,
|
||||
updateManyMock,
|
||||
recordSuccessMock,
|
||||
recordFailureMock,
|
||||
recomputeMock,
|
||||
@@ -32,6 +33,7 @@ const {
|
||||
fetchResilienceMock: vi.fn(),
|
||||
refreshMock: vi.fn(),
|
||||
upsertMock: vi.fn(),
|
||||
updateManyMock: vi.fn(),
|
||||
recordSuccessMock: vi.fn(),
|
||||
recordFailureMock: vi.fn(),
|
||||
recomputeMock: vi.fn(),
|
||||
@@ -45,7 +47,7 @@ vi.mock("../credentials", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: { measurement: { upsert: upsertMock } },
|
||||
prisma: { measurement: { upsert: upsertMock, updateMany: updateManyMock } },
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/integrations/status", async (importOriginal) => ({
|
||||
@@ -107,6 +109,7 @@ beforeEach(() => {
|
||||
fetchResilienceMock.mockReset().mockResolvedValue([]);
|
||||
refreshMock.mockReset();
|
||||
upsertMock.mockReset().mockResolvedValue({});
|
||||
updateManyMock.mockReset().mockResolvedValue({ count: 0 });
|
||||
recordSuccessMock.mockReset().mockResolvedValue(undefined);
|
||||
recordFailureMock.mockReset().mockResolvedValue(undefined);
|
||||
recomputeMock.mockReset().mockResolvedValue(undefined);
|
||||
@@ -231,6 +234,69 @@ describe("syncUserOura", () => {
|
||||
expect(new Set(externalIds).size).toBe(externalIds.length);
|
||||
});
|
||||
|
||||
it("sweeps stale sleep rows per fetched record: live-only, record-prefixed, notIn the fresh set, soft-delete (v1.28.25)", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchSleepMock.mockResolvedValue([
|
||||
{
|
||||
id: "rec-T",
|
||||
day: "2026-06-10",
|
||||
bedtime_start: "2026-06-09T23:00:00.000Z",
|
||||
bedtime_end: "2026-06-10T07:00:00.000Z",
|
||||
sleep_phase_5_min: "1122234",
|
||||
},
|
||||
]);
|
||||
|
||||
await syncUserOura("u1");
|
||||
|
||||
expect(updateManyMock).toHaveBeenCalledTimes(1);
|
||||
const arg = updateManyMock.mock.calls[0]![0] as {
|
||||
where: {
|
||||
userId: string;
|
||||
source: string;
|
||||
type: string;
|
||||
deletedAt: null;
|
||||
externalId: { startsWith: string; notIn: string[] };
|
||||
};
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(arg.where.userId).toBe("u1");
|
||||
expect(arg.where.source).toBe("OURA");
|
||||
expect(arg.where.type).toBe("SLEEP_DURATION");
|
||||
expect(arg.where.deletedAt).toBeNull();
|
||||
// Bounded to THIS record. The prefix covers the whole record slice (runs
|
||||
// AND the stage-total fallback shape) so a night that flips between the
|
||||
// two shapes cannot leave the other shape's rows double-counting.
|
||||
expect(arg.where.externalId.startsWith).toBe("sleep:rec-T:");
|
||||
// Every fresh SLEEP_DURATION id of the record is protected.
|
||||
expect(arg.where.externalId.notIn).toEqual([
|
||||
"sleep:rec-T:seg:2026-06-09T23:00:00.000Z",
|
||||
"sleep:rec-T:seg:2026-06-09T23:10:00.000Z",
|
||||
"sleep:rec-T:seg:2026-06-09T23:25:00.000Z",
|
||||
"sleep:rec-T:seg:2026-06-09T23:30:00.000Z",
|
||||
]);
|
||||
// Soft delete only.
|
||||
expect(arg.data).toEqual({ deletedAt: expect.any(Date) });
|
||||
|
||||
// A legacy run-indexed row for this record falls inside the sweep.
|
||||
const legacyId = "sleep:rec-T:seg:0";
|
||||
expect(legacyId.startsWith(arg.where.externalId.startsWith)).toBe(true);
|
||||
expect(arg.where.externalId.notIn).not.toContain(legacyId);
|
||||
|
||||
// Replace-then-write: the sweep runs before the first upsert.
|
||||
expect(updateManyMock.mock.invocationCallOrder[0]!).toBeLessThan(
|
||||
upsertMock.mock.invocationCallOrder[0]!,
|
||||
);
|
||||
});
|
||||
|
||||
it("never sweeps when the sleep collection returns no records", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchReadinessMock.mockResolvedValue([
|
||||
{ id: "1", day: "2026-06-10", score: 70 },
|
||||
]);
|
||||
await syncUserOura("u1");
|
||||
expect(updateManyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("writes the Sleep Score and SpO2 from the new collections", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchDailySleepMock.mockResolvedValue([
|
||||
|
||||
+11
-5
@@ -548,7 +548,6 @@ function mapSleepTimeline(s: OuraSleep): MappedMeasurement[] | null {
|
||||
if (Number.isNaN(onset.getTime())) return null;
|
||||
|
||||
const out: MappedMeasurement[] = [];
|
||||
let segIndex = 0;
|
||||
let i = 0;
|
||||
while (i < hyp.length) {
|
||||
const digit = hyp[i];
|
||||
@@ -559,7 +558,15 @@ function mapSleepTimeline(s: OuraSleep): MappedMeasurement[] | null {
|
||||
if (stage) {
|
||||
const segStartMs = onset.getTime() + i * HYPNOGRAM_INTERVAL_MS;
|
||||
const segEndMs = segStartMs + run * HYPNOGRAM_INTERVAL_MS;
|
||||
const fieldTag = `seg:${segIndex}`;
|
||||
// Keyed by the run's own START instant, NOT a positional run index. A
|
||||
// revised hypnogram re-segments the night (runs merge/split), so an
|
||||
// index shifted every following run onto the wrong row — wrong
|
||||
// boundaries updated in place plus phantom tail rows the night-total
|
||||
// double-counted. A run's start instant is a stable coordinate: an
|
||||
// unchanged run keeps its id (re-classification overwrites in place),
|
||||
// and whatever the revision orphaned is swept by the record-scoped
|
||||
// cleanup in the sync (which also clears legacy `seg:<i>` rows).
|
||||
const fieldTag = `seg:${new Date(segStartMs).toISOString()}`;
|
||||
out.push({
|
||||
type: "SLEEP_DURATION",
|
||||
value: round2(run * HYPNOGRAM_INTERVAL_MIN),
|
||||
@@ -567,11 +574,10 @@ function mapSleepTimeline(s: OuraSleep): MappedMeasurement[] | null {
|
||||
measuredAt: new Date(segEndMs),
|
||||
fieldTag,
|
||||
sleepStage: stage,
|
||||
// Record-scoped + segment-indexed so a nap's segments never collide
|
||||
// with the main sleep's and a re-fetch overwrites in place.
|
||||
// Record-scoped so a nap's segments never collide with the main
|
||||
// sleep's and a re-fetch overwrites in place.
|
||||
externalId: `sleep:${s.id}:${fieldTag}`,
|
||||
});
|
||||
segIndex += 1;
|
||||
}
|
||||
i += run;
|
||||
}
|
||||
|
||||
+54
-7
@@ -47,6 +47,10 @@ import {
|
||||
recomputeBucketsForMeasurement,
|
||||
} from "@/lib/rollups/measurement-rollups";
|
||||
import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate";
|
||||
import {
|
||||
sweepStaleSleepSegments,
|
||||
type SleepSegmentSweep,
|
||||
} from "@/lib/sleep/sweep-stale-segments";
|
||||
import {
|
||||
fetchCardiovascularAge,
|
||||
fetchDailyActivity,
|
||||
@@ -133,6 +137,13 @@ export interface OuraCollectionFailure {
|
||||
export interface OuraFetchResult {
|
||||
readings: OuraMeasurementUpsert[];
|
||||
failures: OuraCollectionFailure[];
|
||||
/**
|
||||
* One sweep entry per fetched sleep record (`sleep:<record-id>:` prefix +
|
||||
* that record's fresh SLEEP_DURATION externalIds). Drives the record-scoped
|
||||
* cleanup of rows a revised hypnogram orphaned — including every legacy
|
||||
* run-indexed `seg:<i>` row — before the fresh set upserts.
|
||||
*/
|
||||
sleepSweeps: SleepSegmentSweep[];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -162,6 +173,12 @@ async function fetchAll(
|
||||
const start = new Date(now.getTime() - lookbackDays * 24 * 60 * 60 * 1000);
|
||||
const query = { startDate: ymd(start), endDate: ymd(now) };
|
||||
|
||||
// Filled by the sleep collection's closure as it maps records; carried on
|
||||
// the result so the caller can run the record-scoped sweep before the
|
||||
// upsert. A collection that throws pushes nothing (the fetch is its first
|
||||
// await), so a 401-retry cannot double-enter a record.
|
||||
const sleepSweeps: SleepSegmentSweep[] = [];
|
||||
|
||||
const collections: OuraCollectionSpec[] = [
|
||||
{
|
||||
name: "readiness",
|
||||
@@ -172,10 +189,27 @@ async function fetchAll(
|
||||
},
|
||||
{
|
||||
name: "sleep",
|
||||
collect: async (t) =>
|
||||
(await fetchSleep(t, query)).flatMap((s) =>
|
||||
toUpsert(mapSleep(s), "sleep"),
|
||||
),
|
||||
collect: async (t) => {
|
||||
const records = await fetchSleep(t, query);
|
||||
const ups: OuraMeasurementUpsert[] = [];
|
||||
for (const s of records) {
|
||||
const rows = toUpsert(mapSleep(s), "sleep");
|
||||
ups.push(...rows);
|
||||
// Record-scoped sweep entry: every SLEEP_DURATION row of this
|
||||
// record keys under `sleep:<id>:` (hypnogram runs AND the
|
||||
// stage-total fallback), so the prefix bounds the cleanup to this
|
||||
// one record. Clears whatever a revised hypnogram orphaned —
|
||||
// shifted run boundaries, legacy run-indexed `seg:<i>` rows, and a
|
||||
// stale stage-total set once the hypnogram appears.
|
||||
sleepSweeps.push({
|
||||
prefix: `sleep:${s.id}:`,
|
||||
keepIds: rows
|
||||
.filter((r) => r.type === "SLEEP_DURATION")
|
||||
.map((r) => r.externalId),
|
||||
});
|
||||
}
|
||||
return ups;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "activity",
|
||||
@@ -256,14 +290,15 @@ async function fetchAll(
|
||||
|
||||
const first = await attempt(accessToken, collections);
|
||||
if (!first.auth401) {
|
||||
return { readings: first.readings, failures: first.failures };
|
||||
return { readings: first.readings, failures: first.failures, sleepSweeps };
|
||||
}
|
||||
|
||||
// Reactive refresh: the access token expired (a 401 on the shared token).
|
||||
// Refresh once (rotating both tokens) and retry ONLY the collections that
|
||||
// failed. A failed refresh throws so the caller parks reauth_required.
|
||||
const creds = await getOuraClientCredentials(userId);
|
||||
if (!creds) return { readings: first.readings, failures: first.failures };
|
||||
if (!creds)
|
||||
return { readings: first.readings, failures: first.failures, sleepSweeps };
|
||||
|
||||
const rotated = await refreshAccessToken(refreshToken, creds);
|
||||
// Compare-and-swap persist: on a lost race against a concurrent sync this
|
||||
@@ -276,12 +311,13 @@ async function fetchAll(
|
||||
refreshTokenCiphertext,
|
||||
);
|
||||
if (!usableToken)
|
||||
return { readings: first.readings, failures: first.failures };
|
||||
return { readings: first.readings, failures: first.failures, sleepSweeps };
|
||||
|
||||
const retry = await attempt(usableToken, first.failed);
|
||||
return {
|
||||
readings: [...first.readings, ...retry.readings],
|
||||
failures: retry.failures,
|
||||
sleepSweeps,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -322,6 +358,13 @@ export async function syncUserOura(
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Clear whatever an earlier scoring left under the re-fetched sleep records
|
||||
// before the fresh set upserts (mirrors Google Health's replace-by-window
|
||||
// order). Best-effort inside the helper — a sweep failure never fails the
|
||||
// sync; the record stays inside the lookback window, so the next tick
|
||||
// retries it.
|
||||
await sweepStaleSleepSegments(userId, "OURA", result.sleepSweeps);
|
||||
|
||||
// Import everything the healthy collections returned regardless of whether a
|
||||
// sibling collection failed — one bad collection must not blank the source.
|
||||
const imported = await upsertOuraMeasurements(userId, result.readings);
|
||||
@@ -393,6 +436,10 @@ export async function upsertOuraMeasurements(
|
||||
unit: r.unit,
|
||||
measuredAt: r.measuredAt,
|
||||
sleepStage: r.sleepStage ?? null,
|
||||
// No-op on a live row, a deliberate RESURRECTION on a tombstoned
|
||||
// one — Oura is the source of truth for its own rows, so a
|
||||
// re-fetched reading brings the row back (mirrors Google / Fitbit).
|
||||
deletedAt: null,
|
||||
syncVersion: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -250,10 +250,13 @@ describe("mapSleep", () => {
|
||||
);
|
||||
expect(instants.size).toBe(3);
|
||||
|
||||
// Reconstructed flag + indexed externalId keep the segment rows distinct.
|
||||
// Reconstructed flag + stage-tagged externalId keep the segment rows
|
||||
// distinct. No positional index: an index renumbered on a re-score when a
|
||||
// stage's duration flipped 0↔positive, minting fresh ids that inserted a
|
||||
// duplicate night.
|
||||
expect(core.reconstructed).toBe(true);
|
||||
expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core:0");
|
||||
expect(rem.externalId).toBe("sleep:2026-06-10:seg:sleep_rem:2");
|
||||
expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core");
|
||||
expect(rem.externalId).toBe("sleep:2026-06-10:seg:sleep_rem");
|
||||
|
||||
// IN_BED envelope + score stamp at the sleep END instant.
|
||||
const endMs = Date.parse("2026-06-10T07:00:00+02:00");
|
||||
@@ -287,15 +290,45 @@ describe("mapSleep", () => {
|
||||
expect(awake).toBeDefined();
|
||||
expect(awake.value).toBe(15);
|
||||
expect(awake.reconstructed).toBe(true);
|
||||
expect(awake.externalId).toBe("sleep:2026-06-10:seg:sleep_awake:0");
|
||||
// AWAKE ends at +15m; CORE then starts there and is index 1.
|
||||
expect(awake.externalId).toBe("sleep:2026-06-10:seg:sleep_awake");
|
||||
// AWAKE ends at +15m; CORE then starts there.
|
||||
expect(awake.measuredAt.getTime()).toBe(onset + 15 * 60_000);
|
||||
|
||||
const core = mapped.find((m) => m.sleepStage === "CORE")!;
|
||||
expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core:1");
|
||||
expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core");
|
||||
expect(core.measuredAt.getTime()).toBe(onset + (15 + 60) * 60_000);
|
||||
});
|
||||
|
||||
it("keeps IDENTICAL externalIds across a re-score that adds an interruption block", () => {
|
||||
// The same night scored twice: first with no interruption time, then with
|
||||
// a positive AWAKE block. Under the retired positional index the AWAKE
|
||||
// 0↔positive flip renumbered CORE/DEEP/REM (0,1,2 → 1,2,3), minting
|
||||
// fresh externalIds the upsert then INSERTED next to the old rows — the
|
||||
// night double-counted. The stage-tagged ids must be identical.
|
||||
const night = {
|
||||
date: "2026-06-10",
|
||||
sleep_start_time: "2026-06-09T23:00:00+02:00",
|
||||
sleep_end_time: "2026-06-10T07:00:00+02:00",
|
||||
light_sleep: 3600,
|
||||
deep_sleep: 1800,
|
||||
rem_sleep: 5400,
|
||||
};
|
||||
const idsByStage = (rows: ReturnType<typeof mapSleep>) =>
|
||||
new Map(
|
||||
rows
|
||||
.filter((m) => m.reconstructed)
|
||||
.map((m) => [m.sleepStage, m.externalId]),
|
||||
);
|
||||
const first = idsByStage(mapSleep(night));
|
||||
const rescored = idsByStage(
|
||||
mapSleep({ ...night, total_interruption_duration: 900 }),
|
||||
);
|
||||
for (const stage of ["CORE", "DEEP", "REM"] as const) {
|
||||
expect(rescored.get(stage)).toBe(first.get(stage));
|
||||
}
|
||||
expect(rescored.get("AWAKE")).toBe("sleep:2026-06-10:seg:sleep_awake");
|
||||
});
|
||||
|
||||
it("omits AWAKE when no interruption time is reported", () => {
|
||||
const mapped = mapSleep({
|
||||
date: "2026-06-10",
|
||||
@@ -306,9 +339,9 @@ describe("mapSleep", () => {
|
||||
rem_sleep: 5400,
|
||||
});
|
||||
expect(mapped.find((m) => m.sleepStage === "AWAKE")).toBeUndefined();
|
||||
// CORE stays index 0 when AWAKE is absent.
|
||||
// CORE keeps the same stage-tagged id whether AWAKE is present or not.
|
||||
expect(mapped.find((m) => m.sleepStage === "CORE")!.externalId).toBe(
|
||||
"sleep:2026-06-10:seg:sleep_core:0",
|
||||
"sleep:2026-06-10:seg:sleep_core",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ const {
|
||||
fetchCardioLoadsMock,
|
||||
fetchSpo2Mock,
|
||||
upsertMock,
|
||||
updateManyMock,
|
||||
recordSuccessMock,
|
||||
recordFailureMock,
|
||||
recomputeMock,
|
||||
@@ -20,6 +21,7 @@ const {
|
||||
fetchCardioLoadsMock: vi.fn(),
|
||||
fetchSpo2Mock: vi.fn(),
|
||||
upsertMock: vi.fn(),
|
||||
updateManyMock: vi.fn(),
|
||||
recordSuccessMock: vi.fn(),
|
||||
recordFailureMock: vi.fn(),
|
||||
recomputeMock: vi.fn(),
|
||||
@@ -31,7 +33,7 @@ vi.mock("../credentials", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: { measurement: { upsert: upsertMock } },
|
||||
prisma: { measurement: { upsert: upsertMock, updateMany: updateManyMock } },
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/integrations/status", async (importOriginal) => ({
|
||||
@@ -75,6 +77,7 @@ beforeEach(() => {
|
||||
fetchCardioLoadsMock.mockReset();
|
||||
fetchSpo2Mock.mockReset();
|
||||
upsertMock.mockReset().mockResolvedValue({});
|
||||
updateManyMock.mockReset().mockResolvedValue({ count: 0 });
|
||||
recordSuccessMock.mockReset().mockResolvedValue(undefined);
|
||||
recordFailureMock.mockReset().mockResolvedValue(undefined);
|
||||
recomputeMock.mockReset().mockResolvedValue(undefined);
|
||||
@@ -131,7 +134,7 @@ describe("syncUserPolar", () => {
|
||||
expect(arg.create.value).toBe(123.45);
|
||||
});
|
||||
|
||||
it("keeps reconstructed sleep segments distinct under their indexed externalId", async () => {
|
||||
it("keeps reconstructed sleep segments distinct under their stage-tagged externalId", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchSleepsMock.mockResolvedValue([
|
||||
{
|
||||
@@ -147,12 +150,77 @@ describe("syncUserPolar", () => {
|
||||
const externalIds = upsertMock.mock.calls.map(
|
||||
(c) => c[0].where.userId_type_source_externalId.externalId,
|
||||
);
|
||||
expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_core:0");
|
||||
expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_rem:2");
|
||||
expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_core");
|
||||
expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_rem");
|
||||
// The several segment rows stay distinct (no collision).
|
||||
expect(new Set(externalIds).size).toBe(externalIds.length);
|
||||
});
|
||||
|
||||
it("sweeps stale segment rows per fetched night: live-only, seg-prefixed, notIn the fresh set, soft-delete (v1.28.25)", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchSleepsMock.mockResolvedValue([
|
||||
{
|
||||
date: "2026-06-10",
|
||||
sleep_start_time: "2026-06-09T23:00:00+02:00",
|
||||
sleep_end_time: "2026-06-10T07:00:00+02:00",
|
||||
light_sleep: 3600,
|
||||
deep_sleep: 1800,
|
||||
rem_sleep: 5400,
|
||||
},
|
||||
]);
|
||||
|
||||
await syncUserPolar("u1");
|
||||
|
||||
expect(updateManyMock).toHaveBeenCalledTimes(1);
|
||||
const arg = updateManyMock.mock.calls[0]![0] as {
|
||||
where: {
|
||||
userId: string;
|
||||
source: string;
|
||||
type: string;
|
||||
deletedAt: null;
|
||||
externalId: { startsWith: string; notIn: string[] };
|
||||
};
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(arg.where.userId).toBe("u1");
|
||||
expect(arg.where.source).toBe("POLAR");
|
||||
expect(arg.where.type).toBe("SLEEP_DURATION");
|
||||
expect(arg.where.deletedAt).toBeNull();
|
||||
// Bounded to THIS night's reconstructed segments. The prefix stays on
|
||||
// `:seg:` (not the whole `sleep:<date>:` slice) because the IN_BED
|
||||
// envelope keys on its measuredAt's UTC date, which can drift a calendar
|
||||
// day from `date` — a broader bound could cross nights.
|
||||
expect(arg.where.externalId.startsWith).toBe("sleep:2026-06-10:seg:");
|
||||
// Every fresh SLEEP_DURATION id is protected (segments + IN_BED).
|
||||
expect(arg.where.externalId.notIn.sort()).toEqual([
|
||||
"sleep:2026-06-10:seg:sleep_core",
|
||||
"sleep:2026-06-10:seg:sleep_deep",
|
||||
"sleep:2026-06-10:seg:sleep_rem",
|
||||
"sleep:2026-06-10:sleep_in_bed",
|
||||
]);
|
||||
// Soft delete only.
|
||||
expect(arg.data).toEqual({ deletedAt: expect.any(Date) });
|
||||
|
||||
// A legacy indexed row for this night falls inside the sweep.
|
||||
const legacyId = "sleep:2026-06-10:seg:sleep_core:0";
|
||||
expect(legacyId.startsWith(arg.where.externalId.startsWith)).toBe(true);
|
||||
expect(arg.where.externalId.notIn).not.toContain(legacyId);
|
||||
|
||||
// Replace-then-write: the sweep runs before the first sleep upsert.
|
||||
expect(updateManyMock.mock.invocationCallOrder[0]!).toBeLessThan(
|
||||
upsertMock.mock.invocationCallOrder[0]!,
|
||||
);
|
||||
});
|
||||
|
||||
it("never sweeps when the fetch returns no sleep records", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchRechargesMock.mockResolvedValue([
|
||||
{ date: "2026-06-10", nightly_recharge_status: 3 },
|
||||
]);
|
||||
await syncUserPolar("u1");
|
||||
expect(updateManyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("is idempotent — a re-sync upserts on the same externalId", async () => {
|
||||
getConnMock.mockResolvedValue(CONN);
|
||||
fetchRechargesMock.mockResolvedValue([
|
||||
|
||||
@@ -467,7 +467,7 @@ export function mapNightlyRecharge(
|
||||
* emitting one timed row per segment ending at the running cursor, and stamp the
|
||||
* `IN_BED` envelope + the score row at `sleep_end_time`. The ORDER is synthetic,
|
||||
* so every reconstructed segment is flagged `reconstructed: true` and keyed by
|
||||
* an indexed externalId so the several rows of one night stay distinct.
|
||||
* a stage-tagged externalId so the several rows of one night stay distinct.
|
||||
*
|
||||
* If the night carries no usable start/end (older records, missing fields) we
|
||||
* fall back to a midnight-UTC anchor and emit untimed stage rows — degraded but
|
||||
@@ -517,10 +517,12 @@ export function mapSleep(s: PolarSleep): MappedMeasurement[] {
|
||||
measuredAt: new Date(endMs),
|
||||
fieldTag: "sleep_in_bed",
|
||||
},
|
||||
// Indexed externalId keeps the several segment rows of one night
|
||||
// distinct under userId_type_source_externalId.
|
||||
externalIdFor: (fieldTag, index) =>
|
||||
`sleep:${s.date}:seg:${fieldTag}:${index}`,
|
||||
// Stage-tagged externalId keeps the several segment rows of one night
|
||||
// distinct under userId_type_source_externalId (one segment per stage,
|
||||
// no positional index — an index renumbered on a re-score when a
|
||||
// stage's duration flipped 0↔positive, minting fresh ids that
|
||||
// double-counted the night).
|
||||
externalIdFor: (fieldTag) => `sleep:${s.date}:seg:${fieldTag}`,
|
||||
}),
|
||||
);
|
||||
|
||||
|
||||
+29
-1
@@ -48,6 +48,10 @@ import {
|
||||
mapSpo2,
|
||||
type MappedMeasurement,
|
||||
} from "./client";
|
||||
import {
|
||||
sweepStaleSleepSegments,
|
||||
type SleepSegmentSweep,
|
||||
} from "@/lib/sleep/sweep-stale-segments";
|
||||
import { getPolarConnection } from "./credentials";
|
||||
import { PolarApiError, classifyPolarError } from "./response-classifier";
|
||||
|
||||
@@ -98,6 +102,7 @@ export async function syncUserPolar(userId: string): Promise<number> {
|
||||
if (!conn) return 0;
|
||||
|
||||
const readings: PolarMeasurementUpsert[] = [];
|
||||
const sleepSweeps: SleepSegmentSweep[] = [];
|
||||
try {
|
||||
const [recharges, sleeps, activities, cardioLoads, spo2Tests] =
|
||||
await Promise.all([
|
||||
@@ -111,7 +116,21 @@ export async function syncUserPolar(userId: string): Promise<number> {
|
||||
readings.push(...toUpsert(mapNightlyRecharge(r), "recharge"));
|
||||
}
|
||||
for (const s of sleeps) {
|
||||
readings.push(...toUpsert(mapSleep(s), "sleep"));
|
||||
const rows = toUpsert(mapSleep(s), "sleep");
|
||||
readings.push(...rows);
|
||||
// Night-scoped sweep entry: the reconstructed segments of this date all
|
||||
// key under `sleep:<date>:seg:` (mapper-supplied). Any live row under
|
||||
// that prefix this fetch did NOT re-produce is a re-score orphan or a
|
||||
// legacy `:seg:<tag>:<i>` indexed row — tombstoned before the upsert.
|
||||
// The prefix deliberately stays on `:seg:` — the IN_BED envelope keys
|
||||
// on its measuredAt's UTC date, which can drift a calendar day from
|
||||
// `s.date`, so a broader `sleep:<date>:` bound could cross nights.
|
||||
sleepSweeps.push({
|
||||
prefix: `sleep:${s.date}:seg:`,
|
||||
keepIds: rows
|
||||
.filter((r) => r.type === "SLEEP_DURATION")
|
||||
.map((r) => r.externalId),
|
||||
});
|
||||
}
|
||||
for (const a of activities) {
|
||||
readings.push(...toUpsert(mapActivity(a), "activity"));
|
||||
@@ -136,6 +155,11 @@ export async function syncUserPolar(userId: string): Promise<number> {
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Clear whatever an earlier scoring left under the re-fetched nights before
|
||||
// the fresh set upserts (mirrors Google Health's replace-by-window order).
|
||||
// Best-effort inside the helper — a sweep failure never fails the sync.
|
||||
await sweepStaleSleepSegments(userId, "POLAR", sleepSweeps);
|
||||
|
||||
const imported = await upsertPolarMeasurements(userId, readings);
|
||||
await recordSyncSuccess(userId, "polar");
|
||||
return imported;
|
||||
@@ -183,6 +207,10 @@ export async function upsertPolarMeasurements(
|
||||
unit: r.unit,
|
||||
measuredAt: r.measuredAt,
|
||||
sleepStage: r.sleepStage ?? null,
|
||||
// No-op on a live row, a deliberate RESURRECTION on a tombstoned
|
||||
// one — Polar is the source of truth for its own rows, so a
|
||||
// re-fetched reading brings the row back (mirrors Google / Fitbit).
|
||||
deletedAt: null,
|
||||
syncVersion: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -14,7 +14,7 @@ describe("reconstructContiguousSleepTimeline", () => {
|
||||
{ durationMs: 30 * 60_000, stage: "DEEP", fieldTag: "sleep_deep" },
|
||||
{ durationMs: 90 * 60_000, stage: "REM", fieldTag: "sleep_rem" },
|
||||
],
|
||||
externalIdFor: (tag, i) => `night:seg:${tag}:${i}`,
|
||||
externalIdFor: (tag) => `night:seg:${tag}`,
|
||||
});
|
||||
const core = rows.find((r) => r.sleepStage === "CORE")!;
|
||||
const deep = rows.find((r) => r.sleepStage === "DEEP")!;
|
||||
@@ -27,18 +27,18 @@ describe("reconstructContiguousSleepTimeline", () => {
|
||||
expect(new Set(rows.map((r) => r.measuredAt.getTime())).size).toBe(3);
|
||||
});
|
||||
|
||||
it("flags every laid segment reconstructed and keys an indexed externalId", () => {
|
||||
it("flags every laid segment reconstructed and keys a stage-tagged externalId", () => {
|
||||
const rows = reconstructContiguousSleepTimeline({
|
||||
startMs: onset,
|
||||
stages: [
|
||||
{ durationMs: 15 * 60_000, stage: "AWAKE", fieldTag: "sleep_awake" },
|
||||
{ durationMs: 60 * 60_000, stage: "CORE", fieldTag: "sleep_core" },
|
||||
],
|
||||
externalIdFor: (tag, i) => `night:seg:${tag}:${i}`,
|
||||
externalIdFor: (tag) => `night:seg:${tag}`,
|
||||
});
|
||||
expect(rows.every((r) => r.reconstructed === true)).toBe(true);
|
||||
expect(rows[0].externalId).toBe("night:seg:sleep_awake:0");
|
||||
expect(rows[1].externalId).toBe("night:seg:sleep_core:1");
|
||||
expect(rows[0].externalId).toBe("night:seg:sleep_awake");
|
||||
expect(rows[1].externalId).toBe("night:seg:sleep_core");
|
||||
});
|
||||
|
||||
it("skips non-positive, null, undefined and non-finite durations", () => {
|
||||
@@ -51,14 +51,48 @@ describe("reconstructContiguousSleepTimeline", () => {
|
||||
{ durationMs: Number.NaN, stage: "REM", fieldTag: "sleep_rem" },
|
||||
{ durationMs: 30 * 60_000, stage: "CORE", fieldTag: "sleep_core" },
|
||||
],
|
||||
externalIdFor: (tag, i) => `night:seg:${tag}:${i}`,
|
||||
externalIdFor: (tag) => `night:seg:${tag}`,
|
||||
});
|
||||
expect(rows).toHaveLength(1);
|
||||
// The only laid stage keeps index 0 (skipped stages do not consume indices).
|
||||
expect(rows[0].externalId).toBe("night:seg:sleep_core:0");
|
||||
// The laid stage keys on its tag alone — skipped stages cannot influence
|
||||
// the id of any other stage (no positional index to renumber).
|
||||
expect(rows[0].externalId).toBe("night:seg:sleep_core");
|
||||
expect(rows[0].measuredAt.getTime()).toBe(onset + 30 * 60_000);
|
||||
});
|
||||
|
||||
it("keeps stable externalIds when a re-score flips a stage 0↔positive", () => {
|
||||
// First scoring: no awake time. Re-score: WHOOP/Polar now report a
|
||||
// positive AWAKE block. Under the retired positional index the re-score
|
||||
// renumbered CORE/DEEP (0→1, 1→2), minting fresh externalIds the upsert
|
||||
// then INSERTED next to the old rows — the night double-counted. The
|
||||
// stage-tagged key must be identical across both scorings.
|
||||
const stages = (awakeMs: number) =>
|
||||
[
|
||||
{ durationMs: awakeMs, stage: "AWAKE", fieldTag: "sleep_awake" },
|
||||
{ durationMs: 60 * 60_000, stage: "CORE", fieldTag: "sleep_core" },
|
||||
{ durationMs: 30 * 60_000, stage: "DEEP", fieldTag: "sleep_deep" },
|
||||
] as const;
|
||||
const externalIdFor = (tag: string) => `night:seg:${tag}`;
|
||||
|
||||
const first = reconstructContiguousSleepTimeline({
|
||||
startMs: onset,
|
||||
stages: stages(0),
|
||||
externalIdFor,
|
||||
});
|
||||
const rescored = reconstructContiguousSleepTimeline({
|
||||
startMs: onset,
|
||||
stages: stages(15 * 60_000),
|
||||
externalIdFor,
|
||||
});
|
||||
|
||||
const idsOf = (rows: typeof first, stage: string) =>
|
||||
rows.find((r) => r.sleepStage === stage)?.externalId;
|
||||
// CORE and DEEP keep their ids — the awake flip cannot renumber them.
|
||||
expect(idsOf(rescored, "CORE")).toBe(idsOf(first, "CORE"));
|
||||
expect(idsOf(rescored, "DEEP")).toBe(idsOf(first, "DEEP"));
|
||||
expect(idsOf(rescored, "AWAKE")).toBe("night:seg:sleep_awake");
|
||||
});
|
||||
|
||||
it("appends a single IN_BED envelope at the given END instant", () => {
|
||||
const rows = reconstructContiguousSleepTimeline({
|
||||
startMs: onset,
|
||||
@@ -70,7 +104,7 @@ describe("reconstructContiguousSleepTimeline", () => {
|
||||
measuredAt: new Date(end),
|
||||
fieldTag: "sleep_in_bed",
|
||||
},
|
||||
externalIdFor: (tag, i) => `night:seg:${tag}:${i}`,
|
||||
externalIdFor: (tag) => `night:seg:${tag}`,
|
||||
});
|
||||
const inBed = rows.find((r) => r.sleepStage === "IN_BED")!;
|
||||
expect(inBed.measuredAt.getTime()).toBe(end);
|
||||
@@ -91,7 +125,7 @@ describe("reconstructContiguousSleepTimeline", () => {
|
||||
measuredAt: new Date(end),
|
||||
fieldTag: "sleep_in_bed",
|
||||
},
|
||||
externalIdFor: (tag, i) => `night:seg:${tag}:${i}`,
|
||||
externalIdFor: (tag) => `night:seg:${tag}`,
|
||||
});
|
||||
expect(rows.find((r) => r.sleepStage === "IN_BED")).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* Pins the safety contract of `sweepStaleSleepSegments` — the session-scoped
|
||||
* cleanup that keeps a re-scored wearable night from double-counting (the
|
||||
* prefix-bound sibling of Google Health's `replaceStaleGoogleHealthSleep`,
|
||||
* whose test this mirrors). The query MUST be tightly bounded: only LIVE rows,
|
||||
* only `SLEEP_DURATION`, only ONE source, only under ONE session's externalId
|
||||
* prefix, and NEVER a row in the fresh keep-set. An entry with no fresh ids
|
||||
* (or no prefix) is skipped entirely — an unbounded delete would be data loss.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { updateManyMock } = vi.hoisted(() => ({
|
||||
updateManyMock: vi.fn(async () => ({ count: 0 })),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: { measurement: { updateMany: updateManyMock } },
|
||||
}));
|
||||
|
||||
import { sweepStaleSleepSegments } from "../sweep-stale-segments";
|
||||
|
||||
beforeEach(() => {
|
||||
updateManyMock.mockClear();
|
||||
updateManyMock.mockResolvedValue({ count: 2 });
|
||||
});
|
||||
|
||||
describe("sweepStaleSleepSegments — bounded session-scoped sweep", () => {
|
||||
it("soft-deletes only live SLEEP_DURATION rows under the session prefix, excluding the fresh set", async () => {
|
||||
const keepIds = ["night-1:seg:sleep_core", "night-1:seg:sleep_deep"];
|
||||
|
||||
const removed = await sweepStaleSleepSegments("user-1", "WHOOP", [
|
||||
{ prefix: "night-1:", keepIds },
|
||||
]);
|
||||
|
||||
expect(removed).toBe(2);
|
||||
expect(updateManyMock).toHaveBeenCalledTimes(1);
|
||||
const arg = (updateManyMock.mock.calls[0]! as unknown[])[0] as {
|
||||
where: Record<string, unknown>;
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(arg.where).toEqual({
|
||||
userId: "user-1",
|
||||
source: "WHOOP",
|
||||
type: "SLEEP_DURATION",
|
||||
deletedAt: null,
|
||||
externalId: { startsWith: "night-1:", notIn: keepIds },
|
||||
});
|
||||
// A soft delete — the row is tombstoned, never hard-removed.
|
||||
expect(arg.data).toEqual({ deletedAt: expect.any(Date) });
|
||||
});
|
||||
|
||||
it("sweeps a legacy indexed row: inside the prefix, outside the keep-set", async () => {
|
||||
// The legacy volatile formats (`night-1:seg:sleep_core:1`, Withings'
|
||||
// running index, Oura's `seg:<i>`) all live under the session prefix and
|
||||
// can never appear in a fresh keep-set — so the where-clause the sweep
|
||||
// issues matches them by construction. Pin that with a literal check
|
||||
// against the emitted filter.
|
||||
await sweepStaleSleepSegments("user-1", "WHOOP", [
|
||||
{ prefix: "night-1:", keepIds: ["night-1:seg:sleep_core"] },
|
||||
]);
|
||||
const where = (
|
||||
(updateManyMock.mock.calls[0]! as unknown[])[0] as {
|
||||
where: { externalId: { startsWith: string; notIn: string[] } };
|
||||
}
|
||||
).where;
|
||||
const legacyId = "night-1:seg:sleep_core:1";
|
||||
expect(legacyId.startsWith(where.externalId.startsWith)).toBe(true);
|
||||
expect(where.externalId.notIn).not.toContain(legacyId);
|
||||
// …while the fresh row is protected.
|
||||
expect(where.externalId.notIn).toContain("night-1:seg:sleep_core");
|
||||
});
|
||||
|
||||
it("skips an entry with no fresh ids or no prefix (never an unbounded delete)", async () => {
|
||||
await sweepStaleSleepSegments("user-1", "OURA", [
|
||||
{ prefix: "sleep:rec-1:", keepIds: [] },
|
||||
{ prefix: "", keepIds: ["sleep:rec-2:seg:x"] },
|
||||
]);
|
||||
expect(updateManyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("is bounded to the sessions of THIS fetch — one updateMany per entry, each under its own prefix", async () => {
|
||||
await sweepStaleSleepSegments("user-1", "POLAR", [
|
||||
{ prefix: "sleep:2026-06-10:seg:", keepIds: ["a"] },
|
||||
{ prefix: "sleep:2026-06-11:seg:", keepIds: ["b"] },
|
||||
]);
|
||||
expect(updateManyMock).toHaveBeenCalledTimes(2);
|
||||
const prefixes = updateManyMock.mock.calls.map(
|
||||
(c) =>
|
||||
(
|
||||
(c as unknown[])[0] as {
|
||||
where: { externalId: { startsWith: string } };
|
||||
}
|
||||
).where.externalId.startsWith,
|
||||
);
|
||||
expect(prefixes).toEqual([
|
||||
"sleep:2026-06-10:seg:",
|
||||
"sleep:2026-06-11:seg:",
|
||||
]);
|
||||
});
|
||||
|
||||
it("never throws when the cleanup query fails (best-effort)", async () => {
|
||||
updateManyMock.mockRejectedValueOnce(new Error("db down"));
|
||||
await expect(
|
||||
sweepStaleSleepSegments("user-1", "WITHINGS", [
|
||||
{ prefix: "withings:sleep:u:1:", keepIds: ["withings:sleep:u:1:5"] },
|
||||
]),
|
||||
).resolves.toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -63,11 +63,16 @@ export interface ReconstructTimelineOptions {
|
||||
/** Optional IN_BED envelope row. */
|
||||
inBed?: ReconstructedInBed;
|
||||
/**
|
||||
* Builds the indexed externalId for a laid segment so the several rows of one
|
||||
* night stay distinct under `userId_type_source_externalId`. Called with the
|
||||
* segment's fieldTag and its running index.
|
||||
* Builds the externalId for a laid segment so the several rows of one night
|
||||
* stay distinct under `userId_type_source_externalId`. Called with the
|
||||
* segment's fieldTag ONLY — the builder emits at most one segment per stage,
|
||||
* so the tag alone is unique within a night. A positional index was
|
||||
* deliberately dropped from the key: it skipped zero-duration stages, so a
|
||||
* 0↔positive flip on a vendor re-score renumbered every following stage,
|
||||
* minted fresh externalIds, and the upsert double-counted the night instead
|
||||
* of overwriting (the Google Health v1.28.18 bug class).
|
||||
*/
|
||||
externalIdFor: (fieldTag: string, index: number) => string;
|
||||
externalIdFor: (fieldTag: string) => string;
|
||||
}
|
||||
|
||||
const MS_TO_MIN = 1 / 60_000;
|
||||
@@ -79,8 +84,9 @@ function round2(n: number): number {
|
||||
/**
|
||||
* Lay the asleep/awake stages contiguously from `startMs` in the given order,
|
||||
* emitting one timed `SLEEP_DURATION` row per stage (`measuredAt` = that
|
||||
* segment's END), each flagged `reconstructed: true` and keyed by an indexed
|
||||
* externalId. Appends a single IN_BED envelope row when `inBed` is provided.
|
||||
* segment's END), each flagged `reconstructed: true` and keyed by a
|
||||
* stage-tagged externalId. Appends a single IN_BED envelope row when `inBed`
|
||||
* is provided.
|
||||
*/
|
||||
export function reconstructContiguousSleepTimeline(
|
||||
opts: ReconstructTimelineOptions,
|
||||
@@ -89,7 +95,6 @@ export function reconstructContiguousSleepTimeline(
|
||||
const out: ReconstructedSleepRow[] = [];
|
||||
|
||||
let cursor = startMs;
|
||||
let segIndex = 0;
|
||||
for (const { durationMs, stage, fieldTag } of stages) {
|
||||
if (
|
||||
typeof durationMs !== "number" ||
|
||||
@@ -105,12 +110,11 @@ export function reconstructContiguousSleepTimeline(
|
||||
unit: "minutes",
|
||||
measuredAt: new Date(segEnd),
|
||||
fieldTag,
|
||||
externalId: externalIdFor(fieldTag, segIndex),
|
||||
externalId: externalIdFor(fieldTag),
|
||||
sleepStage: stage,
|
||||
reconstructed: true,
|
||||
});
|
||||
cursor = segEnd;
|
||||
segIndex += 1;
|
||||
}
|
||||
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Session-scoped sweep of stale sleep-segment rows.
|
||||
*
|
||||
* Wearables re-score a night after the fact (WHOOP re-scores, Withings
|
||||
* re-aggregates, Oura revises the hypnogram, Polar re-settles the stage
|
||||
* totals). Before the stable-externalId fix, a re-scored night minted FRESH
|
||||
* segment externalIds (positional indexes renumbered, boundaries shifted), so
|
||||
* the upsert inserted a second set of rows next to the first and the
|
||||
* night-total silently double-counted. The externalIds are stable now, but two
|
||||
* classes of orphans remain: rows a re-score genuinely dropped (a segment that
|
||||
* no longer exists in the fresh scoring) and every legacy row still keyed on
|
||||
* the old volatile format. This sweep clears both, so the id change self-heals
|
||||
* without a migration — the same posture as Google Health's
|
||||
* `replaceStaleGoogleHealthSleep`, expressed as an externalId-prefix bound
|
||||
* instead of a time-window bound because these integrations key every segment
|
||||
* row under a per-session externalId prefix.
|
||||
*
|
||||
* Safety contract (pinned by `__tests__/sweep-stale-segments.test.ts`):
|
||||
* - Bounded to the sessions of THIS fetch — each entry sweeps only rows whose
|
||||
* `externalId` starts with that one session's prefix. Never a blanket
|
||||
* delete.
|
||||
* - LIVE rows only (`deletedAt: null`), `SLEEP_DURATION` only, one source.
|
||||
* - The fresh id set (`keepIds`) is never touched; an entry with no fresh ids
|
||||
* is skipped entirely (a mapper hiccup must not wipe a night).
|
||||
* - Soft-delete (`deletedAt = now`), never a hard remove.
|
||||
* - Best-effort: a failed sweep warns and moves on — it never fails the
|
||||
* user's sync. The session stays inside the integration's re-fetch window,
|
||||
* so the next tick retries the sweep.
|
||||
*/
|
||||
import type { MeasurementSource } from "@/generated/prisma/client";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
/** One just-fetched sleep session: its externalId prefix + the fresh ids. */
|
||||
export interface SleepSegmentSweep {
|
||||
/**
|
||||
* The session-scoped externalId prefix (e.g. `<sleep-uuid>:` for WHOOP,
|
||||
* `withings:sleep:<user>:<sessionId>:` for Withings). Every live
|
||||
* `SLEEP_DURATION` row under it that is not in `keepIds` is considered a
|
||||
* re-score orphan or a legacy-format duplicate.
|
||||
*/
|
||||
prefix: string;
|
||||
/** The fresh externalIds this fetch produced for the session — never swept. */
|
||||
keepIds: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft-delete every live `SLEEP_DURATION` row of `source` whose externalId
|
||||
* falls under one of the given session prefixes but was not re-produced by
|
||||
* this fetch. Returns the number of rows tombstoned.
|
||||
*/
|
||||
export async function sweepStaleSleepSegments(
|
||||
userId: string,
|
||||
source: MeasurementSource,
|
||||
sessions: SleepSegmentSweep[],
|
||||
): Promise<number> {
|
||||
let removed = 0;
|
||||
for (const s of sessions) {
|
||||
// An empty prefix would unbound the sweep to the whole source; an empty
|
||||
// keep-set means this fetch produced nothing for the session (unscored /
|
||||
// mapper hiccup) — in both cases deleting would be data loss, so skip.
|
||||
if (!s.prefix || s.keepIds.length === 0) continue;
|
||||
try {
|
||||
const res = await prisma.measurement.updateMany({
|
||||
where: {
|
||||
userId,
|
||||
source,
|
||||
type: "SLEEP_DURATION",
|
||||
deletedAt: null,
|
||||
externalId: { startsWith: s.prefix, notIn: s.keepIds },
|
||||
},
|
||||
data: { deletedAt: new Date() },
|
||||
});
|
||||
removed += res.count;
|
||||
} catch (err) {
|
||||
getEvent()?.addWarning(
|
||||
`${source.toLowerCase()}: stale sleep-segment sweep failed for prefix ${s.prefix}: ${err}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
@@ -245,29 +245,42 @@ export async function buildTargetsResponse(user: AuthedUser) {
|
||||
// Absolute latest measurement per type within a one-year floor.
|
||||
//
|
||||
// Prisma's `distinct` does not compile to Postgres `DISTINCT ON` — the
|
||||
// driver dedups after pulling the rows, so an unbounded scan on a
|
||||
// 347 k-row tenant pulls every measurement back to Node only to drop
|
||||
// all but seven (audit Critical Finding #2, v1.4.40). The 365-day
|
||||
// floor caps the planner-side walk to the existing
|
||||
// `(user_id, type, measured_at)` index range; the trade-off is that a
|
||||
// user who hasn't measured a metric in over a year sees the tile's
|
||||
// `current` value as null instead of pulling a stale year-old reading
|
||||
// forward — which is the correct behaviour for the consistency strip
|
||||
// (the strip already gates on "fewer than 3 readings in 30 days" →
|
||||
// insufficientData).
|
||||
limit(() =>
|
||||
prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type: { in: types },
|
||||
measuredAt: { gte: oneYearAgo },
|
||||
deletedAt: null,
|
||||
},
|
||||
orderBy: { measuredAt: "desc" },
|
||||
distinct: ["type"],
|
||||
select: { type: true, value: true },
|
||||
}),
|
||||
),
|
||||
// driver dedups after pulling the rows, so even with the 365-day floor
|
||||
// a dense-type tenant (per-sample PULSE, hourly HR buckets) shipped a
|
||||
// year of raw rows back to Node only to drop all but seven (audit
|
||||
// Critical Finding #2, v1.4.40 flagged the unbounded variant; the
|
||||
// bounded one stayed O(rows-in-year)). Real `DISTINCT ON` resolves it
|
||||
// index-side: one row per type, ordered walk over the existing
|
||||
// `(user_id, type, measured_at)` index. The 365-day floor keeps its
|
||||
// documented product semantics — a metric not measured in over a year
|
||||
// reads `current: null` instead of pulling a stale reading forward.
|
||||
limit(() => {
|
||||
// Whitelist-splice: `types` is the closed compile-time enum list
|
||||
// above; assert each member against the enum-literal shape before
|
||||
// splicing (same pattern as the rollup aggregates in
|
||||
// `measurement-rollups.ts`). user id + date bind as $1 / $2.
|
||||
for (const t of types) {
|
||||
if (!/^[A-Z0-9_]+$/.test(t)) {
|
||||
throw new Error(`invalid measurement type: ${t}`);
|
||||
}
|
||||
}
|
||||
const typeList = types.map((t) => `'${t}'::"measurement_type"`).join(",");
|
||||
return prisma.$queryRawUnsafe<
|
||||
Array<{ type: MeasurementType; value: number }>
|
||||
>(
|
||||
`SELECT DISTINCT ON (m."type")
|
||||
m."type"::text AS type,
|
||||
m."value" AS value
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = $1
|
||||
AND m."type" IN (${typeList})
|
||||
AND m."measured_at" >= $2
|
||||
AND m."deleted_at" IS NULL
|
||||
ORDER BY m."type" ASC, m."measured_at" DESC`,
|
||||
userId,
|
||||
oneYearAgo,
|
||||
);
|
||||
}),
|
||||
// Sleep stage rows for the per-night reconstruction (section 4).
|
||||
limit(() =>
|
||||
prisma.measurement.findMany({
|
||||
|
||||
@@ -375,14 +375,52 @@ describe("mapSleep", () => {
|
||||
expect(r.reconstructed).toBeUndefined();
|
||||
});
|
||||
|
||||
it("gives each reconstructed segment a distinct indexed externalId", () => {
|
||||
it("gives each reconstructed segment a distinct stage-tagged externalId", () => {
|
||||
const ids = mapSleep(base)
|
||||
.filter((r) => r.reconstructed)
|
||||
.map((r) => r.externalId);
|
||||
expect(ids.every((id) => id?.startsWith("sleep-uuid:seg:"))).toBe(true);
|
||||
// One segment per stage — the tag alone keys the row. No positional index:
|
||||
// an index renumbered on a re-score whenever a stage's duration flipped
|
||||
// 0↔positive, minting fresh externalIds that double-counted the night.
|
||||
expect(ids).toEqual([
|
||||
"sleep-uuid:seg:sleep_awake",
|
||||
"sleep-uuid:seg:sleep_core",
|
||||
"sleep-uuid:seg:sleep_deep",
|
||||
"sleep-uuid:seg:sleep_rem",
|
||||
]);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
});
|
||||
|
||||
it("keeps IDENTICAL externalIds across a re-score that flips a stage 0↔positive", () => {
|
||||
// First scoring: no awake time (the leading AWAKE segment is skipped).
|
||||
// Re-score of the SAME record: WHOOP now reports a positive awake block.
|
||||
// Under the retired indexed key the re-score renumbered CORE/DEEP/REM,
|
||||
// so the upsert inserted a second set instead of overwriting.
|
||||
const first = mapSleep({
|
||||
...base,
|
||||
score: {
|
||||
...base.score!,
|
||||
stage_summary: {
|
||||
...base.score!.stage_summary,
|
||||
total_awake_time_milli: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
const rescored = mapSleep(base);
|
||||
|
||||
const idsByStage = (rows: typeof first) =>
|
||||
new Map(
|
||||
rows
|
||||
.filter((r) => r.reconstructed)
|
||||
.map((r) => [r.sleepStage, r.externalId]),
|
||||
);
|
||||
const a = idsByStage(first);
|
||||
const b = idsByStage(rescored);
|
||||
for (const stage of ["CORE", "DEEP", "REM"] as const) {
|
||||
expect(b.get(stage)).toBe(a.get(stage));
|
||||
}
|
||||
});
|
||||
|
||||
it("sums SLEEP_NEED components ms→min", () => {
|
||||
const need = mapSleep(base).find((r) => r.type === "SLEEP_NEED");
|
||||
expect(need?.value).toBe(490); // 450 + 30 + 10 + 0
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Pins the record-scoped stale-segment sweep in the WHOOP sleep sync
|
||||
* (v1.28.25). A WHOOP re-score used to renumber the reconstructed segments'
|
||||
* indexed externalIds and insert a duplicate night; the ids are stage-tagged
|
||||
* (stable) now, and the sync sweeps whatever an earlier scoring left under a
|
||||
* re-fetched record's prefix — including every legacy `:seg:<tag>:<i>` row.
|
||||
* The sweep must be bounded to the records of THIS fetch, live-only,
|
||||
* SLEEP_DURATION-only, keep-protected, and soft-delete.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { updateManyMock, upsertMeasurementsMock } = vi.hoisted(() => ({
|
||||
updateManyMock: vi.fn(async () => ({ count: 0 })),
|
||||
upsertMeasurementsMock: vi.fn(async () => 1),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: { updateMany: updateManyMock },
|
||||
whoopConnection: {
|
||||
findUnique: vi.fn(async () => ({
|
||||
lastSyncedAt: null,
|
||||
resourceCursors: null,
|
||||
})),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../sync", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../sync")>()),
|
||||
getValidToken: vi.fn(async () => ({ accessToken: "tok" })),
|
||||
markResourceSynced: vi.fn(async () => {}),
|
||||
upsertWhoopMeasurements: upsertMeasurementsMock,
|
||||
}));
|
||||
|
||||
const { fetchSleepsMock } = vi.hoisted(() => ({ fetchSleepsMock: vi.fn() }));
|
||||
vi.mock("../client", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../client")>()),
|
||||
fetchSleeps: fetchSleepsMock,
|
||||
fetchSleepById: fetchSleepsMock,
|
||||
}));
|
||||
|
||||
import type { WhoopSleep } from "../client";
|
||||
import { syncUserSleep } from "../sync-sleep";
|
||||
|
||||
const NIGHT: WhoopSleep = {
|
||||
id: "sleep-uuid",
|
||||
user_id: 42,
|
||||
created_at: "2026-06-01T05:00:00.000Z",
|
||||
updated_at: "2026-06-01T07:00:00.000Z",
|
||||
start: "2026-05-31T23:00:00.000Z",
|
||||
end: "2026-06-01T07:00:00.000Z",
|
||||
nap: false,
|
||||
score_state: "SCORED",
|
||||
score: {
|
||||
stage_summary: {
|
||||
total_in_bed_time_milli: 28_800_000,
|
||||
total_awake_time_milli: 1_800_000,
|
||||
total_light_sleep_time_milli: 14_400_000,
|
||||
total_slow_wave_sleep_time_milli: 5_400_000,
|
||||
total_rem_sleep_time_milli: 7_200_000,
|
||||
},
|
||||
sleep_needed: {
|
||||
baseline_milli: 27_000_000,
|
||||
need_from_sleep_debt_milli: 0,
|
||||
need_from_recent_strain_milli: 0,
|
||||
need_from_recent_nap_milli: 0,
|
||||
},
|
||||
respiratory_rate: 15.2,
|
||||
sleep_performance_percentage: 88,
|
||||
sleep_efficiency_percentage: 93.5,
|
||||
sleep_consistency_percentage: 71,
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
updateManyMock.mockClear().mockResolvedValue({ count: 0 });
|
||||
upsertMeasurementsMock.mockClear().mockResolvedValue(1);
|
||||
fetchSleepsMock.mockReset();
|
||||
});
|
||||
|
||||
describe("syncUserSleep — record-scoped stale-segment sweep", () => {
|
||||
it("sweeps live WHOOP SLEEP_DURATION rows under the record prefix, keeping the fresh set", async () => {
|
||||
fetchSleepsMock.mockResolvedValue([NIGHT]);
|
||||
|
||||
await syncUserSleep("user-1");
|
||||
|
||||
expect(updateManyMock).toHaveBeenCalledTimes(1);
|
||||
const arg = (updateManyMock.mock.calls[0]! as unknown[])[0] as {
|
||||
where: {
|
||||
userId: string;
|
||||
source: string;
|
||||
type: string;
|
||||
deletedAt: null;
|
||||
externalId: { startsWith: string; notIn: string[] };
|
||||
};
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(arg.where.userId).toBe("user-1");
|
||||
expect(arg.where.source).toBe("WHOOP");
|
||||
expect(arg.where.type).toBe("SLEEP_DURATION");
|
||||
expect(arg.where.deletedAt).toBeNull();
|
||||
expect(arg.where.externalId.startsWith).toBe("sleep-uuid:");
|
||||
// The fresh set: the four stage-tagged segments + the IN_BED envelope.
|
||||
expect(arg.where.externalId.notIn.sort()).toEqual([
|
||||
"sleep-uuid:seg:sleep_awake",
|
||||
"sleep-uuid:seg:sleep_core",
|
||||
"sleep-uuid:seg:sleep_deep",
|
||||
"sleep-uuid:seg:sleep_rem",
|
||||
"sleep-uuid:sleep_in_bed",
|
||||
]);
|
||||
// Soft delete only.
|
||||
expect(arg.data).toEqual({ deletedAt: expect.any(Date) });
|
||||
|
||||
// A legacy indexed row for this record falls inside the sweep.
|
||||
const legacyId = "sleep-uuid:seg:sleep_core:1";
|
||||
expect(legacyId.startsWith(arg.where.externalId.startsWith)).toBe(true);
|
||||
expect(arg.where.externalId.notIn).not.toContain(legacyId);
|
||||
});
|
||||
|
||||
it("sweeps BEFORE the fresh set upserts (replace-then-write, google parity)", async () => {
|
||||
fetchSleepsMock.mockResolvedValue([NIGHT]);
|
||||
await syncUserSleep("user-1");
|
||||
const sweepOrder = updateManyMock.mock.invocationCallOrder[0]!;
|
||||
const upsertOrder = upsertMeasurementsMock.mock.invocationCallOrder[0]!;
|
||||
expect(sweepOrder).toBeLessThan(upsertOrder);
|
||||
});
|
||||
|
||||
it("skips the sweep for an unscored record (no fresh ids — deleting would wipe the night)", async () => {
|
||||
fetchSleepsMock.mockResolvedValue([
|
||||
{ ...NIGHT, score_state: "PENDING_SCORE", score: undefined },
|
||||
]);
|
||||
await syncUserSleep("user-1");
|
||||
expect(updateManyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never fails the sync when the sweep query throws", async () => {
|
||||
fetchSleepsMock.mockResolvedValue([NIGHT]);
|
||||
updateManyMock.mockRejectedValueOnce(new Error("db down"));
|
||||
await expect(syncUserSleep("user-1")).resolves.toBe(1);
|
||||
expect(upsertMeasurementsMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -646,10 +646,12 @@ export function mapSleep(s: WhoopSleep): MappedMeasurement[] {
|
||||
measuredAt,
|
||||
fieldTag: "sleep_in_bed",
|
||||
},
|
||||
// The sync layer keys reconstructed rows by this externalId verbatim; the
|
||||
// index keeps the several rows of one night distinct under
|
||||
// userId_type_source_externalId.
|
||||
externalIdFor: (tag, index) => `${s.id}:seg:${tag}:${index}`,
|
||||
// The sync layer keys reconstructed rows by this externalId verbatim;
|
||||
// the stage tag keeps the several rows of one night distinct under
|
||||
// userId_type_source_externalId (one segment per stage, no positional
|
||||
// index — an index renumbered on a re-score when a stage's duration
|
||||
// flipped 0↔positive, minting fresh ids that double-counted the night).
|
||||
externalIdFor: (tag) => `${s.id}:seg:${tag}`,
|
||||
}),
|
||||
);
|
||||
|
||||
|
||||
+44
-10
@@ -9,10 +9,23 @@
|
||||
* timestamps), so `mapSleep` RECONSTRUCTS an ordered, contiguous per-segment
|
||||
* timeline (CORE/DEEP/REM/AWAKE laid back-to-back from sleep onset) and flags
|
||||
* those rows `reconstructed`; each reconstructed segment carries its own
|
||||
* indexed externalId `<sleep_id>:seg:<tag>:<i>`. IN_BED stays a single
|
||||
* envelope row keyed `<sleep_id>:sleep_in_bed`. The non-segment scores keep
|
||||
* the `<sleep_id>:<fieldTag>` shape.
|
||||
* stage-tagged externalId `<sleep_id>:seg:<tag>` — stable across a WHOOP
|
||||
* re-score (the retired positional index renumbered whenever a stage's
|
||||
* duration flipped 0↔positive). IN_BED stays a single envelope row keyed
|
||||
* `<sleep_id>:sleep_in_bed`. The non-segment scores keep the
|
||||
* `<sleep_id>:<fieldTag>` shape.
|
||||
*
|
||||
* Before the upsert, `sweepStaleSleepSegments` tombstones any live
|
||||
* SLEEP_DURATION row under a re-fetched record's `<sleep_id>:` prefix that
|
||||
* this fetch did not re-produce — clearing re-score orphans AND every legacy
|
||||
* `<sleep_id>:seg:<tag>:<i>` row, so the id change self-heals without a
|
||||
* migration (mirrors Google Health's replace-by-window cleanup).
|
||||
*/
|
||||
import {
|
||||
sweepStaleSleepSegments,
|
||||
type SleepSegmentSweep,
|
||||
} from "@/lib/sleep/sweep-stale-segments";
|
||||
|
||||
import { fetchSleeps, fetchSleepById, mapSleep } from "./client";
|
||||
import {
|
||||
getValidToken,
|
||||
@@ -52,22 +65,34 @@ export async function syncUserSleep(
|
||||
}
|
||||
|
||||
const readings: WhoopMeasurementUpsert[] = [];
|
||||
const sweeps: SleepSegmentSweep[] = [];
|
||||
for (const s of records) {
|
||||
const keepIds: string[] = [];
|
||||
for (const m of mapSleep(s)) {
|
||||
// Reconstructed sleep segments carry their own stage-tagged externalId
|
||||
// so the several rows of one night stay distinct; everything else keeps
|
||||
// the `<sleep_id>:<fieldTag>` shape.
|
||||
const externalId = m.externalId ?? `${s.id}:${m.fieldTag}`;
|
||||
if (m.type === "SLEEP_DURATION") keepIds.push(externalId);
|
||||
readings.push({
|
||||
type: m.type,
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
measuredAt: m.measuredAt,
|
||||
// Reconstructed sleep segments carry their own indexed externalId so
|
||||
// the several rows of one stage stay distinct; everything else keeps
|
||||
// the `<sleep_id>:<fieldTag>` shape.
|
||||
externalId: m.externalId ?? `${s.id}:${m.fieldTag}`,
|
||||
externalId,
|
||||
sleepStage: m.sleepStage ?? null,
|
||||
});
|
||||
}
|
||||
// Record-scoped sweep entry — only rows under THIS record's prefix are
|
||||
// candidates, and only when the fetch re-produced the record (an unscored
|
||||
// record maps to nothing and is skipped by the sweep's keep-guard).
|
||||
sweeps.push({ prefix: `${s.id}:`, keepIds });
|
||||
}
|
||||
|
||||
// Clear whatever an earlier scoring left under the re-fetched records before
|
||||
// the fresh set upserts (mirrors Google Health's replace-by-window order).
|
||||
await sweepStaleSleepSegments(userId, "WHOOP", sweeps);
|
||||
|
||||
const imported = await upsertWhoopMeasurements(userId, readings);
|
||||
await markResourceSynced(userId, "sleep");
|
||||
return imported;
|
||||
@@ -95,18 +120,27 @@ export async function syncWhoopSleepById(
|
||||
}
|
||||
|
||||
const readings: WhoopMeasurementUpsert[] = [];
|
||||
const keepIds: string[] = [];
|
||||
for (const m of mapSleep(record)) {
|
||||
// Reconstructed sleep segments carry their own stage-tagged externalId
|
||||
// (see syncUserSleep).
|
||||
const externalId = m.externalId ?? `${record.id}:${m.fieldTag}`;
|
||||
if (m.type === "SLEEP_DURATION") keepIds.push(externalId);
|
||||
readings.push({
|
||||
type: m.type,
|
||||
value: m.value,
|
||||
unit: m.unit,
|
||||
measuredAt: m.measuredAt,
|
||||
// Reconstructed sleep segments carry their own indexed externalId
|
||||
// (see syncUserSleep).
|
||||
externalId: m.externalId ?? `${record.id}:${m.fieldTag}`,
|
||||
externalId,
|
||||
sleepStage: m.sleepStage ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
// A webhook refresh IS a re-score in most cases — sweep the record's stale
|
||||
// rows before the fresh set upserts (see syncUserSleep).
|
||||
await sweepStaleSleepSegments(userId, "WHOOP", [
|
||||
{ prefix: `${record.id}:`, keepIds },
|
||||
]);
|
||||
|
||||
return upsertWhoopMeasurements(userId, readings);
|
||||
}
|
||||
@@ -363,6 +363,10 @@ export async function upsertWhoopMeasurements(
|
||||
unit: r.unit,
|
||||
measuredAt: r.measuredAt,
|
||||
sleepStage: r.sleepStage ?? null,
|
||||
// No-op on a live row, a deliberate RESURRECTION on a tombstoned
|
||||
// one — WHOOP is the source of truth for its own rows, so a
|
||||
// re-fetched reading brings the row back (mirrors Google / Fitbit).
|
||||
deletedAt: null,
|
||||
// Surface the server-side mutation to the iOS LWW reconciler.
|
||||
syncVersion: { increment: 1 },
|
||||
},
|
||||
|
||||
@@ -212,18 +212,26 @@ describe("syncUserActivity — field mapping + idempotency", () => {
|
||||
// overwrite (the value moved). Keyed by the same four columns the
|
||||
// batched read matches on.
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{ id: "row-steps", type: "ACTIVITY_STEPS", measuredAt, value: 1 },
|
||||
{
|
||||
id: "row-steps",
|
||||
type: "ACTIVITY_STEPS",
|
||||
measuredAt,
|
||||
value: 1,
|
||||
deletedAt: null,
|
||||
},
|
||||
{
|
||||
id: "row-distance",
|
||||
type: "WALKING_RUNNING_DISTANCE",
|
||||
measuredAt,
|
||||
value: 1,
|
||||
deletedAt: null,
|
||||
},
|
||||
{
|
||||
id: "row-calories",
|
||||
type: "ACTIVE_ENERGY_BURNED",
|
||||
measuredAt,
|
||||
value: 1,
|
||||
deletedAt: null,
|
||||
},
|
||||
] as never);
|
||||
vi.mocked(prisma.measurement.update).mockResolvedValue({} as never);
|
||||
@@ -241,7 +249,13 @@ describe("syncUserActivity — field mapping + idempotency", () => {
|
||||
installFetchMock([{ date: "2026-05-12", steps: 8420 }]);
|
||||
const measuredAt = new Date("2026-05-12T12:00:00.000Z");
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{ id: "row-steps", type: "ACTIVITY_STEPS", measuredAt, value: 8420 },
|
||||
{
|
||||
id: "row-steps",
|
||||
type: "ACTIVITY_STEPS",
|
||||
measuredAt,
|
||||
value: 8420,
|
||||
deletedAt: null,
|
||||
},
|
||||
] as never);
|
||||
|
||||
const imported = await syncUserActivity("user-1");
|
||||
@@ -254,6 +268,34 @@ describe("syncUserActivity — field mapping + idempotency", () => {
|
||||
expect(prisma.$transaction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resurrects a tombstoned slot even when the value is unchanged (deletedAt: null)", async () => {
|
||||
installFetchMock([{ date: "2026-05-12", steps: 8420 }]);
|
||||
const measuredAt = new Date("2026-05-12T12:00:00.000Z");
|
||||
// Same value, but the row is tombstoned: the write is what resurrects
|
||||
// it — Withings owns the row, so a re-fetched day brings it back.
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{
|
||||
id: "row-steps",
|
||||
type: "ACTIVITY_STEPS",
|
||||
measuredAt,
|
||||
value: 8420,
|
||||
deletedAt: new Date("2026-05-13T00:00:00.000Z"),
|
||||
},
|
||||
] as never);
|
||||
vi.mocked(prisma.measurement.update).mockResolvedValue({} as never);
|
||||
|
||||
const imported = await syncUserActivity("user-1");
|
||||
|
||||
expect(imported).toBe(1);
|
||||
expect(prisma.measurement.createMany).not.toHaveBeenCalled();
|
||||
expect(prisma.measurement.update).toHaveBeenCalledTimes(1);
|
||||
const arg = vi.mocked(prisma.measurement.update).mock.calls[0][0] as {
|
||||
data: { value: number; deletedAt: Date | null };
|
||||
};
|
||||
expect(arg.data.value).toBe(8420);
|
||||
expect(arg.data.deletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("anchors measuredAt at noon UTC so the instant lands inside the local day for every supported tz", async () => {
|
||||
installFetchMock([{ date: "2026-05-12", steps: 1000 }]);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
|
||||
@@ -15,6 +15,7 @@ vi.mock("@/lib/db", () => ({
|
||||
create: vi.fn(),
|
||||
update: vi.fn(),
|
||||
upsert: vi.fn(),
|
||||
updateMany: vi.fn(),
|
||||
},
|
||||
withingsConnection: { findUnique: vi.fn() },
|
||||
},
|
||||
@@ -314,11 +315,14 @@ describe("syncUserSleep — segment writes + idempotency", () => {
|
||||
value: 60,
|
||||
measuredAt: new Date(1715003600 * 1000),
|
||||
sleepStage: "DEEP",
|
||||
// No-op on a live row, resurrection on a tombstoned one — the
|
||||
// source-owned resurrect rule.
|
||||
deletedAt: null,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("stamps every row with an externalId tied to the segment", async () => {
|
||||
it("stamps every row with an externalId keyed on session id + segment START", async () => {
|
||||
installFetchMock([
|
||||
{ startdate: 1715000000, enddate: 1715003600, state: 2, id: 42 },
|
||||
]);
|
||||
@@ -329,7 +333,102 @@ describe("syncUserSleep — segment writes + idempotency", () => {
|
||||
const arg = vi.mocked(prisma.measurement.create).mock.calls[0][0] as {
|
||||
data: { externalId: string };
|
||||
};
|
||||
expect(arg.data.externalId).toBe("withings:sleep:user-1:42:0");
|
||||
// Session id + the segment's own startdate — both stable within a session.
|
||||
// The retired running index counted across the whole rolling 30-day fetch
|
||||
// window, so the window slide renumbered every segment on each sync and a
|
||||
// re-aggregated night inserted a duplicate set.
|
||||
expect(arg.data.externalId).toBe("withings:sleep:user-1:42:1715000000");
|
||||
});
|
||||
|
||||
it("mints IDENTICAL externalIds when the fetch window slides (re-fetch of the same night)", async () => {
|
||||
// The same two segments of session 42, fetched twice: the second fetch's
|
||||
// window has slid so the series now carries an EXTRA leading segment from
|
||||
// a different session — under the retired running index this shifted
|
||||
// every index of session 42 (0,1 → 1,2), minting fresh ids that inserted
|
||||
// a duplicate night. The start-keyed ids must be identical across fetches.
|
||||
const night = [
|
||||
{ startdate: 1715000000, enddate: 1715003600, state: 1, id: 42 },
|
||||
{ startdate: 1715003600, enddate: 1715005400, state: 2, id: 42 },
|
||||
];
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null);
|
||||
vi.mocked(prisma.measurement.create).mockResolvedValue({} as never);
|
||||
|
||||
installFetchMock(night);
|
||||
await syncUserSleep("user-1");
|
||||
const firstIds = vi
|
||||
.mocked(prisma.measurement.create)
|
||||
.mock.calls.map((c) => (c[0].data as { externalId: string }).externalId);
|
||||
|
||||
vi.mocked(prisma.measurement.create).mockClear();
|
||||
installFetchMock([
|
||||
{ startdate: 1714990000, enddate: 1714993600, state: 1, id: 7 },
|
||||
...night,
|
||||
]);
|
||||
await syncUserSleep("user-1");
|
||||
const secondIds = vi
|
||||
.mocked(prisma.measurement.create)
|
||||
.mock.calls.map((c) => (c[0].data as { externalId: string }).externalId)
|
||||
.filter((id) => id.includes(":42:"));
|
||||
|
||||
expect(secondIds).toEqual(firstIds);
|
||||
});
|
||||
|
||||
it("sweeps stale rows per fetched session: live-only, session-prefixed, notIn the fresh set, soft-delete (v1.28.25)", async () => {
|
||||
installFetchMock([
|
||||
{ startdate: 1715000000, enddate: 1715003600, state: 1, id: 42 },
|
||||
{ startdate: 1715003600, enddate: 1715005400, state: 2, id: 42 },
|
||||
]);
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null);
|
||||
vi.mocked(prisma.measurement.create).mockResolvedValue({} as never);
|
||||
vi.mocked(prisma.measurement.updateMany).mockResolvedValue({
|
||||
count: 0,
|
||||
} as never);
|
||||
|
||||
await syncUserSleep("user-1");
|
||||
|
||||
expect(prisma.measurement.updateMany).toHaveBeenCalledTimes(1);
|
||||
const arg = vi.mocked(prisma.measurement.updateMany).mock.calls[0][0] as {
|
||||
where: Record<string, unknown>;
|
||||
data: Record<string, unknown>;
|
||||
};
|
||||
expect(arg.where).toEqual({
|
||||
userId: "user-1",
|
||||
source: "WITHINGS",
|
||||
type: "SLEEP_DURATION",
|
||||
deletedAt: null,
|
||||
externalId: {
|
||||
startsWith: "withings:sleep:user-1:42:",
|
||||
notIn: [
|
||||
"withings:sleep:user-1:42:1715000000",
|
||||
"withings:sleep:user-1:42:1715003600",
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(arg.data).toEqual({ deletedAt: expect.any(Date) });
|
||||
|
||||
// A legacy running-index row for this session falls inside the sweep:
|
||||
// under the session prefix, never in the fresh start-keyed set.
|
||||
const where = arg.where as {
|
||||
externalId: { startsWith: string; notIn: string[] };
|
||||
};
|
||||
const legacyId = "withings:sleep:user-1:42:0";
|
||||
expect(legacyId.startsWith(where.externalId.startsWith)).toBe(true);
|
||||
expect(where.externalId.notIn).not.toContain(legacyId);
|
||||
});
|
||||
|
||||
it("never sweeps sessions absent from this fetch (no id → no sweep entry)", async () => {
|
||||
// Segments without a session id cannot be bounded to one night — their
|
||||
// shared `no-id` prefix would span every id-less night in history — so
|
||||
// they must not produce a sweep.
|
||||
installFetchMock([
|
||||
{ startdate: 1715000000, enddate: 1715003600, state: 2 },
|
||||
]);
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null);
|
||||
vi.mocked(prisma.measurement.create).mockResolvedValue({} as never);
|
||||
|
||||
const imported = await syncUserSleep("user-1");
|
||||
expect(imported).toBe(1);
|
||||
expect(prisma.measurement.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calls recordSyncSuccess after a clean round-trip", async () => {
|
||||
|
||||
@@ -372,13 +372,23 @@ export async function syncUserActivity(
|
||||
type: { in: distinctTypes },
|
||||
measuredAt: { in: distinctMeasuredAt },
|
||||
},
|
||||
select: { id: true, type: true, measuredAt: true, value: true },
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
measuredAt: true,
|
||||
value: true,
|
||||
deletedAt: true,
|
||||
},
|
||||
});
|
||||
const existingBySlot = new Map<string, { id: string; value: number }>();
|
||||
const existingBySlot = new Map<
|
||||
string,
|
||||
{ id: string; value: number; deletedAt: Date | null }
|
||||
>();
|
||||
for (const row of existingRows) {
|
||||
existingBySlot.set(slotKey(row.type, row.measuredAt), {
|
||||
id: row.id,
|
||||
value: row.value,
|
||||
deletedAt: row.deletedAt,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -388,8 +398,11 @@ export async function syncUserActivity(
|
||||
const existing = existingBySlot.get(slotKey(p.type, p.measuredAt));
|
||||
if (existing) {
|
||||
// Only write when the value actually moved — a re-sync of an
|
||||
// unchanged day is then a pure no-op on the write path.
|
||||
if (existing.value !== p.value) {
|
||||
// unchanged LIVE day is then a pure no-op on the write path. A
|
||||
// TOMBSTONED row always writes: the update is what resurrects it
|
||||
// (`deletedAt: null`) — Withings is the source of truth for its
|
||||
// own rows (mirrors Google / Fitbit).
|
||||
if (existing.value !== p.value || existing.deletedAt !== null) {
|
||||
toUpdate.push({ id: existing.id, value: p.value });
|
||||
}
|
||||
} else {
|
||||
@@ -425,7 +438,9 @@ export async function syncUserActivity(
|
||||
toUpdate.map((u) =>
|
||||
prisma.measurement.update({
|
||||
where: { id: u.id },
|
||||
data: { value: u.value },
|
||||
// `deletedAt: null` — no-op on a live row, deliberate
|
||||
// RESURRECTION on a tombstoned one (see the skip above).
|
||||
data: { value: u.value, deletedAt: null },
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
@@ -19,11 +19,14 @@
|
||||
* reserved for HealthKit ingest (pre-iOS-16 `asleepUnspecified` and
|
||||
* iOS-16+ `inBed` respectively) and are NOT emitted from this path.
|
||||
*
|
||||
* Idempotency: every segment writes through the v1.4.25 W17b/c
|
||||
* composite (Migration 0055 — `(userId, type, measuredAt, source,
|
||||
* sleepStage)` with NULLS NOT DISTINCT). The five-tuple guarantees a
|
||||
* re-sync upserts the same row rather than inserting a duplicate, even
|
||||
* when Withings re-emits the night with adjusted segment boundaries.
|
||||
* Idempotency: every segment keys on the stable externalId
|
||||
* `withings:sleep:<user>:<sessionId>:<segment-start-unix>` — the session id
|
||||
* plus the segment's own start instant, both stable across re-syncs of an
|
||||
* unchanged night. When Withings re-aggregates a night with adjusted
|
||||
* boundaries, the shifted segments mint new ids and the session-scoped sweep
|
||||
* (`sweepStaleSleepSegments`) tombstones whatever the re-aggregation
|
||||
* orphaned — including every legacy row from the retired running-index
|
||||
* format, so that fix self-heals without a migration.
|
||||
*
|
||||
* Date semantics: each segment's `measuredAt` is the segment's
|
||||
* `enddate` (unix seconds → `Date`). Every reader treats `measuredAt`
|
||||
@@ -46,6 +49,11 @@ import {
|
||||
} from "@/lib/rollups/measurement-rollups";
|
||||
import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate";
|
||||
|
||||
import {
|
||||
sweepStaleSleepSegments,
|
||||
type SleepSegmentSweep,
|
||||
} from "@/lib/sleep/sweep-stale-segments";
|
||||
|
||||
import { hasActivityScope } from "./client";
|
||||
import {
|
||||
WithingsApiError,
|
||||
@@ -389,17 +397,20 @@ export async function syncUserSleep(
|
||||
}
|
||||
|
||||
let imported = 0;
|
||||
let segmentIndex = 0;
|
||||
// v1.4.39.1 — track every (type, measuredAt) we touched so the
|
||||
// persistent rollup tier can be re-folded at the end. See sync.ts for
|
||||
// the full rationale.
|
||||
const touched: Array<{ type: MeasurementType; measuredAt: Date }> = [];
|
||||
// Fresh segment externalIds per session id, driving the session-scoped
|
||||
// sweep below. Segments without a session id can't be bounded to one night
|
||||
// and are excluded (their `no-id` prefix would span every id-less night in
|
||||
// history, not just this fetch).
|
||||
const freshBySession = new Map<number, string[]>();
|
||||
for (const segment of segments) {
|
||||
const stage = mapWithingsSleepState(segment.state);
|
||||
if (!stage) {
|
||||
// State 4 (manual / synthetic marker) is ignored — see file
|
||||
// header.
|
||||
segmentIndex++;
|
||||
continue;
|
||||
}
|
||||
// `measuredAt` is the segment END (`enddate`). Every reader treats
|
||||
@@ -413,15 +424,29 @@ export async function syncUserSleep(
|
||||
// floating-point noise.
|
||||
const minutes = Math.round(durationSec / 60);
|
||||
|
||||
const externalId = `withings:sleep:${userId}:${segment.id ?? "no-id"}:${segmentIndex}`;
|
||||
// Keyed by the segment's own START instant — a coordinate that is stable
|
||||
// within a session — NOT a positional index. The old running index counted
|
||||
// across the whole 30-day fetch window, so the sliding window renumbered
|
||||
// every segment on each sync: the probe below never matched, the update
|
||||
// branch was dead code, and a re-aggregated night inserted a second set of
|
||||
// rows next to the first (the same double-count bug class Google Health
|
||||
// fixed with its stable session anchor). Legacy indexed rows are swept by
|
||||
// the session-scoped cleanup below, so the format change self-heals.
|
||||
const externalId = `withings:sleep:${userId}:${segment.id ?? "no-id"}:${segment.startdate}`;
|
||||
if (typeof segment.id === "number") {
|
||||
const fresh = freshBySession.get(segment.id) ?? [];
|
||||
fresh.push(externalId);
|
||||
freshBySession.set(segment.id, fresh);
|
||||
}
|
||||
|
||||
try {
|
||||
// Key the re-sync lookup on the STABLE `externalId` (segment id +
|
||||
// index), not `measuredAt`. Withings re-aggregates a night between
|
||||
// syncs, so a segment's `enddate` — and therefore its END-stamped
|
||||
// `measuredAt` — shifts; keying on `measuredAt` would miss the prior
|
||||
// row and then collide with the unique `externalId`, leaving the night
|
||||
// stuck at the stale value. Update both `value` and `measuredAt`.
|
||||
// Key the re-sync lookup on the STABLE `externalId` (session id +
|
||||
// segment start), not `measuredAt`. Withings re-aggregates a night
|
||||
// between syncs, so a segment's `enddate` — and therefore its
|
||||
// END-stamped `measuredAt` — shifts; keying on `measuredAt` would miss
|
||||
// the prior row and then collide with the unique `externalId`, leaving
|
||||
// the night stuck at the stale value. Update both `value` and
|
||||
// `measuredAt`.
|
||||
const existing = await prisma.measurement.findFirst({
|
||||
where: {
|
||||
userId,
|
||||
@@ -434,7 +459,16 @@ export async function syncUserSleep(
|
||||
if (existing) {
|
||||
await prisma.measurement.update({
|
||||
where: { id: existing.id },
|
||||
data: { value: minutes, measuredAt, sleepStage: stage },
|
||||
// `deletedAt: null` is a no-op on a live row, a deliberate
|
||||
// RESURRECTION on a tombstoned one — Withings is the source of
|
||||
// truth for its own rows, so a re-fetched segment brings the row
|
||||
// back (mirrors Google / Fitbit).
|
||||
data: {
|
||||
value: minutes,
|
||||
measuredAt,
|
||||
sleepStage: stage,
|
||||
deletedAt: null,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
await prisma.measurement.create({
|
||||
@@ -457,9 +491,23 @@ export async function syncUserSleep(
|
||||
`Failed to upsert sleep segment (${stage}, ${measuredAt.toISOString()}): ${err}`,
|
||||
);
|
||||
}
|
||||
segmentIndex++;
|
||||
}
|
||||
|
||||
// Session-scoped sweep (mirrors Google Health's replace-by-window): for each
|
||||
// session this fetch re-produced, soft-delete any live WITHINGS
|
||||
// SLEEP_DURATION row under the session's externalId prefix that is NOT in
|
||||
// the fresh set. Collapses re-aggregation orphans AND every legacy
|
||||
// running-index row for re-fetched sessions. Sessions absent from this fetch
|
||||
// are never touched; a sweep failure never fails the sync (the session stays
|
||||
// inside the rolling 30-day window, so the hourly cron retries it).
|
||||
const sweeps: SleepSegmentSweep[] = [...freshBySession].map(
|
||||
([sessionId, keepIds]) => ({
|
||||
prefix: `withings:sleep:${userId}:${sessionId}:`,
|
||||
keepIds,
|
||||
}),
|
||||
);
|
||||
await sweepStaleSleepSegments(userId, "WITHINGS", sweeps);
|
||||
|
||||
// v1.18.10 P0 — nightly sleep vitals (avg HR / respiratory rate / SDNN HRV
|
||||
// / avg SpO2 / sleep score) from the per-night summary. A summary-fetch
|
||||
// failure must not lose the stage segments already written, so it runs in
|
||||
@@ -502,6 +550,10 @@ export async function syncUserSleep(
|
||||
update: {
|
||||
value: vital.value,
|
||||
measuredAt,
|
||||
// No-op on a live row, a deliberate RESURRECTION on a
|
||||
// tombstoned one — Withings is the source of truth for its
|
||||
// own rows (mirrors Google / Fitbit).
|
||||
deletedAt: null,
|
||||
syncVersion: { increment: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -237,7 +237,11 @@ export async function syncUserMeasurements(
|
||||
if (existing) {
|
||||
await prisma.measurement.update({
|
||||
where: { id: existing.id },
|
||||
data: { value: m.value },
|
||||
// `deletedAt: null` is a no-op on a live row, a deliberate
|
||||
// RESURRECTION on a tombstoned one — Withings is the source of
|
||||
// truth for its own rows, so a re-fetched measure brings the row
|
||||
// back (mirrors Google / Fitbit).
|
||||
data: { value: m.value, deletedAt: null },
|
||||
});
|
||||
} else {
|
||||
await prisma.measurement.create({
|
||||
|
||||
Reference in new issue
Block a user