release: v1.32.0 — hardening and scalability (#580)

* release: prepare v1.32.0

* test: stabilize water capture re-entry
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-07-22 04:14:15 +02:00
1 parent b01822be48
commit 62eb431d66
372 files changed
+25178 -8561

No files matched your search

@@ -6,13 +6,13 @@ process.env.ENCRYPTION_KEY =
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
const mocks = vi.hoisted(() => ({
findMeasurementsPaged: vi.fn(),
iterateMeasurementPages: vi.fn(),
buildCycleBackupSection: vi.fn(),
buildRecordsBackupSection: vi.fn(),
}));
vi.mock("@/lib/export/paged-measurements", () => ({
findMeasurementsPaged: mocks.findMeasurementsPaged,
iterateMeasurementPages: mocks.iterateMeasurementPages,
}));
vi.mock("@/lib/cycle/backup", () => ({
buildCycleBackupSection: mocks.buildCycleBackupSection,
@@ -181,30 +181,38 @@ function makePrisma() {
function installSectionMocks() {
void measurementNote;
mocks.findMeasurementsPaged.mockResolvedValue([
{
id: "measurement-tombstone",
type: "SLEEP_DURATION",
value: 75,
valueMin: 60,
valueMax: 90,
unit: "minutes",
measuredAt: new Date("2026-07-19T07:00:00.000Z"),
source: "IMPORT",
notes: null,
notesEncrypted: measurementNote,
externalId: "oura-stage-deep",
externalSourceVersion: "oura-v3",
glucoseContext: null,
sleepStage: "DEEP",
rhythmClassification: null,
deviceType: "ring",
syncVersion: 7,
deletedAt,
createdAt: new Date("2026-07-19T07:01:00.000Z"),
updatedAt: new Date("2026-07-19T07:02:00.000Z"),
},
]);
const measurement = {
id: "measurement-tombstone",
type: "SLEEP_DURATION",
value: 75,
valueMin: 60,
valueMax: 90,
unit: "minutes",
measuredAt: new Date("2026-07-19T07:00:00.000Z"),
source: "IMPORT",
notes: null,
notesEncrypted: measurementNote,
externalId: "oura-stage-deep",
externalSourceVersion: "oura-v3",
glucoseContext: null,
sleepStage: "DEEP",
rhythmClassification: null,
deviceType: "ring",
syncVersion: 7,
deletedAt,
createdAt: new Date("2026-07-19T07:01:00.000Z"),
updatedAt: new Date("2026-07-19T07:02:00.000Z"),
};
mocks.iterateMeasurementPages.mockImplementation(async function* () {
yield [measurement];
yield [
{
...measurement,
id: "measurement-page-2",
measuredAt: new Date("2026-07-18T07:00:00.000Z"),
},
];
});
mocks.buildCycleBackupSection.mockResolvedValue({
cycleProfile: null,
cycles: [],
@@ -223,10 +231,11 @@ function installSectionMocks() {
workouts: { included: "summary-only", note: "included" },
},
});
return measurement;
}
describe("buildFullBackupPayload disaster-recovery mode", () => {
it("preserves measurement and mood tombstones with stable ids", async () => {
it("preserves tombstones while consuming measurement pages into the final payload", async () => {
installSectionMocks();
const prisma = makePrisma();
@@ -239,7 +248,7 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
},
);
expect(mocks.findMeasurementsPaged).toHaveBeenCalledWith(
expect(mocks.iterateMeasurementPages).toHaveBeenCalledWith(
prisma,
{ userId: "user-1" },
expect.objectContaining({ id: true, deletedAt: true }),
@@ -287,6 +296,10 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
createdAt: "2026-07-19T07:01:00.000Z",
updatedAt: "2026-07-19T07:02:00.000Z",
},
{
id: "measurement-page-2",
measuredAt: "2026-07-18T07:00:00.000Z",
},
],
medications: [
expect.objectContaining({
@@ -323,4 +336,19 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
],
});
});
it("propagates a failure from a later measurement page", async () => {
const measurement = installSectionMocks();
const failure = new Error("backup measurement page failed");
mocks.iterateMeasurementPages.mockImplementation(async function* () {
yield [measurement];
throw failure;
});
await expect(
buildFullBackupPayload(makePrisma() as never, "user-1", {
purpose: "disaster-recovery",
}),
).rejects.toBe(failure);
});
});
@@ -0,0 +1,207 @@
import { describe, expect, it, vi } from "vitest";
import {
reconstructSleepSessions,
type SleepStageRow,
} from "@/lib/analytics/sleep-night";
import {
groupMeasurementPagesForExport,
iterateMeasurementPages,
} from "../paged-measurements";
function row(id: string, measuredAt: string) {
return { id, measuredAt: new Date(measuredAt) };
}
describe("iterateMeasurementPages", () => {
it("yields bounded keyset pages without retaining them in one result", async () => {
const first = [
row("m-3", "2026-07-21T12:00:00.000Z"),
row("m-2", "2026-07-21T12:00:00.000Z"),
];
const second = [row("m-1", "2026-07-20T12:00:00.000Z")];
const findMany = vi
.fn()
.mockResolvedValueOnce(first)
.mockResolvedValueOnce(second);
const db = { measurement: { findMany } };
const iterator = iterateMeasurementPages(
db as never,
{ userId: "user-1" },
{ id: true, measuredAt: true },
2,
)[Symbol.asyncIterator]();
await expect(iterator.next()).resolves.toEqual({
done: false,
value: first,
});
expect(findMany).toHaveBeenCalledTimes(1);
await expect(iterator.next()).resolves.toEqual({
done: false,
value: second,
});
expect(findMany).toHaveBeenCalledTimes(2);
await expect(iterator.next()).resolves.toEqual({
done: true,
value: undefined,
});
expect(findMany).toHaveBeenNthCalledWith(1, {
where: { userId: "user-1" },
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
take: 2,
select: { id: true, measuredAt: true },
});
expect(findMany).toHaveBeenNthCalledWith(2, {
where: { userId: "user-1" },
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
take: 2,
cursor: { id: "m-2" },
skip: 1,
select: { id: true, measuredAt: true },
});
});
it("does not yield an empty terminal page after a full page", async () => {
const full = [
row("m-2", "2026-07-21T12:00:00.000Z"),
row("m-1", "2026-07-20T12:00:00.000Z"),
];
const findMany = vi
.fn()
.mockResolvedValueOnce(full)
.mockResolvedValueOnce([]);
const db = { measurement: { findMany } };
const pages = [];
for await (const page of iterateMeasurementPages(
db as never,
{},
{ id: true, measuredAt: true },
2,
)) {
pages.push(page);
}
expect(pages).toEqual([full]);
expect(findMany).toHaveBeenCalledTimes(2);
});
it("propagates a database failure after releasing the prior page", async () => {
const failure = new Error("page two failed");
const findMany = vi
.fn()
.mockResolvedValueOnce([
row("m-2", "2026-07-21T12:00:00.000Z"),
row("m-1", "2026-07-20T12:00:00.000Z"),
])
.mockRejectedValueOnce(failure);
const db = { measurement: { findMany } };
const iterator = iterateMeasurementPages(
db as never,
{},
{ id: true, measuredAt: true },
2,
)[Symbol.asyncIterator]();
await expect(iterator.next()).resolves.toMatchObject({ done: false });
await expect(iterator.next()).rejects.toBe(failure);
});
it("stops querying when the consumer cancels after one page", async () => {
const findMany = vi
.fn()
.mockResolvedValue([
row("m-2", "2026-07-21T12:00:00.000Z"),
row("m-1", "2026-07-20T12:00:00.000Z"),
]);
const db = { measurement: { findMany } };
const iterator = iterateMeasurementPages(
db as never,
{},
{ id: true, measuredAt: true },
2,
)[Symbol.asyncIterator]();
await iterator.next();
await iterator.return?.();
expect(findMany).toHaveBeenCalledTimes(1);
});
it("keeps a sleep wake-day carry intact across database page boundaries", async () => {
const newer = {
type: "WEIGHT",
value: 80,
measuredAt: new Date("2026-07-21T09:00:00.000Z"),
};
const rem = {
type: "SLEEP_DURATION",
value: 60,
measuredAt: new Date("2026-07-21T08:00:00.000Z"),
};
const deep = {
type: "SLEEP_DURATION",
value: 60,
measuredAt: new Date("2026-07-21T07:00:00.000Z"),
};
const older = {
type: "WEIGHT",
value: 79,
measuredAt: new Date("2026-07-20T00:00:00.000Z"),
};
async function* pages() {
yield [newer, rem];
yield [deep, older];
}
const groups = [];
for await (const group of groupMeasurementPagesForExport(
pages(),
"UTC",
"night",
)) {
groups.push(group);
}
expect(groups).toEqual([[newer], [rem, deep], [older]]);
});
it("keeps a contiguous cross-midnight stage together across pages", async () => {
const crossMidnight = {
type: "SLEEP_DURATION",
value: 240,
measuredAt: new Date("2026-07-21T02:00:00.000Z"),
sleepStage: "CORE" as const,
};
const precedingStage = {
type: "SLEEP_DURATION",
value: 60,
measuredAt: new Date("2026-07-20T22:00:00.000Z"),
sleepStage: "DEEP" as const,
};
const older = {
type: "WEIGHT",
value: 80,
measuredAt: new Date("2026-07-19T12:00:00.000Z"),
};
const sleepRows: SleepStageRow[] = [crossMidnight, precedingStage];
expect(reconstructSleepSessions(sleepRows, "UTC")).toHaveLength(1);
async function* pages() {
yield [crossMidnight];
yield [precedingStage, older];
}
const groups = [];
for await (const group of groupMeasurementPagesForExport(
pages(),
"UTC",
"night",
)) {
groups.push(group);
}
expect(groups).toEqual([[crossMidnight, precedingStage], [older]]);
});
});
+76 -64
View File
@@ -13,7 +13,7 @@
import { Buffer } from "node:buffer";
import type { PrismaClient } from "@/generated/prisma/client";
import { readNote } from "@/lib/crypto/note-cipher";
import { findMeasurementsPaged } from "@/lib/export/paged-measurements";
import { iterateMeasurementPages } from "@/lib/export/paged-measurements";
import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup";
import { buildCycleBackupSection } from "@/lib/cycle/backup";
import {
@@ -63,32 +63,80 @@ export async function buildFullBackupPayload(
disasterRecovery
? prisma.appSettings.findUnique({ where: { id: "singleton" } })
: Promise.resolve(null),
findMeasurementsPaged(
prisma,
disasterRecovery ? { userId } : { userId, deletedAt: null },
{
id: true,
type: true,
value: true,
valueMin: true,
valueMax: true,
unit: true,
measuredAt: true,
source: true,
notes: true,
notesEncrypted: true,
externalId: true,
externalSourceVersion: true,
glucoseContext: true,
sleepStage: true,
rhythmClassification: true,
deviceType: true,
syncVersion: true,
deletedAt: true,
createdAt: true,
updatedAt: true,
},
),
(async () => {
const rows: Array<Record<string, unknown>> = [];
const pages = iterateMeasurementPages(
prisma,
disasterRecovery ? { userId } : { userId, deletedAt: null },
{
id: true,
type: true,
value: true,
valueMin: true,
valueMax: true,
unit: true,
measuredAt: true,
source: true,
notes: true,
notesEncrypted: true,
externalId: true,
externalSourceVersion: true,
glucoseContext: true,
sleepStage: true,
rhythmClassification: true,
deviceType: true,
syncVersion: true,
deletedAt: true,
createdAt: true,
updatedAt: true,
},
);
for await (const page of pages) {
for (const measurement of page) {
rows.push(
disasterRecovery
? {
id: measurement.id,
type: measurement.type,
value: measurement.value,
valueMin: measurement.valueMin,
valueMax: measurement.valueMax,
unit: measurement.unit,
measuredAt: measurement.measuredAt.toISOString(),
source: measurement.source,
notes: measurement.notes,
notesEncrypted: measurement.notesEncrypted
? Buffer.from(measurement.notesEncrypted).toString("base64")
: null,
externalId: measurement.externalId,
externalSourceVersion: measurement.externalSourceVersion,
glucoseContext: measurement.glucoseContext,
sleepStage: measurement.sleepStage,
rhythmClassification: measurement.rhythmClassification,
deviceType: measurement.deviceType,
syncVersion: measurement.syncVersion,
deletedAt: measurement.deletedAt?.toISOString() ?? null,
createdAt: measurement.createdAt.toISOString(),
updatedAt: measurement.updatedAt.toISOString(),
}
: {
id: measurement.id,
type: measurement.type,
value: measurement.value,
unit: measurement.unit,
measuredAt: measurement.measuredAt.toISOString(),
source: measurement.source,
notes: readNote(
measurement.notesEncrypted,
measurement.notes,
),
deletedAt: measurement.deletedAt?.toISOString() ?? null,
},
);
}
}
return rows;
})(),
prisma.medication.findMany({
where: { userId },
include: { schedules: true },
@@ -134,43 +182,7 @@ export async function buildFullBackupPayload(
documentQuotaBytes: appSettings.documentQuotaBytes.toString(),
}
: null,
measurements: measurements.map((m) =>
disasterRecovery
? {
id: m.id,
type: m.type,
value: m.value,
valueMin: m.valueMin,
valueMax: m.valueMax,
unit: m.unit,
measuredAt: m.measuredAt.toISOString(),
source: m.source,
notes: m.notes,
notesEncrypted: m.notesEncrypted
? Buffer.from(m.notesEncrypted).toString("base64")
: null,
externalId: m.externalId,
externalSourceVersion: m.externalSourceVersion,
glucoseContext: m.glucoseContext,
sleepStage: m.sleepStage,
rhythmClassification: m.rhythmClassification,
deviceType: m.deviceType,
syncVersion: m.syncVersion,
deletedAt: m.deletedAt?.toISOString() ?? null,
createdAt: m.createdAt.toISOString(),
updatedAt: m.updatedAt.toISOString(),
}
: {
id: m.id,
type: m.type,
value: m.value,
unit: m.unit,
measuredAt: m.measuredAt.toISOString(),
source: m.source,
notes: readNote(m.notesEncrypted, m.notes),
deletedAt: m.deletedAt?.toISOString() ?? null,
},
),
measurements,
medications: medications.map((m) => ({
...(disasterRecovery
? {
+191 -37
View File
@@ -1,53 +1,207 @@
/**
* Keyset-paginated measurement reader for the export surfaces.
* Keyset-paginated measurement readers for export surfaces.
*
* The legacy exports (`GET /api/export`, `GET /api/export/measurements`, the
* full-backup builder) read the user's ENTIRE measurement table in one
* `findMany`. On a CGM + per-sample-HR account that is a six-figure row set
* materialised by a single query — the v1.28.2x incident class. This helper
* keeps the output row set (and its `measuredAt desc` order) identical while
* bounding each database round-trip to `MEASUREMENT_EXPORT_PAGE_SIZE` rows:
* the driver never buffers one giant result, and the event loop yields
* between chunks.
*
* Ordering: `measuredAt desc, id desc`. The exports always sorted
* `measuredAt desc`; the `id` tie-breaker makes the keyset cursor total (two
* rows can share a timestamp) — for equal-timestamp rows the previous order
* was unspecified heap order, so pinning it is not a behaviour change any
* consumer could have relied on.
*
* The `select` MUST include `id` (the cursor key); the generic constraint
* enforces it at compile time.
* Pages are ordered by `measuredAt desc, id desc`. The `id` tie-breaker makes
* the keyset cursor total when measurements share a timestamp. Consumers own
* one page at a time; advancing the iterator releases the previous page before
* the next database request begins.
*/
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
import type {
Prisma,
PrismaClient,
SleepStage,
} from "@/generated/prisma/client";
import {
formatMeasurementsForExport,
type ExportableRecord,
} from "@/lib/export";
import { shapeMeasurementNotes } from "@/lib/crypto/note-cipher";
import type { GlucoseUnit } from "@/lib/glucose";
import { userDayKey } from "@/lib/tz/format";
export const MEASUREMENT_EXPORT_PAGE_SIZE = 10_000;
export async function findMeasurementsPaged<
type MeasurementPayload<S extends Prisma.MeasurementSelect> =
Prisma.MeasurementGetPayload<{ select: S }>;
/**
* Yield bounded measurement pages. Empty terminal pages are not exposed.
*
* A full page advances with an `id` cursor and `skip: 1`; a short page ends
* iteration without another query. Database failures are deliberately not
* caught so response streams and backup builders observe the original error.
*/
export async function* iterateMeasurementPages<
S extends Prisma.MeasurementSelect & { id: true },
>(
db: PrismaClient,
where: Prisma.MeasurementWhereInput,
select: S,
pageSize: number = MEASUREMENT_EXPORT_PAGE_SIZE,
): Promise<Array<Prisma.MeasurementGetPayload<{ select: S }>>> {
const rows: Array<Prisma.MeasurementGetPayload<{ select: S }>> = [];
): AsyncGenerator<Array<MeasurementPayload<S>>, void, void> {
if (!Number.isInteger(pageSize) || pageSize <= 0) {
throw new RangeError(
"Measurement export page size must be a positive integer",
);
}
let cursorId: string | null = null;
for (;;) {
const batch: Array<Prisma.MeasurementGetPayload<{ select: S }>> =
(await db.measurement.findMany({
where,
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
take: pageSize,
...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}),
select,
})) as Array<Prisma.MeasurementGetPayload<{ select: S }>>;
// Loop, not `push(...batch)` — a 10k-element spread is fine today, but
// this file exists because of stack-overflow-by-spread regressions on
// large arrays (v1.28.22 class); keep the pattern boring.
for (const row of batch) rows.push(row);
if (batch.length < pageSize) break;
cursorId = (batch[batch.length - 1] as { id: string }).id;
let page = (await db.measurement.findMany({
where,
orderBy: [{ measuredAt: "desc" }, { id: "desc" }],
take: pageSize,
...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}),
select,
})) as Array<MeasurementPayload<S>>;
if (page.length === 0) return;
const isLastPage = page.length < pageSize;
const lastRow: unknown = page[page.length - 1];
if (
lastRow === null ||
typeof lastRow !== "object" ||
!("id" in lastRow) ||
typeof lastRow.id !== "string"
) {
throw new TypeError("Measurement export page is missing its cursor id");
}
const nextCursorId = lastRow.id;
yield page;
// Drop the array reference before awaiting the next query. Consumers may
// retain rows intentionally, but the pager itself never retains a prior
// page or combines pages into an all-history result.
page = [];
if (isLastPage) return;
cursorId = nextCursorId;
}
}
const SLEEP_SESSION_GAP_MS = 3 * 60 * 60 * 1000;
/**
* Turn database pages into formatter-safe chunks.
*
* Raw exports can release every page immediately. Night-granularity exports
* must keep all sleep sessions assigned to one local wake day together so
* `formatMeasurementsForExport` can select the same main night and naps as the
* historical all-at-once formatter. The carry is released once iteration has
* moved to an older local day and more than the sleep-session gap before the
* earliest carried sleep-segment start. Non-sleep-only pages never enter it.
*/
export async function* groupMeasurementPagesForExport<
T extends { type: string; value: number; measuredAt: Date },
>(
pages: AsyncIterable<readonly T[]>,
userTz: string,
granularity: "night" | "raw",
): AsyncGenerator<readonly T[], void, void> {
if (granularity === "raw") {
for await (const page of pages) {
if (page.length > 0) yield page;
}
return;
}
let carry: T[] = [];
let carryWakeDay: string | null = null;
let earliestSleepStartAtMs: number | null = null;
for await (const page of pages) {
let ready: T[] = [];
for (const row of page) {
const rowAtMs = row.measuredAt.getTime();
if (
carry.length > 0 &&
carryWakeDay !== null &&
earliestSleepStartAtMs !== null &&
earliestSleepStartAtMs - rowAtMs > SLEEP_SESSION_GAP_MS &&
userDayKey(row.measuredAt, userTz) < carryWakeDay
) {
if (ready.length > 0) {
yield ready;
ready = [];
}
yield carry;
carry = [];
carryWakeDay = null;
earliestSleepStartAtMs = null;
}
if (carry.length === 0 && row.type !== "SLEEP_DURATION") {
ready.push(row);
continue;
}
if (ready.length > 0) {
yield ready;
ready = [];
}
carry.push(row);
if (row.type === "SLEEP_DURATION") {
carryWakeDay ??= userDayKey(row.measuredAt, userTz);
const rowStartAtMs =
rowAtMs - (Number.isFinite(row.value) ? row.value : 0) * 60_000;
earliestSleepStartAtMs =
earliestSleepStartAtMs === null
? rowStartAtMs
: Math.min(earliestSleepStartAtMs, rowStartAtMs);
}
}
if (ready.length > 0) yield ready;
}
if (carry.length > 0) yield carry;
}
export interface MeasurementPageFormatOptions {
granularity: "night" | "raw";
sourcePriorityJson: unknown;
glucoseUnit: GlucoseUnit;
}
/**
* Decrypt and format one safe measurement chunk at a time. Each yielded array
* is ready for CSV or JSON framing and can be released after serialization.
*/
export async function* formatMeasurementPageChunks<
T extends {
type: string;
value: number;
unit: string;
measuredAt: Date;
source: string;
notes: string | null;
notesEncrypted: Uint8Array | null;
glucoseContext?: string | null;
sleepStage?: SleepStage | null;
deviceType?: string | null;
},
>(
pages: AsyncIterable<readonly T[]>,
userTz: string,
options: MeasurementPageFormatOptions,
): AsyncGenerator<ExportableRecord[], void, void> {
const groupedPages = groupMeasurementPagesForExport(
pages,
userTz,
options.granularity,
);
for await (const page of groupedPages) {
const records = formatMeasurementsForExport(
page.map(shapeMeasurementNotes),
userTz,
{
granularity: options.granularity,
sleepTz: userTz,
sourcePriorityJson: options.sourcePriorityJson,
glucoseUnit: options.glucoseUnit,
},
);
if (records.length > 0) yield records;
}
return rows;
}