Merge branch 'feat/v1462-scale' into release/v1.46.2

# Conflicts:
#	src/__tests__/baselines/version-stamp-comments.json
This commit is contained in:
Marc-André Bombeck committed 2026-10-29 10:18:22 +01:00
commit ad5d154561
57 files changed
+2736 -425

No files matched your search

@@ -1,5 +1,5 @@
{
"registeredQueueCount": 125,
"registeredQueueCount": 126,
"registeredQueues": [
"account-deletion-purge",
"achievement-unlock-sweep",
@@ -112,6 +112,7 @@
"strava-sync",
"stress-score-compute",
"supplement-nutrient-recompute",
"system-job-alert",
"tls-pin-monitor",
"whoop-backfill",
"whoop-cycle-sync",
@@ -1793,6 +1794,21 @@
"retry_delay_max": null,
"retry_limit": 2
},
{
"dead_letter": null,
"deletion_seconds": 604800,
"expire_seconds": 900,
"heartbeat_seconds": null,
"name": "system-job-alert",
"notify": false,
"partition": false,
"policy": "standard",
"retention_seconds": 1209600,
"retry_backoff": false,
"retry_delay": 0,
"retry_delay_max": null,
"retry_limit": 2
},
{
"dead_letter": null,
"deletion_seconds": 604800,
@@ -2825,6 +2841,18 @@
},
"timezone": "Europe/Berlin"
},
{
"cron": "43 * * * *",
"data": {},
"key": "",
"kind": "cron",
"name": "system-job-alert",
"options": {
"retryLimit": 0,
"tz": "Europe/Berlin"
},
"timezone": "Europe/Berlin"
},
{
"cron": "7 */6 * * *",
"data": {},
@@ -3682,6 +3710,13 @@
"localConcurrency": 1
}
},
{
"queue": "system-job-alert",
"options": {
"localConcurrency": 1,
"pollingIntervalSeconds": 30
}
},
{
"queue": "tls-pin-monitor",
"options": {
@@ -12,10 +12,8 @@ import { beforeAll, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import type { Prisma } from "@/generated/prisma/client";
import {
enqueueBootTimeRollupBackfill,
recomputeUserRollups,
} from "@/lib/rollups/measurement-rollups";
import { enqueueBootTimeRollupBackfill } from "@/lib/rollups/full-backfill";
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
@@ -0,0 +1,295 @@
/**
* A large Google Health account end to end (#1297, #1298).
*
* One account with a dense heart-rate history, written through the real
* Google Health backfill (Google's API replaced at the `safeFetch` seam, as in
* `google-health-dense-backfill-memory.test.ts`), then taken through what a
* self-hosted instance with such an account went through:
*
* 1. The account's DAY rollups lose their reading extremes, as rows folded
* before migration 0417 have them. The boot discovery finds the account,
* and the full backfill, a calendar year per pass with every statement
* under its own limit, refills every bucket inside the fold window.
* 2. The dense fold thins the heart rate older than the retention window to
* hourly `stats:` rows of Google Health, and the DAY rollups keep the
* figures they had from the raw samples.
* 3. A second full-history backfill over the same samples inserts nothing:
* the folded hours are not written again.
*
* Each step reports its duration on stderr (`[large-account]`); the size is
* `POINTS` readings, one every five minutes, about four years.
*/
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import type { Job } from "pg-boss";
import { Prisma } from "@/generated/prisma/client";
import { getPrismaClient, truncateAllTables } from "./setup";
const USER = "large-account-google-health";
const POINTS = 420_000;
const STEP_MS = 5 * 60_000;
const DAY_MS = 86_400_000;
const fake = vi.hoisted(() => ({ total: 0, now: 0 }));
vi.mock("@/lib/safe-fetch", async (importOriginal) => {
const actual = await importOriginal<typeof import("@/lib/safe-fetch")>();
return {
...actual,
safeFetch: async (url: string) => {
const u = new URL(url);
let body: unknown = {};
if (u.pathname.endsWith("/dataTypes/heart-rate/dataPoints")) {
const pageSize = Number(u.searchParams.get("pageSize") ?? 1000);
const page = Number(u.searchParams.get("pageToken") ?? 0);
const end = Math.min(fake.total, (page + 1) * pageSize);
const dataPoints = [];
for (let g = page * pageSize; g < end; g++) {
dataPoints.push({
name: `users/me/dataTypes/heart-rate/dataPoints/${8_000_000_000_000 + g}`,
dataSource: {
recordingMethod: "PASSIVE_MEASUREMENT",
device: { manufacturer: "Google", displayName: "Watch" },
platform: "FITBIT",
},
heartRate: {
beatsPerMinute: String(52 + ((g * 7) % 61)),
sampleTime: {
physicalTime: new Date(fake.now - g * STEP_MS).toISOString(),
},
},
});
}
body = {
dataPoints,
...(end < fake.total ? { nextPageToken: String(page + 1) } : {}),
};
}
return new Response(JSON.stringify(body), {
status: 200,
headers: { "content-type": "application/json" },
});
},
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/jobs/boss-instance", () => ({
getGlobalBoss: vi.fn(() => null),
}));
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
const prisma = getPrismaClient();
function report(line: string): void {
process.stderr.write(`[large-account] ${line}\n`);
}
async function timed<T>(label: string, fn: () => Promise<T>): Promise<T> {
const started = Date.now();
const result = await fn();
report(`${label}: ${((Date.now() - started) / 1000).toFixed(1)} s`);
return result;
}
async function backfillToTheEnd(): Promise<number> {
const { runGoogleHealthBackfillForUser } =
await import("@/lib/jobs/google-health-backfill");
let imported = 0;
for (let slice = 0; slice < 20; slice++) {
// One slice for the whole stream: Google's API is faked here, and
// the slice budget is not what this file is about.
const result = await runGoogleHealthBackfillForUser(
USER,
() => false,
slice,
{ sliceMs: 1_800_000, slicePages: 100_000 },
);
imported += result.imported;
const connection = await prisma.googleHealthConnection.findUniqueOrThrow({
where: { userId: USER },
select: { backfillCompletedAt: true },
});
if (connection.backfillCompletedAt !== null) return imported;
}
throw new Error("the backfill never completed");
}
function pulseRows(where: Prisma.MeasurementWhereInput = {}) {
return prisma.measurement.count({
where: {
userId: USER,
type: "PULSE",
source: "GOOGLE_HEALTH",
deletedAt: null,
...where,
},
});
}
async function dayPulseRollups() {
return prisma.measurementRollup.findMany({
where: { userId: USER, type: "PULSE", granularity: "DAY" },
select: {
bucketStart: true,
source: true,
count: true,
minValue: true,
maxValue: true,
mean: true,
readingMinValue: true,
readingMaxValue: true,
},
orderBy: [{ bucketStart: "asc" }, { source: "asc" }],
});
}
beforeAll(async () => {
await truncateAllTables(prisma);
const { encrypt } = await import("@/lib/crypto");
await prisma.user.create({
data: { id: USER, username: USER, timezone: "Europe/Berlin" },
});
await prisma.googleHealthConnection.create({
data: {
userId: USER,
googleUserId: `google-${USER}`,
accessToken: encrypt("access"),
refreshToken: encrypt("refresh"),
tokenExpiresAt: new Date(Date.now() + 86_400_000),
},
});
fake.total = POINTS;
fake.now = Date.now() - 3_600_000;
}, 120_000);
afterAll(async () => {
await truncateAllTables(prisma);
});
describe("a large Google Health account (#1297, #1298)", () => {
it("backfills, refills the reading extremes, folds, and does not take folded samples back", async () => {
// ── The history arrives ───────────────────────────────────────────
const imported = await timed("backfill", backfillToTheEnd);
expect(imported).toBe(POINTS);
expect(await pulseRows()).toBe(POINTS);
// ── Rows folded before 0417 ───────────────────────────────────────
await prisma.$executeRaw`
UPDATE measurement_rollups
SET reading_min_value = NULL, reading_max_value = NULL
WHERE user_id = ${USER}
`;
const { enqueueBootTimeRollupBackfill, ROLLUP_FULL_BACKFILL_QUEUE } =
await import("@/lib/rollups/full-backfill");
const send = vi.fn(async () => "job-id");
vi.mocked(getGlobalBoss).mockReturnValue({ send } as never);
const discovery = await timed("discovery", () =>
enqueueBootTimeRollupBackfill(),
);
vi.mocked(getGlobalBoss).mockReturnValue(null as never);
expect(discovery).toMatchObject({ enqueued: 1, error: null });
const { handleRollupFullBackfill } =
await import("@/lib/jobs/reminder/rollup-handlers");
const job = {
id: "full-backfill",
name: ROLLUP_FULL_BACKFILL_QUEUE,
data: { userId: USER, enqueuedAt: new Date().toISOString() },
expireInSeconds: 3600,
signal: new AbortController().signal,
} as unknown as Job<{ userId: string; enqueuedAt: string }>;
const outcome = await timed("full backfill", () =>
handleRollupFullBackfill([job]),
);
expect(outcome.ok).toBe(true);
const { ROLLUP_FOLD_WINDOW_MS } =
await import("@/lib/rollups/measurement-rollups");
const floor = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS);
const before = await dayPulseRollups();
expect(before.length).toBeGreaterThan(1000);
expect(
before.filter(
(r) =>
r.bucketStart >= floor &&
(r.readingMinValue === null || r.readingMaxValue === null),
),
).toEqual([]);
// The extremes are the raw readings' own.
const raw = await prisma.$queryRaw<
Array<{ day: Date; min: number; max: number }>
>`
SELECT date_trunc('day', measured_at) AS day,
MIN(value)::double precision AS min,
MAX(value)::double precision AS max
FROM measurements
WHERE user_id = ${USER} AND type = 'PULSE' AND deleted_at IS NULL
GROUP BY 1
`;
const rawByDay = new Map(raw.map((r) => [r.day.getTime(), r]));
for (const row of before) {
const day = rawByDay.get(row.bucketStart.getTime());
expect(day).toBeDefined();
expect(row.readingMinValue).toBe(day!.min);
expect(row.readingMaxValue).toBe(day!.max);
}
// ── The fold thins the old samples ────────────────────────────────
const { runDenseIntradayRetention, DENSE_INTRADAY_RETENTION_DAYS } =
await import("@/lib/measurements/dense-intraday-retention");
const summary = await timed("dense fold", () =>
runDenseIntradayRetention(prisma, { userId: USER, log: () => {} }),
);
expect(summary.stoppedEarly).toBe(false);
const retained = new Date(
Date.now() - (DENSE_INTRADAY_RETENTION_DAYS + 2) * DAY_MS,
);
const rawOld = await pulseRows({
measuredAt: { lt: retained },
NOT: { externalId: { startsWith: "stats:" } },
});
const hourlyOld = await pulseRows({
measuredAt: { lt: retained },
externalId: { startsWith: "stats:" },
});
const afterFold = await pulseRows();
report(
`rows: ${POINTS} raw → ${afterFold} after the fold (${hourlyOld} hourly rows before the window)`,
);
expect(rawOld).toBe(0);
// At most one hourly row per hour, twelve samples an hour before.
expect(hourlyOld).toBeGreaterThan(0);
expect(afterFold).toBeLessThan(POINTS / 5);
// The DAY tier keeps the figures it took from the raw samples.
const after = await dayPulseRollups();
expect(after.map((r) => r.bucketStart.getTime())).toEqual(
before.map((r) => r.bucketStart.getTime()),
);
after.forEach((row, i) => {
expect(row.count).toBe(before[i].count);
expect(row.minValue).toBe(before[i].minValue);
expect(row.maxValue).toBe(before[i].maxValue);
expect(row.readingMinValue).toBe(before[i].readingMinValue);
expect(row.mean).toBeCloseTo(before[i].mean, 6);
});
// ── A second full-history backfill over the same samples ──────────
await prisma.googleHealthConnection.update({
where: { userId: USER },
data: {
backfillCompletedAt: null,
backfillProgress: Prisma.DbNull,
},
});
const again = await timed("second backfill", backfillToTheEnd);
expect(again).toBe(0);
expect(await pulseRows()).toBe(afterFold);
}, 1_800_000);
});
@@ -29,6 +29,8 @@ import { getPrismaClient, truncateAllTables } from "./setup";
import {
ROLLUP_FULL_BACKFILL_QUEUE,
enqueueBootTimeRollupBackfill,
} from "@/lib/rollups/full-backfill";
import {
recomputeBucketsForKeys,
recomputeBucketsForMeasurement,
recomputeUserRollups,
@@ -0,0 +1,185 @@
/**
* The dense hourly fold over the provider syncs that store heart rate one row
* per sample, and their ingest guard, against Postgres (#1297).
*
* - Google Health and Oura heart rate older than the retention window folds
* to hourly `stats:` rows under its own source, the raw rows go, and the
* DAY rollup keeps the figures it had from the raw samples.
* - A daily SpO2 value of the same sources is no intraday stream and stays.
* - Writing the same old samples again, as a backfill or a full sync does,
* inserts nothing and leaves every hourly mean as it was, including the
* hour whose anchor a sample fell on (Google Health's natural-key rescue
* used to re-key that row back into the sample).
* - Samples inside the window still write as before.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { upsertGoogleHealthMeasurements } from "@/lib/google-health/sync-core";
import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention";
import {
TouchedDays,
writeOuraHeartRatePage,
} from "@/lib/oura/sync-heart-rate";
import { makeUser } from "../kit/factories";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
const prisma = getPrismaClient();
const DAY_MS = 86_400_000;
const MIN_MS = 60_000;
beforeEach(async () => {
await truncateAllTables(prisma);
});
/** Noon UTC, `days` ago: every sample below stays inside that UTC day. */
function dayStart(days: number): number {
const now = Date.now();
return Math.floor(now / DAY_MS) * DAY_MS - days * DAY_MS;
}
/** Samples of 10:00-11:59 UTC, one every 10 minutes, 10:30 included. */
function samples(base: number): Array<{ at: Date; bpm: number }> {
const out: Array<{ at: Date; bpm: number }> = [];
for (let m = 0; m < 120; m += 10) {
out.push({
at: new Date(base + 10 * 60 * MIN_MS + m * MIN_MS),
bpm: 50 + ((m * 7) % 45),
});
}
return out;
}
const gh = (s: { at: Date; bpm: number }) => ({
type: "PULSE",
value: s.bpm,
unit: "bpm",
measuredAt: s.at,
externalId: `${s.at.getTime()}:hr`,
});
async function liveRows(userId: string, source: "GOOGLE_HEALTH" | "OURA") {
return prisma.measurement.findMany({
where: { userId, source, deletedAt: null },
select: { type: true, value: true, externalId: true, measuredAt: true },
orderBy: [{ type: "asc" }, { measuredAt: "asc" }],
});
}
describe("dense fold of provider heart rate (#1297)", () => {
it("folds old Google Health heart rate to hourly means, and a re-sync writes none of it back", async () => {
const user = await makeUser({ timezone: "UTC" });
const old = samples(dayStart(120));
const recent = samples(dayStart(5));
await upsertGoogleHealthMeasurements(user.id, old.map(gh));
// A nightly SpO2 average: one value a day, never folded.
await prisma.measurement.create({
data: {
userId: user.id,
type: "OXYGEN_SATURATION",
value: 96,
unit: "%",
source: "GOOGLE_HEALTH",
measuredAt: new Date(dayStart(120) + 7 * 3_600_000),
externalId: "2026-01-01:spo2",
},
});
const dayBefore = await prisma.measurementRollup.findFirstOrThrow({
where: { userId: user.id, type: "PULSE", granularity: "DAY" },
});
const summary = await runDenseIntradayRetention(prisma, {
userId: user.id,
log: () => {},
});
expect(summary.totals.perSampleRowsSoftDeleted).toBeGreaterThan(0);
const folded = await liveRows(user.id, "GOOGLE_HEALTH");
const pulse = folded.filter((r) => r.type === "PULSE");
// Two hours, two hourly means; no raw sample of them is left.
expect(pulse).toHaveLength(2);
for (const row of pulse) {
expect(row.externalId).toMatch(
/^stats:HKQuantityTypeIdentifierHeartRate:\d{4}-\d{2}-\d{2}T1[01]$/,
);
}
const firstHour = old.slice(0, 6).map((s) => s.bpm);
expect(pulse[0].value).toBeCloseTo(
firstHour.reduce((a, b) => a + b, 0) / firstHour.length,
6,
);
expect(folded.filter((r) => r.type === "OXYGEN_SATURATION")).toHaveLength(
1,
);
// The DAY bucket keeps the raw figures.
const dayAfter = await prisma.measurementRollup.findFirstOrThrow({
where: { userId: user.id, type: "PULSE", granularity: "DAY" },
});
expect(dayAfter.count).toBe(dayBefore.count);
expect(dayAfter.minValue).toBe(dayBefore.minValue);
expect(dayAfter.maxValue).toBe(dayBefore.maxValue);
expect(dayAfter.mean).toBeCloseTo(dayBefore.mean, 9);
// A backfill walks the same history again.
const again = await upsertGoogleHealthMeasurements(
user.id,
[...old, ...recent].map(gh),
);
expect(again.imported).toBe(recent.length);
const afterResync = await liveRows(user.id, "GOOGLE_HEALTH");
expect(afterResync.filter((r) => r.type === "PULSE")).toHaveLength(
2 + recent.length,
);
// The hourly means are untouched, the 10:30 one included.
expect(
afterResync.filter((r) => r.externalId?.startsWith("stats:")),
).toEqual(pulse);
});
it("folds old Oura heart rate the same way, and a re-read writes none of it back", async () => {
const user = await makeUser({ timezone: "UTC" });
const old = samples(dayStart(150)).map((s) => ({
bpm: s.bpm,
timestamp: s.at.toISOString(),
source: "awake",
}));
expect(
await writeOuraHeartRatePage(user.id, old, new TouchedDays(), {
emitArrivals: false,
}),
).toBe(old.length);
await runDenseIntradayRetention(prisma, {
userId: user.id,
log: () => {},
});
const folded = await liveRows(user.id, "OURA");
expect(folded).toHaveLength(2);
expect(folded.every((r) => r.externalId?.startsWith("stats:"))).toBe(true);
expect(
await writeOuraHeartRatePage(user.id, old, new TouchedDays(), {
emitArrivals: false,
}),
).toBe(0);
expect(await liveRows(user.id, "OURA")).toEqual(folded);
});
it("leaves heart rate inside the window raw", async () => {
const user = await makeUser({ timezone: "UTC" });
const recent = samples(dayStart(10));
await upsertGoogleHealthMeasurements(user.id, recent.map(gh));
await runDenseIntradayRetention(prisma, {
userId: user.id,
log: () => {},
});
expect(await liveRows(user.id, "GOOGLE_HEALTH")).toHaveLength(
recent.length,
);
});
});
@@ -0,0 +1,347 @@
/**
* The full backfill against Postgres (#1298).
*
* - The windowed fold (`foldUserRollupHistory`, a UTC calendar year per
* pass, newest first) writes exactly the rollup rows one pass of
* `recomputeUserRollups` over the whole five-year window writes: every
* granularity, every source, a week across New Year, the YEAR bucket the
* window's oldest edge snaps out to, and a reading older than the window.
* - A fold stopped part way and resumed with `before` ends in the same rows.
* - The boot discovery, rewritten to walk days instead of reading every
* reading in the window, finds the same accounts as the statement it
* replaced, kept verbatim below as the reference.
*/
import { PgBoss } from "pg-boss";
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from "vitest";
import { Prisma, type MeasurementSource } from "@/generated/prisma/client";
import { activeAccountSql } from "@/lib/account-deletion/active-account";
import {
enqueueBootTimeRollupBackfill,
foldUserRollupHistory,
isRollupFullBackfillPending,
ROLLUP_FULL_BACKFILL_QUEUE,
ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS,
} from "@/lib/rollups/full-backfill";
import {
recomputeUserRollups,
ROLLUP_FOLD_WINDOW_MS,
} from "@/lib/rollups/measurement-rollups";
import { makeUser } from "../kit/factories";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("@/lib/jobs/boss-instance", () => ({
getGlobalBoss: vi.fn(() => null),
}));
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
const prisma = getPrismaClient();
const DAY_MS = 86_400_000;
const HOUR_MS = 3_600_000;
beforeEach(async () => {
await truncateAllTables(prisma);
vi.mocked(getGlobalBoss).mockReturnValue(null as never);
});
function utc(y: number, m: number, d: number, h = 12, min = 0): Date {
return new Date(Date.UTC(y, m - 1, d, h, min));
}
/** Readings that can tell a windowed fold from one pass. */
async function seedHistory(userId: string): Promise<void> {
const now = new Date();
const year = now.getUTCFullYear();
const floor = new Date(now.getTime() - ROLLUP_FOLD_WINDOW_MS);
const rows: Prisma.MeasurementCreateManyInput[] = [];
const add = (
type: "WEIGHT" | "PULSE",
source: MeasurementSource,
at: Date,
value: number,
) =>
rows.push({
userId,
type,
source,
value,
unit: type === "WEIGHT" ? "kg" : "bpm",
measuredAt: at,
externalId: `${source}:${type}:${at.toISOString()}`,
});
const days: Date[] = [];
// A week across each of two New Years.
for (let d = 0; d < 9; d++) {
days.push(new Date(utc(year - 3, 12, 27).getTime() + d * DAY_MS));
days.push(new Date(utc(year - 1, 12, 28).getTime() + d * DAY_MS));
}
days.push(utc(year - 2, 6, 15), utc(year - 2, 6, 16));
days.push(new Date(now.getTime() - 3 * DAY_MS));
// Just inside the window, and inside the YEAR bucket its edge snaps to.
days.push(new Date(floor.getTime() + 2 * DAY_MS));
days.push(new Date(floor.getTime() - 20 * DAY_MS));
days.forEach((day, i) => {
const base = Date.UTC(
day.getUTCFullYear(),
day.getUTCMonth(),
day.getUTCDate(),
);
add("WEIGHT", "MANUAL", new Date(base + 7 * HOUR_MS), 80 + (i % 5) / 3);
if (i % 2 === 0) {
add("WEIGHT", "WITHINGS", new Date(base + 8 * HOUR_MS), 81 + i / 7);
}
for (let k = 0; k < 6; k++) {
add(
"PULSE",
"APPLE_HEALTH",
new Date(base + (k < 4 ? 9 : 15) * HOUR_MS + k * 7 * 60_000),
55 + ((i * 7 + k * 11) % 40),
);
}
add("PULSE", "GOOGLE_HEALTH", new Date(base + 23 * HOUR_MS), 60 + i);
});
await prisma.measurement.createMany({ data: rows });
}
async function snapshot(userId: string) {
const rows = await prisma.measurementRollup.findMany({
where: { userId },
orderBy: [
{ type: "asc" },
{ granularity: "asc" },
{ bucketStart: "asc" },
{ source: "asc" },
],
});
return rows.map(({ computedAt, ...rest }) => {
void computedAt;
return rest;
});
}
function expectSameRows(
actual: Awaited<ReturnType<typeof snapshot>>,
expected: Awaited<ReturnType<typeof snapshot>>,
): void {
expect(actual.length).toBe(expected.length);
expected.forEach((want, i) => {
const got = actual[i] as Record<string, unknown>;
for (const [key, value] of Object.entries(want)) {
if (typeof value === "number" && key !== "count") {
// A week across two windows is read in another range, so its sums
// may be added in another order: equal to rounding, not to the bit.
expect(got[key] as number).toBeCloseTo(value, 6);
} else {
expect(got[key]).toEqual(value);
}
}
});
}
describe("foldUserRollupHistory", () => {
it("writes the rows one pass over the whole window writes", async () => {
const user = await makeUser();
await seedHistory(user.id);
await recomputeUserRollups(user.id);
const onePass = await snapshot(user.id);
expect(onePass.length).toBeGreaterThan(50);
expect(onePass.some((r) => r.granularity === "WEEK")).toBe(true);
await prisma.measurementRollup.deleteMany({ where: { userId: user.id } });
const done = await foldUserRollupHistory(user.id, {
statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS,
});
expect(done.windowsFolded).toBeGreaterThanOrEqual(5);
expect(done.resumeBefore).toBeUndefined();
expectSameRows(await snapshot(user.id), onePass);
expect(
(await snapshot(user.id)).every(
(r) => r.readingMinValue !== null && r.readingMaxValue !== null,
),
).toBe(true);
});
it("ends in the same rows when stopped part way and resumed", async () => {
const user = await makeUser();
await seedHistory(user.id);
await recomputeUserRollups(user.id);
const onePass = await snapshot(user.id);
await prisma.measurementRollup.deleteMany({ where: { userId: user.id } });
// Stop on the seventh question: inside the second window.
let asked = 0;
const first = await foldUserRollupHistory(user.id, {
shouldStop: () => ++asked >= 7,
});
expect(first.windowsFolded).toBe(1);
expect(first.resumeBefore).toBeDefined();
const rest = await foldUserRollupHistory(user.id, {
before: first.resumeBefore,
});
expect(rest.resumeBefore).toBeUndefined();
expectSameRows(await snapshot(user.id), onePass);
});
});
/** The discovery statement before the day walk, verbatim. */
async function referenceDiscovery(): Promise<string[]> {
const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS);
const users = await prisma.$queryRaw<Array<{ id: string }>>`
SELECT DISTINCT id FROM (
SELECT DISTINCT mt."user_id" AS id
FROM (
SELECT DISTINCT
m."user_id",
m."type",
date_trunc('day', m."measured_at") AS bucket_start
FROM measurements m
WHERE m."deleted_at" IS NULL
AND m."measured_at" >= ${foldWindowStart}
) mt
LEFT JOIN measurement_rollups r
ON r."user_id" = mt."user_id"
AND r."type" = mt."type"
AND r."granularity" = 'DAY'
AND r."bucket_start" = mt."bucket_start"
WHERE r."bucket_start" IS NULL
UNION
SELECT DISTINCT r."user_id" AS id
FROM measurement_rollups r
WHERE r."granularity" = 'DAY'
AND r."bucket_start" >= ${foldWindowStart}
AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL)
) discovered
WHERE ${activeAccountSql('discovered."id"')}
`;
return users.map((u) => u.id).sort();
}
describe("enqueueBootTimeRollupBackfill", () => {
it("finds the accounts the statement it replaced found", async () => {
const covered = await makeUser();
const missingDay = await makeUser();
const nullExtremes = await makeUser();
const tombstoneOnly = await makeUser();
const tooOld = await makeUser();
const pendingDeletion = await makeUser();
for (const u of [covered, missingDay, nullExtremes, pendingDeletion]) {
await seedHistory(u.id);
await recomputeUserRollups(u.id);
}
// A reading on a day with no DAY bucket.
await prisma.measurement.create({
data: {
userId: missingDay.id,
type: "BODY_FAT",
value: 20,
unit: "%",
source: "MANUAL",
measuredAt: new Date(Date.now() - 40 * DAY_MS),
},
});
await prisma.measurement.create({
data: {
userId: pendingDeletion.id,
type: "BODY_FAT",
value: 20,
unit: "%",
source: "MANUAL",
measuredAt: new Date(Date.now() - 40 * DAY_MS),
},
});
await prisma.user.update({
where: { id: pendingDeletion.id },
data: { deletionRequestedAt: new Date() },
});
// A row folded before 0417 added the reading extremes.
await prisma.$executeRaw`
UPDATE measurement_rollups SET reading_min_value = NULL
WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY'
AND bucket_start = (
SELECT max(bucket_start) FROM measurement_rollups
WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY'
)
`;
// Only a tombstone inside the window: nothing to fold.
await prisma.measurement.create({
data: {
userId: tombstoneOnly.id,
type: "WEIGHT",
value: 70,
unit: "kg",
source: "MANUAL",
measuredAt: new Date(Date.now() - 10 * DAY_MS),
deletedAt: new Date(),
},
});
// Only readings older than the window: the fold writes nothing there.
await prisma.measurement.create({
data: {
userId: tooOld.id,
type: "WEIGHT",
value: 70,
unit: "kg",
source: "MANUAL",
measuredAt: new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS - 5 * DAY_MS),
},
});
const send = vi.fn(async () => "job-id");
vi.mocked(getGlobalBoss).mockReturnValue({ send } as never);
const result = await enqueueBootTimeRollupBackfill();
expect(result.error).toBeNull();
const found = (
send.mock.calls as unknown as Array<[string, { userId: string }]>
)
.map((call) => call[1].userId)
.sort();
expect(found).toEqual([missingDay.id, nullExtremes.id].sort());
expect(found).toEqual(await referenceDiscovery());
});
});
describe("isRollupFullBackfillPending", () => {
let boss: PgBoss;
beforeAll(async () => {
boss = new PgBoss({
connectionString: process.env.DATABASE_URL,
supervise: false,
schedule: false,
});
await boss.start();
await boss.createQueue(ROLLUP_FULL_BACKFILL_QUEUE);
}, 60_000);
afterAll(async () => {
await boss?.stop({ graceful: false, close: true });
await prisma.$executeRawUnsafe(`DROP SCHEMA IF EXISTS pgboss CASCADE`);
});
it("is true while the account's full fold is queued or running, and only for that account", async () => {
expect(await isRollupFullBackfillPending("user-a")).toBe(false);
await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, { userId: "user-a" });
expect(await isRollupFullBackfillPending("user-a")).toBe(true);
expect(await isRollupFullBackfillPending("user-b")).toBe(false);
const [job] = await boss.fetch(ROLLUP_FULL_BACKFILL_QUEUE);
expect(await isRollupFullBackfillPending("user-a")).toBe(true);
await boss.complete(ROLLUP_FULL_BACKFILL_QUEUE, job.id);
expect(await isRollupFullBackfillPending("user-a")).toBe(false);
});
});
+178
View File
@@ -0,0 +1,178 @@
/**
* The system job alert against a ledger pg-boss itself wrote (#1298).
*
* A background job of the instance that keeps failing pushes every admin
* once, a reminder at most once a day while it keeps failing, and once when it
* runs successfully again. Fewer failures than the streak, and queues that
* have an account-facing surface of their own, push nothing.
*
* pg-boss provisions its own schema here and the jobs are failed and completed
* through its own API, as in `job-failures.test.ts`, so the reader is proven
* against the table pg-boss writes rather than one the test wrote.
*/
import { PgBoss } from "pg-boss";
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from "vitest";
const dispatched = vi.hoisted(() => ({
calls: [] as Array<{
userId: string;
titleKey: string;
messageKey: string;
params?: Record<string, string | number>;
eventType?: string;
}>,
}));
vi.mock("@/lib/notifications/dispatch-localised", () => ({
dispatchLocalisedNotification: vi.fn(
async (opts: (typeof dispatched.calls)[number]) => {
dispatched.calls.push(opts);
},
),
}));
import {
notifySystemJobFailures,
SYSTEM_JOB_FAILURE_STREAK,
} from "@/lib/jobs/system-job-alert";
import { makeUser } from "../kit/factories";
import { getPrismaClient, truncateAllTables } from "./setup";
const SYSTEM_QUEUE = "rollup-full-backfill";
const ACCOUNT_QUEUE = "google-health-sync";
const HOUR_MS = 3_600_000;
let boss: PgBoss;
async function runOnce(queue: string, outcome: "fail" | "complete") {
const id = await boss.send(queue, { userId: "u" }, { retryLimit: 0 });
if (!id) throw new Error("queue rejected the job");
const [job] = await boss.fetch(queue);
if (!job) throw new Error("nothing to fetch");
if (outcome === "fail") {
await boss.fail(
queue,
job.id,
new Error("canceling statement due to statement timeout"),
);
} else {
await boss.complete(queue, job.id, { ok: true });
}
}
beforeAll(async () => {
boss = new PgBoss({
connectionString: process.env.DATABASE_URL,
supervise: false,
schedule: false,
});
await boss.start();
await boss.createQueue(SYSTEM_QUEUE);
await boss.createQueue(ACCOUNT_QUEUE);
}, 60_000);
afterAll(async () => {
await boss?.stop({ graceful: false, close: true });
await getPrismaClient().$executeRawUnsafe(
`DROP SCHEMA IF EXISTS pgboss CASCADE`,
);
});
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
await getPrismaClient().$executeRawUnsafe(`DELETE FROM pgboss.job`);
dispatched.calls.length = 0;
});
describe("notifySystemJobFailures", () => {
it("tells every admin once, reminds a day later, and announces the recovery once", async () => {
const admin = await makeUser({ role: "ADMIN" });
const second = await makeUser({ role: "ADMIN" });
await makeUser({ role: "USER" });
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK; i++) {
await runOnce(SYSTEM_QUEUE, "fail");
}
const now = new Date();
const first = await notifySystemJobFailures(now);
expect(first.failing).toBe(1);
expect(first.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "problem" }]);
expect(dispatched.calls.map((c) => c.userId).sort()).toEqual(
[admin.id, second.id].sort(),
);
const call = dispatched.calls[0];
expect(call.eventType).toBe("SYSTEM_ALERT");
expect(call.titleKey).toBe("notifications.systemJobAlert.titleFailing");
expect(call.params).toMatchObject({
queue: SYSTEM_QUEUE,
failures: SYSTEM_JOB_FAILURE_STREAK,
});
expect(String(call.params?.error)).toContain("statement timeout");
// The next hour: still failing, already told.
dispatched.calls.length = 0;
const again = await notifySystemJobFailures(
new Date(now.getTime() + HOUR_MS),
);
expect(again.sent).toEqual([]);
expect(dispatched.calls).toEqual([]);
// A day later: one reminder.
const reminder = await notifySystemJobFailures(
new Date(now.getTime() + 25 * HOUR_MS),
);
expect(reminder.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "reminder" }]);
// It runs again: one recovery notice, then silence.
await runOnce(SYSTEM_QUEUE, "complete");
dispatched.calls.length = 0;
const recovered = await notifySystemJobFailures(
new Date(now.getTime() + 26 * HOUR_MS),
);
expect(recovered.sent).toEqual([
{ queue: SYSTEM_QUEUE, kind: "recovered" },
]);
expect(dispatched.calls[0].titleKey).toBe(
"notifications.systemJobAlert.titleRecovered",
);
const quiet = await notifySystemJobFailures(
new Date(now.getTime() + 27 * HOUR_MS),
);
expect(quiet.sent).toEqual([]);
});
it("stays quiet below the streak and for queues with an account-facing surface", async () => {
await makeUser({ role: "ADMIN" });
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK - 1; i++) {
await runOnce(SYSTEM_QUEUE, "fail");
}
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK + 2; i++) {
await runOnce(ACCOUNT_QUEUE, "fail");
}
const outcome = await notifySystemJobFailures();
expect(outcome.failing).toBe(0);
expect(outcome.sent).toEqual([]);
expect(dispatched.calls).toEqual([]);
});
it("counts a success between failures as the end of the streak", async () => {
await makeUser({ role: "ADMIN" });
await runOnce(SYSTEM_QUEUE, "fail");
await runOnce(SYSTEM_QUEUE, "fail");
await runOnce(SYSTEM_QUEUE, "complete");
await runOnce(SYSTEM_QUEUE, "fail");
const outcome = await notifySystemJobFailures();
expect(outcome.failing).toBe(0);
expect(outcome.sent).toEqual([]);
});
});