mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 00:12:18 +00:00
Merge branch 'feat/v1462-scale' into release/v1.46.2
# Conflicts: # src/__tests__/baselines/version-stamp-comments.json
This commit is contained in:
commit
ad5d154561
57 files changed
+2736
-425
No files matched your search
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"registeredQueueCount": 125,
|
||||
"registeredQueueCount": 126,
|
||||
"registeredQueues": [
|
||||
"account-deletion-purge",
|
||||
"achievement-unlock-sweep",
|
||||
@@ -112,6 +112,7 @@
|
||||
"strava-sync",
|
||||
"stress-score-compute",
|
||||
"supplement-nutrient-recompute",
|
||||
"system-job-alert",
|
||||
"tls-pin-monitor",
|
||||
"whoop-backfill",
|
||||
"whoop-cycle-sync",
|
||||
@@ -1793,6 +1794,21 @@
|
||||
"retry_delay_max": null,
|
||||
"retry_limit": 2
|
||||
},
|
||||
{
|
||||
"dead_letter": null,
|
||||
"deletion_seconds": 604800,
|
||||
"expire_seconds": 900,
|
||||
"heartbeat_seconds": null,
|
||||
"name": "system-job-alert",
|
||||
"notify": false,
|
||||
"partition": false,
|
||||
"policy": "standard",
|
||||
"retention_seconds": 1209600,
|
||||
"retry_backoff": false,
|
||||
"retry_delay": 0,
|
||||
"retry_delay_max": null,
|
||||
"retry_limit": 2
|
||||
},
|
||||
{
|
||||
"dead_letter": null,
|
||||
"deletion_seconds": 604800,
|
||||
@@ -2825,6 +2841,18 @@
|
||||
},
|
||||
"timezone": "Europe/Berlin"
|
||||
},
|
||||
{
|
||||
"cron": "43 * * * *",
|
||||
"data": {},
|
||||
"key": "",
|
||||
"kind": "cron",
|
||||
"name": "system-job-alert",
|
||||
"options": {
|
||||
"retryLimit": 0,
|
||||
"tz": "Europe/Berlin"
|
||||
},
|
||||
"timezone": "Europe/Berlin"
|
||||
},
|
||||
{
|
||||
"cron": "7 */6 * * *",
|
||||
"data": {},
|
||||
@@ -3682,6 +3710,13 @@
|
||||
"localConcurrency": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"queue": "system-job-alert",
|
||||
"options": {
|
||||
"localConcurrency": 1,
|
||||
"pollingIntervalSeconds": 30
|
||||
}
|
||||
},
|
||||
{
|
||||
"queue": "tls-pin-monitor",
|
||||
"options": {
|
||||
|
||||
@@ -12,10 +12,8 @@ import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
import {
|
||||
enqueueBootTimeRollupBackfill,
|
||||
recomputeUserRollups,
|
||||
} from "@/lib/rollups/measurement-rollups";
|
||||
import { enqueueBootTimeRollupBackfill } from "@/lib/rollups/full-backfill";
|
||||
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* A large Google Health account end to end (#1297, #1298).
|
||||
*
|
||||
* One account with a dense heart-rate history, written through the real
|
||||
* Google Health backfill (Google's API replaced at the `safeFetch` seam, as in
|
||||
* `google-health-dense-backfill-memory.test.ts`), then taken through what a
|
||||
* self-hosted instance with such an account went through:
|
||||
*
|
||||
* 1. The account's DAY rollups lose their reading extremes, as rows folded
|
||||
* before migration 0417 have them. The boot discovery finds the account,
|
||||
* and the full backfill, a calendar year per pass with every statement
|
||||
* under its own limit, refills every bucket inside the fold window.
|
||||
* 2. The dense fold thins the heart rate older than the retention window to
|
||||
* hourly `stats:` rows of Google Health, and the DAY rollups keep the
|
||||
* figures they had from the raw samples.
|
||||
* 3. A second full-history backfill over the same samples inserts nothing:
|
||||
* the folded hours are not written again.
|
||||
*
|
||||
* Each step reports its duration on stderr (`[large-account]`); the size is
|
||||
* `POINTS` readings, one every five minutes, about four years.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
const USER = "large-account-google-health";
|
||||
const POINTS = 420_000;
|
||||
const STEP_MS = 5 * 60_000;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
const fake = vi.hoisted(() => ({ total: 0, now: 0 }));
|
||||
|
||||
vi.mock("@/lib/safe-fetch", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/safe-fetch")>();
|
||||
return {
|
||||
...actual,
|
||||
safeFetch: async (url: string) => {
|
||||
const u = new URL(url);
|
||||
let body: unknown = {};
|
||||
if (u.pathname.endsWith("/dataTypes/heart-rate/dataPoints")) {
|
||||
const pageSize = Number(u.searchParams.get("pageSize") ?? 1000);
|
||||
const page = Number(u.searchParams.get("pageToken") ?? 0);
|
||||
const end = Math.min(fake.total, (page + 1) * pageSize);
|
||||
const dataPoints = [];
|
||||
for (let g = page * pageSize; g < end; g++) {
|
||||
dataPoints.push({
|
||||
name: `users/me/dataTypes/heart-rate/dataPoints/${8_000_000_000_000 + g}`,
|
||||
dataSource: {
|
||||
recordingMethod: "PASSIVE_MEASUREMENT",
|
||||
device: { manufacturer: "Google", displayName: "Watch" },
|
||||
platform: "FITBIT",
|
||||
},
|
||||
heartRate: {
|
||||
beatsPerMinute: String(52 + ((g * 7) % 61)),
|
||||
sampleTime: {
|
||||
physicalTime: new Date(fake.now - g * STEP_MS).toISOString(),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
body = {
|
||||
dataPoints,
|
||||
...(end < fake.total ? { nextPageToken: String(page + 1) } : {}),
|
||||
};
|
||||
}
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({
|
||||
getGlobalBoss: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
|
||||
function report(line: string): void {
|
||||
process.stderr.write(`[large-account] ${line}\n`);
|
||||
}
|
||||
|
||||
async function timed<T>(label: string, fn: () => Promise<T>): Promise<T> {
|
||||
const started = Date.now();
|
||||
const result = await fn();
|
||||
report(`${label}: ${((Date.now() - started) / 1000).toFixed(1)} s`);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function backfillToTheEnd(): Promise<number> {
|
||||
const { runGoogleHealthBackfillForUser } =
|
||||
await import("@/lib/jobs/google-health-backfill");
|
||||
let imported = 0;
|
||||
for (let slice = 0; slice < 20; slice++) {
|
||||
// One slice for the whole stream: Google's API is faked here, and
|
||||
// the slice budget is not what this file is about.
|
||||
const result = await runGoogleHealthBackfillForUser(
|
||||
USER,
|
||||
() => false,
|
||||
slice,
|
||||
{ sliceMs: 1_800_000, slicePages: 100_000 },
|
||||
);
|
||||
imported += result.imported;
|
||||
const connection = await prisma.googleHealthConnection.findUniqueOrThrow({
|
||||
where: { userId: USER },
|
||||
select: { backfillCompletedAt: true },
|
||||
});
|
||||
if (connection.backfillCompletedAt !== null) return imported;
|
||||
}
|
||||
throw new Error("the backfill never completed");
|
||||
}
|
||||
|
||||
function pulseRows(where: Prisma.MeasurementWhereInput = {}) {
|
||||
return prisma.measurement.count({
|
||||
where: {
|
||||
userId: USER,
|
||||
type: "PULSE",
|
||||
source: "GOOGLE_HEALTH",
|
||||
deletedAt: null,
|
||||
...where,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function dayPulseRollups() {
|
||||
return prisma.measurementRollup.findMany({
|
||||
where: { userId: USER, type: "PULSE", granularity: "DAY" },
|
||||
select: {
|
||||
bucketStart: true,
|
||||
source: true,
|
||||
count: true,
|
||||
minValue: true,
|
||||
maxValue: true,
|
||||
mean: true,
|
||||
readingMinValue: true,
|
||||
readingMaxValue: true,
|
||||
},
|
||||
orderBy: [{ bucketStart: "asc" }, { source: "asc" }],
|
||||
});
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
const { encrypt } = await import("@/lib/crypto");
|
||||
await prisma.user.create({
|
||||
data: { id: USER, username: USER, timezone: "Europe/Berlin" },
|
||||
});
|
||||
await prisma.googleHealthConnection.create({
|
||||
data: {
|
||||
userId: USER,
|
||||
googleUserId: `google-${USER}`,
|
||||
accessToken: encrypt("access"),
|
||||
refreshToken: encrypt("refresh"),
|
||||
tokenExpiresAt: new Date(Date.now() + 86_400_000),
|
||||
},
|
||||
});
|
||||
fake.total = POINTS;
|
||||
fake.now = Date.now() - 3_600_000;
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
});
|
||||
|
||||
describe("a large Google Health account (#1297, #1298)", () => {
|
||||
it("backfills, refills the reading extremes, folds, and does not take folded samples back", async () => {
|
||||
// ── The history arrives ───────────────────────────────────────────
|
||||
const imported = await timed("backfill", backfillToTheEnd);
|
||||
expect(imported).toBe(POINTS);
|
||||
expect(await pulseRows()).toBe(POINTS);
|
||||
|
||||
// ── Rows folded before 0417 ───────────────────────────────────────
|
||||
await prisma.$executeRaw`
|
||||
UPDATE measurement_rollups
|
||||
SET reading_min_value = NULL, reading_max_value = NULL
|
||||
WHERE user_id = ${USER}
|
||||
`;
|
||||
const { enqueueBootTimeRollupBackfill, ROLLUP_FULL_BACKFILL_QUEUE } =
|
||||
await import("@/lib/rollups/full-backfill");
|
||||
const send = vi.fn(async () => "job-id");
|
||||
vi.mocked(getGlobalBoss).mockReturnValue({ send } as never);
|
||||
const discovery = await timed("discovery", () =>
|
||||
enqueueBootTimeRollupBackfill(),
|
||||
);
|
||||
vi.mocked(getGlobalBoss).mockReturnValue(null as never);
|
||||
expect(discovery).toMatchObject({ enqueued: 1, error: null });
|
||||
|
||||
const { handleRollupFullBackfill } =
|
||||
await import("@/lib/jobs/reminder/rollup-handlers");
|
||||
const job = {
|
||||
id: "full-backfill",
|
||||
name: ROLLUP_FULL_BACKFILL_QUEUE,
|
||||
data: { userId: USER, enqueuedAt: new Date().toISOString() },
|
||||
expireInSeconds: 3600,
|
||||
signal: new AbortController().signal,
|
||||
} as unknown as Job<{ userId: string; enqueuedAt: string }>;
|
||||
const outcome = await timed("full backfill", () =>
|
||||
handleRollupFullBackfill([job]),
|
||||
);
|
||||
expect(outcome.ok).toBe(true);
|
||||
|
||||
const { ROLLUP_FOLD_WINDOW_MS } =
|
||||
await import("@/lib/rollups/measurement-rollups");
|
||||
const floor = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS);
|
||||
const before = await dayPulseRollups();
|
||||
expect(before.length).toBeGreaterThan(1000);
|
||||
expect(
|
||||
before.filter(
|
||||
(r) =>
|
||||
r.bucketStart >= floor &&
|
||||
(r.readingMinValue === null || r.readingMaxValue === null),
|
||||
),
|
||||
).toEqual([]);
|
||||
// The extremes are the raw readings' own.
|
||||
const raw = await prisma.$queryRaw<
|
||||
Array<{ day: Date; min: number; max: number }>
|
||||
>`
|
||||
SELECT date_trunc('day', measured_at) AS day,
|
||||
MIN(value)::double precision AS min,
|
||||
MAX(value)::double precision AS max
|
||||
FROM measurements
|
||||
WHERE user_id = ${USER} AND type = 'PULSE' AND deleted_at IS NULL
|
||||
GROUP BY 1
|
||||
`;
|
||||
const rawByDay = new Map(raw.map((r) => [r.day.getTime(), r]));
|
||||
for (const row of before) {
|
||||
const day = rawByDay.get(row.bucketStart.getTime());
|
||||
expect(day).toBeDefined();
|
||||
expect(row.readingMinValue).toBe(day!.min);
|
||||
expect(row.readingMaxValue).toBe(day!.max);
|
||||
}
|
||||
|
||||
// ── The fold thins the old samples ────────────────────────────────
|
||||
const { runDenseIntradayRetention, DENSE_INTRADAY_RETENTION_DAYS } =
|
||||
await import("@/lib/measurements/dense-intraday-retention");
|
||||
const summary = await timed("dense fold", () =>
|
||||
runDenseIntradayRetention(prisma, { userId: USER, log: () => {} }),
|
||||
);
|
||||
expect(summary.stoppedEarly).toBe(false);
|
||||
const retained = new Date(
|
||||
Date.now() - (DENSE_INTRADAY_RETENTION_DAYS + 2) * DAY_MS,
|
||||
);
|
||||
const rawOld = await pulseRows({
|
||||
measuredAt: { lt: retained },
|
||||
NOT: { externalId: { startsWith: "stats:" } },
|
||||
});
|
||||
const hourlyOld = await pulseRows({
|
||||
measuredAt: { lt: retained },
|
||||
externalId: { startsWith: "stats:" },
|
||||
});
|
||||
const afterFold = await pulseRows();
|
||||
report(
|
||||
`rows: ${POINTS} raw → ${afterFold} after the fold (${hourlyOld} hourly rows before the window)`,
|
||||
);
|
||||
expect(rawOld).toBe(0);
|
||||
// At most one hourly row per hour, twelve samples an hour before.
|
||||
expect(hourlyOld).toBeGreaterThan(0);
|
||||
expect(afterFold).toBeLessThan(POINTS / 5);
|
||||
|
||||
// The DAY tier keeps the figures it took from the raw samples.
|
||||
const after = await dayPulseRollups();
|
||||
expect(after.map((r) => r.bucketStart.getTime())).toEqual(
|
||||
before.map((r) => r.bucketStart.getTime()),
|
||||
);
|
||||
after.forEach((row, i) => {
|
||||
expect(row.count).toBe(before[i].count);
|
||||
expect(row.minValue).toBe(before[i].minValue);
|
||||
expect(row.maxValue).toBe(before[i].maxValue);
|
||||
expect(row.readingMinValue).toBe(before[i].readingMinValue);
|
||||
expect(row.mean).toBeCloseTo(before[i].mean, 6);
|
||||
});
|
||||
|
||||
// ── A second full-history backfill over the same samples ──────────
|
||||
await prisma.googleHealthConnection.update({
|
||||
where: { userId: USER },
|
||||
data: {
|
||||
backfillCompletedAt: null,
|
||||
backfillProgress: Prisma.DbNull,
|
||||
},
|
||||
});
|
||||
const again = await timed("second backfill", backfillToTheEnd);
|
||||
expect(again).toBe(0);
|
||||
expect(await pulseRows()).toBe(afterFold);
|
||||
}, 1_800_000);
|
||||
});
|
||||
@@ -29,6 +29,8 @@ import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import {
|
||||
ROLLUP_FULL_BACKFILL_QUEUE,
|
||||
enqueueBootTimeRollupBackfill,
|
||||
} from "@/lib/rollups/full-backfill";
|
||||
import {
|
||||
recomputeBucketsForKeys,
|
||||
recomputeBucketsForMeasurement,
|
||||
recomputeUserRollups,
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* The dense hourly fold over the provider syncs that store heart rate one row
|
||||
* per sample, and their ingest guard, against Postgres (#1297).
|
||||
*
|
||||
* - Google Health and Oura heart rate older than the retention window folds
|
||||
* to hourly `stats:` rows under its own source, the raw rows go, and the
|
||||
* DAY rollup keeps the figures it had from the raw samples.
|
||||
* - A daily SpO2 value of the same sources is no intraday stream and stays.
|
||||
* - Writing the same old samples again, as a backfill or a full sync does,
|
||||
* inserts nothing and leaves every hourly mean as it was, including the
|
||||
* hour whose anchor a sample fell on (Google Health's natural-key rescue
|
||||
* used to re-key that row back into the sample).
|
||||
* - Samples inside the window still write as before.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { upsertGoogleHealthMeasurements } from "@/lib/google-health/sync-core";
|
||||
import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention";
|
||||
import {
|
||||
TouchedDays,
|
||||
writeOuraHeartRatePage,
|
||||
} from "@/lib/oura/sync-heart-rate";
|
||||
|
||||
import { makeUser } from "../kit/factories";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
const DAY_MS = 86_400_000;
|
||||
const MIN_MS = 60_000;
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
});
|
||||
|
||||
/** Noon UTC, `days` ago: every sample below stays inside that UTC day. */
|
||||
function dayStart(days: number): number {
|
||||
const now = Date.now();
|
||||
return Math.floor(now / DAY_MS) * DAY_MS - days * DAY_MS;
|
||||
}
|
||||
|
||||
/** Samples of 10:00-11:59 UTC, one every 10 minutes, 10:30 included. */
|
||||
function samples(base: number): Array<{ at: Date; bpm: number }> {
|
||||
const out: Array<{ at: Date; bpm: number }> = [];
|
||||
for (let m = 0; m < 120; m += 10) {
|
||||
out.push({
|
||||
at: new Date(base + 10 * 60 * MIN_MS + m * MIN_MS),
|
||||
bpm: 50 + ((m * 7) % 45),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const gh = (s: { at: Date; bpm: number }) => ({
|
||||
type: "PULSE",
|
||||
value: s.bpm,
|
||||
unit: "bpm",
|
||||
measuredAt: s.at,
|
||||
externalId: `${s.at.getTime()}:hr`,
|
||||
});
|
||||
|
||||
async function liveRows(userId: string, source: "GOOGLE_HEALTH" | "OURA") {
|
||||
return prisma.measurement.findMany({
|
||||
where: { userId, source, deletedAt: null },
|
||||
select: { type: true, value: true, externalId: true, measuredAt: true },
|
||||
orderBy: [{ type: "asc" }, { measuredAt: "asc" }],
|
||||
});
|
||||
}
|
||||
|
||||
describe("dense fold of provider heart rate (#1297)", () => {
|
||||
it("folds old Google Health heart rate to hourly means, and a re-sync writes none of it back", async () => {
|
||||
const user = await makeUser({ timezone: "UTC" });
|
||||
const old = samples(dayStart(120));
|
||||
const recent = samples(dayStart(5));
|
||||
await upsertGoogleHealthMeasurements(user.id, old.map(gh));
|
||||
// A nightly SpO2 average: one value a day, never folded.
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: user.id,
|
||||
type: "OXYGEN_SATURATION",
|
||||
value: 96,
|
||||
unit: "%",
|
||||
source: "GOOGLE_HEALTH",
|
||||
measuredAt: new Date(dayStart(120) + 7 * 3_600_000),
|
||||
externalId: "2026-01-01:spo2",
|
||||
},
|
||||
});
|
||||
const dayBefore = await prisma.measurementRollup.findFirstOrThrow({
|
||||
where: { userId: user.id, type: "PULSE", granularity: "DAY" },
|
||||
});
|
||||
|
||||
const summary = await runDenseIntradayRetention(prisma, {
|
||||
userId: user.id,
|
||||
log: () => {},
|
||||
});
|
||||
expect(summary.totals.perSampleRowsSoftDeleted).toBeGreaterThan(0);
|
||||
|
||||
const folded = await liveRows(user.id, "GOOGLE_HEALTH");
|
||||
const pulse = folded.filter((r) => r.type === "PULSE");
|
||||
// Two hours, two hourly means; no raw sample of them is left.
|
||||
expect(pulse).toHaveLength(2);
|
||||
for (const row of pulse) {
|
||||
expect(row.externalId).toMatch(
|
||||
/^stats:HKQuantityTypeIdentifierHeartRate:\d{4}-\d{2}-\d{2}T1[01]$/,
|
||||
);
|
||||
}
|
||||
const firstHour = old.slice(0, 6).map((s) => s.bpm);
|
||||
expect(pulse[0].value).toBeCloseTo(
|
||||
firstHour.reduce((a, b) => a + b, 0) / firstHour.length,
|
||||
6,
|
||||
);
|
||||
expect(folded.filter((r) => r.type === "OXYGEN_SATURATION")).toHaveLength(
|
||||
1,
|
||||
);
|
||||
// The DAY bucket keeps the raw figures.
|
||||
const dayAfter = await prisma.measurementRollup.findFirstOrThrow({
|
||||
where: { userId: user.id, type: "PULSE", granularity: "DAY" },
|
||||
});
|
||||
expect(dayAfter.count).toBe(dayBefore.count);
|
||||
expect(dayAfter.minValue).toBe(dayBefore.minValue);
|
||||
expect(dayAfter.maxValue).toBe(dayBefore.maxValue);
|
||||
expect(dayAfter.mean).toBeCloseTo(dayBefore.mean, 9);
|
||||
|
||||
// A backfill walks the same history again.
|
||||
const again = await upsertGoogleHealthMeasurements(
|
||||
user.id,
|
||||
[...old, ...recent].map(gh),
|
||||
);
|
||||
expect(again.imported).toBe(recent.length);
|
||||
const afterResync = await liveRows(user.id, "GOOGLE_HEALTH");
|
||||
expect(afterResync.filter((r) => r.type === "PULSE")).toHaveLength(
|
||||
2 + recent.length,
|
||||
);
|
||||
// The hourly means are untouched, the 10:30 one included.
|
||||
expect(
|
||||
afterResync.filter((r) => r.externalId?.startsWith("stats:")),
|
||||
).toEqual(pulse);
|
||||
});
|
||||
|
||||
it("folds old Oura heart rate the same way, and a re-read writes none of it back", async () => {
|
||||
const user = await makeUser({ timezone: "UTC" });
|
||||
const old = samples(dayStart(150)).map((s) => ({
|
||||
bpm: s.bpm,
|
||||
timestamp: s.at.toISOString(),
|
||||
source: "awake",
|
||||
}));
|
||||
expect(
|
||||
await writeOuraHeartRatePage(user.id, old, new TouchedDays(), {
|
||||
emitArrivals: false,
|
||||
}),
|
||||
).toBe(old.length);
|
||||
|
||||
await runDenseIntradayRetention(prisma, {
|
||||
userId: user.id,
|
||||
log: () => {},
|
||||
});
|
||||
const folded = await liveRows(user.id, "OURA");
|
||||
expect(folded).toHaveLength(2);
|
||||
expect(folded.every((r) => r.externalId?.startsWith("stats:"))).toBe(true);
|
||||
|
||||
expect(
|
||||
await writeOuraHeartRatePage(user.id, old, new TouchedDays(), {
|
||||
emitArrivals: false,
|
||||
}),
|
||||
).toBe(0);
|
||||
expect(await liveRows(user.id, "OURA")).toEqual(folded);
|
||||
});
|
||||
|
||||
it("leaves heart rate inside the window raw", async () => {
|
||||
const user = await makeUser({ timezone: "UTC" });
|
||||
const recent = samples(dayStart(10));
|
||||
await upsertGoogleHealthMeasurements(user.id, recent.map(gh));
|
||||
await runDenseIntradayRetention(prisma, {
|
||||
userId: user.id,
|
||||
log: () => {},
|
||||
});
|
||||
expect(await liveRows(user.id, "GOOGLE_HEALTH")).toHaveLength(
|
||||
recent.length,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,347 @@
|
||||
/**
|
||||
* The full backfill against Postgres (#1298).
|
||||
*
|
||||
* - The windowed fold (`foldUserRollupHistory`, a UTC calendar year per
|
||||
* pass, newest first) writes exactly the rollup rows one pass of
|
||||
* `recomputeUserRollups` over the whole five-year window writes: every
|
||||
* granularity, every source, a week across New Year, the YEAR bucket the
|
||||
* window's oldest edge snaps out to, and a reading older than the window.
|
||||
* - A fold stopped part way and resumed with `before` ends in the same rows.
|
||||
* - The boot discovery, rewritten to walk days instead of reading every
|
||||
* reading in the window, finds the same accounts as the statement it
|
||||
* replaced, kept verbatim below as the reference.
|
||||
*/
|
||||
import { PgBoss } from "pg-boss";
|
||||
import {
|
||||
afterAll,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
import { Prisma, type MeasurementSource } from "@/generated/prisma/client";
|
||||
import { activeAccountSql } from "@/lib/account-deletion/active-account";
|
||||
import {
|
||||
enqueueBootTimeRollupBackfill,
|
||||
foldUserRollupHistory,
|
||||
isRollupFullBackfillPending,
|
||||
ROLLUP_FULL_BACKFILL_QUEUE,
|
||||
ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS,
|
||||
} from "@/lib/rollups/full-backfill";
|
||||
import {
|
||||
recomputeUserRollups,
|
||||
ROLLUP_FOLD_WINDOW_MS,
|
||||
} from "@/lib/rollups/measurement-rollups";
|
||||
|
||||
import { makeUser } from "../kit/factories";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({
|
||||
getGlobalBoss: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
const DAY_MS = 86_400_000;
|
||||
const HOUR_MS = 3_600_000;
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
vi.mocked(getGlobalBoss).mockReturnValue(null as never);
|
||||
});
|
||||
|
||||
function utc(y: number, m: number, d: number, h = 12, min = 0): Date {
|
||||
return new Date(Date.UTC(y, m - 1, d, h, min));
|
||||
}
|
||||
|
||||
/** Readings that can tell a windowed fold from one pass. */
|
||||
async function seedHistory(userId: string): Promise<void> {
|
||||
const now = new Date();
|
||||
const year = now.getUTCFullYear();
|
||||
const floor = new Date(now.getTime() - ROLLUP_FOLD_WINDOW_MS);
|
||||
const rows: Prisma.MeasurementCreateManyInput[] = [];
|
||||
const add = (
|
||||
type: "WEIGHT" | "PULSE",
|
||||
source: MeasurementSource,
|
||||
at: Date,
|
||||
value: number,
|
||||
) =>
|
||||
rows.push({
|
||||
userId,
|
||||
type,
|
||||
source,
|
||||
value,
|
||||
unit: type === "WEIGHT" ? "kg" : "bpm",
|
||||
measuredAt: at,
|
||||
externalId: `${source}:${type}:${at.toISOString()}`,
|
||||
});
|
||||
const days: Date[] = [];
|
||||
// A week across each of two New Years.
|
||||
for (let d = 0; d < 9; d++) {
|
||||
days.push(new Date(utc(year - 3, 12, 27).getTime() + d * DAY_MS));
|
||||
days.push(new Date(utc(year - 1, 12, 28).getTime() + d * DAY_MS));
|
||||
}
|
||||
days.push(utc(year - 2, 6, 15), utc(year - 2, 6, 16));
|
||||
days.push(new Date(now.getTime() - 3 * DAY_MS));
|
||||
// Just inside the window, and inside the YEAR bucket its edge snaps to.
|
||||
days.push(new Date(floor.getTime() + 2 * DAY_MS));
|
||||
days.push(new Date(floor.getTime() - 20 * DAY_MS));
|
||||
days.forEach((day, i) => {
|
||||
const base = Date.UTC(
|
||||
day.getUTCFullYear(),
|
||||
day.getUTCMonth(),
|
||||
day.getUTCDate(),
|
||||
);
|
||||
add("WEIGHT", "MANUAL", new Date(base + 7 * HOUR_MS), 80 + (i % 5) / 3);
|
||||
if (i % 2 === 0) {
|
||||
add("WEIGHT", "WITHINGS", new Date(base + 8 * HOUR_MS), 81 + i / 7);
|
||||
}
|
||||
for (let k = 0; k < 6; k++) {
|
||||
add(
|
||||
"PULSE",
|
||||
"APPLE_HEALTH",
|
||||
new Date(base + (k < 4 ? 9 : 15) * HOUR_MS + k * 7 * 60_000),
|
||||
55 + ((i * 7 + k * 11) % 40),
|
||||
);
|
||||
}
|
||||
add("PULSE", "GOOGLE_HEALTH", new Date(base + 23 * HOUR_MS), 60 + i);
|
||||
});
|
||||
await prisma.measurement.createMany({ data: rows });
|
||||
}
|
||||
|
||||
async function snapshot(userId: string) {
|
||||
const rows = await prisma.measurementRollup.findMany({
|
||||
where: { userId },
|
||||
orderBy: [
|
||||
{ type: "asc" },
|
||||
{ granularity: "asc" },
|
||||
{ bucketStart: "asc" },
|
||||
{ source: "asc" },
|
||||
],
|
||||
});
|
||||
return rows.map(({ computedAt, ...rest }) => {
|
||||
void computedAt;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
|
||||
function expectSameRows(
|
||||
actual: Awaited<ReturnType<typeof snapshot>>,
|
||||
expected: Awaited<ReturnType<typeof snapshot>>,
|
||||
): void {
|
||||
expect(actual.length).toBe(expected.length);
|
||||
expected.forEach((want, i) => {
|
||||
const got = actual[i] as Record<string, unknown>;
|
||||
for (const [key, value] of Object.entries(want)) {
|
||||
if (typeof value === "number" && key !== "count") {
|
||||
// A week across two windows is read in another range, so its sums
|
||||
// may be added in another order: equal to rounding, not to the bit.
|
||||
expect(got[key] as number).toBeCloseTo(value, 6);
|
||||
} else {
|
||||
expect(got[key]).toEqual(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
describe("foldUserRollupHistory", () => {
|
||||
it("writes the rows one pass over the whole window writes", async () => {
|
||||
const user = await makeUser();
|
||||
await seedHistory(user.id);
|
||||
|
||||
await recomputeUserRollups(user.id);
|
||||
const onePass = await snapshot(user.id);
|
||||
expect(onePass.length).toBeGreaterThan(50);
|
||||
expect(onePass.some((r) => r.granularity === "WEEK")).toBe(true);
|
||||
|
||||
await prisma.measurementRollup.deleteMany({ where: { userId: user.id } });
|
||||
const done = await foldUserRollupHistory(user.id, {
|
||||
statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS,
|
||||
});
|
||||
expect(done.windowsFolded).toBeGreaterThanOrEqual(5);
|
||||
expect(done.resumeBefore).toBeUndefined();
|
||||
expectSameRows(await snapshot(user.id), onePass);
|
||||
expect(
|
||||
(await snapshot(user.id)).every(
|
||||
(r) => r.readingMinValue !== null && r.readingMaxValue !== null,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("ends in the same rows when stopped part way and resumed", async () => {
|
||||
const user = await makeUser();
|
||||
await seedHistory(user.id);
|
||||
await recomputeUserRollups(user.id);
|
||||
const onePass = await snapshot(user.id);
|
||||
await prisma.measurementRollup.deleteMany({ where: { userId: user.id } });
|
||||
|
||||
// Stop on the seventh question: inside the second window.
|
||||
let asked = 0;
|
||||
const first = await foldUserRollupHistory(user.id, {
|
||||
shouldStop: () => ++asked >= 7,
|
||||
});
|
||||
expect(first.windowsFolded).toBe(1);
|
||||
expect(first.resumeBefore).toBeDefined();
|
||||
|
||||
const rest = await foldUserRollupHistory(user.id, {
|
||||
before: first.resumeBefore,
|
||||
});
|
||||
expect(rest.resumeBefore).toBeUndefined();
|
||||
expectSameRows(await snapshot(user.id), onePass);
|
||||
});
|
||||
});
|
||||
|
||||
/** The discovery statement before the day walk, verbatim. */
|
||||
async function referenceDiscovery(): Promise<string[]> {
|
||||
const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS);
|
||||
const users = await prisma.$queryRaw<Array<{ id: string }>>`
|
||||
SELECT DISTINCT id FROM (
|
||||
SELECT DISTINCT mt."user_id" AS id
|
||||
FROM (
|
||||
SELECT DISTINCT
|
||||
m."user_id",
|
||||
m."type",
|
||||
date_trunc('day', m."measured_at") AS bucket_start
|
||||
FROM measurements m
|
||||
WHERE m."deleted_at" IS NULL
|
||||
AND m."measured_at" >= ${foldWindowStart}
|
||||
) mt
|
||||
LEFT JOIN measurement_rollups r
|
||||
ON r."user_id" = mt."user_id"
|
||||
AND r."type" = mt."type"
|
||||
AND r."granularity" = 'DAY'
|
||||
AND r."bucket_start" = mt."bucket_start"
|
||||
WHERE r."bucket_start" IS NULL
|
||||
|
||||
UNION
|
||||
|
||||
SELECT DISTINCT r."user_id" AS id
|
||||
FROM measurement_rollups r
|
||||
WHERE r."granularity" = 'DAY'
|
||||
AND r."bucket_start" >= ${foldWindowStart}
|
||||
AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL)
|
||||
) discovered
|
||||
WHERE ${activeAccountSql('discovered."id"')}
|
||||
`;
|
||||
return users.map((u) => u.id).sort();
|
||||
}
|
||||
|
||||
describe("enqueueBootTimeRollupBackfill", () => {
|
||||
it("finds the accounts the statement it replaced found", async () => {
|
||||
const covered = await makeUser();
|
||||
const missingDay = await makeUser();
|
||||
const nullExtremes = await makeUser();
|
||||
const tombstoneOnly = await makeUser();
|
||||
const tooOld = await makeUser();
|
||||
const pendingDeletion = await makeUser();
|
||||
for (const u of [covered, missingDay, nullExtremes, pendingDeletion]) {
|
||||
await seedHistory(u.id);
|
||||
await recomputeUserRollups(u.id);
|
||||
}
|
||||
// A reading on a day with no DAY bucket.
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: missingDay.id,
|
||||
type: "BODY_FAT",
|
||||
value: 20,
|
||||
unit: "%",
|
||||
source: "MANUAL",
|
||||
measuredAt: new Date(Date.now() - 40 * DAY_MS),
|
||||
},
|
||||
});
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: pendingDeletion.id,
|
||||
type: "BODY_FAT",
|
||||
value: 20,
|
||||
unit: "%",
|
||||
source: "MANUAL",
|
||||
measuredAt: new Date(Date.now() - 40 * DAY_MS),
|
||||
},
|
||||
});
|
||||
await prisma.user.update({
|
||||
where: { id: pendingDeletion.id },
|
||||
data: { deletionRequestedAt: new Date() },
|
||||
});
|
||||
// A row folded before 0417 added the reading extremes.
|
||||
await prisma.$executeRaw`
|
||||
UPDATE measurement_rollups SET reading_min_value = NULL
|
||||
WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY'
|
||||
AND bucket_start = (
|
||||
SELECT max(bucket_start) FROM measurement_rollups
|
||||
WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY'
|
||||
)
|
||||
`;
|
||||
// Only a tombstone inside the window: nothing to fold.
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: tombstoneOnly.id,
|
||||
type: "WEIGHT",
|
||||
value: 70,
|
||||
unit: "kg",
|
||||
source: "MANUAL",
|
||||
measuredAt: new Date(Date.now() - 10 * DAY_MS),
|
||||
deletedAt: new Date(),
|
||||
},
|
||||
});
|
||||
// Only readings older than the window: the fold writes nothing there.
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: tooOld.id,
|
||||
type: "WEIGHT",
|
||||
value: 70,
|
||||
unit: "kg",
|
||||
source: "MANUAL",
|
||||
measuredAt: new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS - 5 * DAY_MS),
|
||||
},
|
||||
});
|
||||
|
||||
const send = vi.fn(async () => "job-id");
|
||||
vi.mocked(getGlobalBoss).mockReturnValue({ send } as never);
|
||||
const result = await enqueueBootTimeRollupBackfill();
|
||||
expect(result.error).toBeNull();
|
||||
const found = (
|
||||
send.mock.calls as unknown as Array<[string, { userId: string }]>
|
||||
)
|
||||
.map((call) => call[1].userId)
|
||||
.sort();
|
||||
|
||||
expect(found).toEqual([missingDay.id, nullExtremes.id].sort());
|
||||
expect(found).toEqual(await referenceDiscovery());
|
||||
});
|
||||
});
|
||||
|
||||
describe("isRollupFullBackfillPending", () => {
|
||||
let boss: PgBoss;
|
||||
|
||||
beforeAll(async () => {
|
||||
boss = new PgBoss({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
supervise: false,
|
||||
schedule: false,
|
||||
});
|
||||
await boss.start();
|
||||
await boss.createQueue(ROLLUP_FULL_BACKFILL_QUEUE);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await boss?.stop({ graceful: false, close: true });
|
||||
await prisma.$executeRawUnsafe(`DROP SCHEMA IF EXISTS pgboss CASCADE`);
|
||||
});
|
||||
|
||||
it("is true while the account's full fold is queued or running, and only for that account", async () => {
|
||||
expect(await isRollupFullBackfillPending("user-a")).toBe(false);
|
||||
await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, { userId: "user-a" });
|
||||
expect(await isRollupFullBackfillPending("user-a")).toBe(true);
|
||||
expect(await isRollupFullBackfillPending("user-b")).toBe(false);
|
||||
const [job] = await boss.fetch(ROLLUP_FULL_BACKFILL_QUEUE);
|
||||
expect(await isRollupFullBackfillPending("user-a")).toBe(true);
|
||||
await boss.complete(ROLLUP_FULL_BACKFILL_QUEUE, job.id);
|
||||
expect(await isRollupFullBackfillPending("user-a")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* The system job alert against a ledger pg-boss itself wrote (#1298).
|
||||
*
|
||||
* A background job of the instance that keeps failing pushes every admin
|
||||
* once, a reminder at most once a day while it keeps failing, and once when it
|
||||
* runs successfully again. Fewer failures than the streak, and queues that
|
||||
* have an account-facing surface of their own, push nothing.
|
||||
*
|
||||
* pg-boss provisions its own schema here and the jobs are failed and completed
|
||||
* through its own API, as in `job-failures.test.ts`, so the reader is proven
|
||||
* against the table pg-boss writes rather than one the test wrote.
|
||||
*/
|
||||
import { PgBoss } from "pg-boss";
|
||||
import {
|
||||
afterAll,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
const dispatched = vi.hoisted(() => ({
|
||||
calls: [] as Array<{
|
||||
userId: string;
|
||||
titleKey: string;
|
||||
messageKey: string;
|
||||
params?: Record<string, string | number>;
|
||||
eventType?: string;
|
||||
}>,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/notifications/dispatch-localised", () => ({
|
||||
dispatchLocalisedNotification: vi.fn(
|
||||
async (opts: (typeof dispatched.calls)[number]) => {
|
||||
dispatched.calls.push(opts);
|
||||
},
|
||||
),
|
||||
}));
|
||||
|
||||
import {
|
||||
notifySystemJobFailures,
|
||||
SYSTEM_JOB_FAILURE_STREAK,
|
||||
} from "@/lib/jobs/system-job-alert";
|
||||
|
||||
import { makeUser } from "../kit/factories";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
const SYSTEM_QUEUE = "rollup-full-backfill";
|
||||
const ACCOUNT_QUEUE = "google-health-sync";
|
||||
const HOUR_MS = 3_600_000;
|
||||
|
||||
let boss: PgBoss;
|
||||
|
||||
async function runOnce(queue: string, outcome: "fail" | "complete") {
|
||||
const id = await boss.send(queue, { userId: "u" }, { retryLimit: 0 });
|
||||
if (!id) throw new Error("queue rejected the job");
|
||||
const [job] = await boss.fetch(queue);
|
||||
if (!job) throw new Error("nothing to fetch");
|
||||
if (outcome === "fail") {
|
||||
await boss.fail(
|
||||
queue,
|
||||
job.id,
|
||||
new Error("canceling statement due to statement timeout"),
|
||||
);
|
||||
} else {
|
||||
await boss.complete(queue, job.id, { ok: true });
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
boss = new PgBoss({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
supervise: false,
|
||||
schedule: false,
|
||||
});
|
||||
await boss.start();
|
||||
await boss.createQueue(SYSTEM_QUEUE);
|
||||
await boss.createQueue(ACCOUNT_QUEUE);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await boss?.stop({ graceful: false, close: true });
|
||||
await getPrismaClient().$executeRawUnsafe(
|
||||
`DROP SCHEMA IF EXISTS pgboss CASCADE`,
|
||||
);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
await getPrismaClient().$executeRawUnsafe(`DELETE FROM pgboss.job`);
|
||||
dispatched.calls.length = 0;
|
||||
});
|
||||
|
||||
describe("notifySystemJobFailures", () => {
|
||||
it("tells every admin once, reminds a day later, and announces the recovery once", async () => {
|
||||
const admin = await makeUser({ role: "ADMIN" });
|
||||
const second = await makeUser({ role: "ADMIN" });
|
||||
await makeUser({ role: "USER" });
|
||||
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK; i++) {
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
}
|
||||
const now = new Date();
|
||||
|
||||
const first = await notifySystemJobFailures(now);
|
||||
expect(first.failing).toBe(1);
|
||||
expect(first.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "problem" }]);
|
||||
expect(dispatched.calls.map((c) => c.userId).sort()).toEqual(
|
||||
[admin.id, second.id].sort(),
|
||||
);
|
||||
const call = dispatched.calls[0];
|
||||
expect(call.eventType).toBe("SYSTEM_ALERT");
|
||||
expect(call.titleKey).toBe("notifications.systemJobAlert.titleFailing");
|
||||
expect(call.params).toMatchObject({
|
||||
queue: SYSTEM_QUEUE,
|
||||
failures: SYSTEM_JOB_FAILURE_STREAK,
|
||||
});
|
||||
expect(String(call.params?.error)).toContain("statement timeout");
|
||||
|
||||
// The next hour: still failing, already told.
|
||||
dispatched.calls.length = 0;
|
||||
const again = await notifySystemJobFailures(
|
||||
new Date(now.getTime() + HOUR_MS),
|
||||
);
|
||||
expect(again.sent).toEqual([]);
|
||||
expect(dispatched.calls).toEqual([]);
|
||||
|
||||
// A day later: one reminder.
|
||||
const reminder = await notifySystemJobFailures(
|
||||
new Date(now.getTime() + 25 * HOUR_MS),
|
||||
);
|
||||
expect(reminder.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "reminder" }]);
|
||||
|
||||
// It runs again: one recovery notice, then silence.
|
||||
await runOnce(SYSTEM_QUEUE, "complete");
|
||||
dispatched.calls.length = 0;
|
||||
const recovered = await notifySystemJobFailures(
|
||||
new Date(now.getTime() + 26 * HOUR_MS),
|
||||
);
|
||||
expect(recovered.sent).toEqual([
|
||||
{ queue: SYSTEM_QUEUE, kind: "recovered" },
|
||||
]);
|
||||
expect(dispatched.calls[0].titleKey).toBe(
|
||||
"notifications.systemJobAlert.titleRecovered",
|
||||
);
|
||||
const quiet = await notifySystemJobFailures(
|
||||
new Date(now.getTime() + 27 * HOUR_MS),
|
||||
);
|
||||
expect(quiet.sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("stays quiet below the streak and for queues with an account-facing surface", async () => {
|
||||
await makeUser({ role: "ADMIN" });
|
||||
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK - 1; i++) {
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
}
|
||||
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK + 2; i++) {
|
||||
await runOnce(ACCOUNT_QUEUE, "fail");
|
||||
}
|
||||
const outcome = await notifySystemJobFailures();
|
||||
expect(outcome.failing).toBe(0);
|
||||
expect(outcome.sent).toEqual([]);
|
||||
expect(dispatched.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("counts a success between failures as the end of the streak", async () => {
|
||||
await makeUser({ role: "ADMIN" });
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
await runOnce(SYSTEM_QUEUE, "complete");
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
const outcome = await notifySystemJobFailures();
|
||||
expect(outcome.failing).toBe(0);
|
||||
expect(outcome.sent).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user