mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 08:02:12 +00:00
fix(restore): order the restore and the consolidation passes by one lock
A restore on a freshly booted instance failed its clearing step with a deadlock (40P01) while the boot-time step consolidation was folding the same account. A consolidation day updates or mints the day's total and then soft-deletes the day's per-sample rows; the restore deletes every reading of the account in one statement. When the restore reached a day's samples before its total, each side held what the other needed next. The restore already took an advisory lock per account before its first delete. Every day transaction of the passes built on runConsolidation (step and mean consolidation, the per-sample cumulative drain, the dense intraday retention) now takes the same lock in shared mode as its first statement. A restore that starts mid-day waits for that day to commit; a day that would start while a restore runs is refused at once, and the pass leaves that account for its next run instead of folding rows the restore is about to replace. The passes still share the lock with each other. A new integration test builds the lock order that deadlocked, with a row trigger holding the consolidation on the day's total while the restore starts; before this change the consolidation was cancelled with a deadlock. A guard test requires every transaction in a pass on runConsolidation to open with the shared lock. Refs #1031
This commit is contained in:
1 parent
f2569b6fdf
commit
add6d1fadc
15 files changed
+531
-8
No files matched your search
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Every day transaction of the consolidation family holds the account
|
||||
* against a restore before it touches a reading (`restore-lock.ts`).
|
||||
*
|
||||
* The passes built on `runConsolidation` fold an account a day at a time, and
|
||||
* each day's transaction takes row locks on the account's readings. A restore
|
||||
* deletes all of them in one statement. Without a common first lock the two
|
||||
* take the same rows in different orders and deadlock (#1031, pinned end to
|
||||
* end by `tests/integration/restore-consolidation-deadlock.test.ts` for the
|
||||
* step consolidation). This guard extends the rule to every pass on the same
|
||||
* base, present and future: each file that calls `runConsolidation` must open
|
||||
* every transaction it starts with `holdAccountAgainstRestore(tx, …)`.
|
||||
*
|
||||
* The matcher is whitespace-tolerant and the file and transaction counts are
|
||||
* asserted to be non-zero, so a pattern that matches nothing fails instead of
|
||||
* passing.
|
||||
*/
|
||||
import { readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const ROOT = join(__dirname, "..", "lib");
|
||||
|
||||
function sourceFiles(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const name of readdirSync(dir)) {
|
||||
const path = join(dir, name);
|
||||
if (statSync(path).isDirectory()) {
|
||||
if (name === "__tests__") continue;
|
||||
out.push(...sourceFiles(path));
|
||||
} else if (/\.ts$/.test(name) && !/\.test\.ts$/.test(name)) {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The text of a file with its comments blanked, so prose cannot match. */
|
||||
function code(path: string): string {
|
||||
return readFileSync(path, "utf8")
|
||||
.replace(/\/\*[\s\S]*?\*\//g, " ")
|
||||
.replace(/(^|[^:])\/\/.*$/gm, "$1");
|
||||
}
|
||||
|
||||
const passes = sourceFiles(ROOT).filter((path) =>
|
||||
/\bawait\s+runConsolidation\b/.test(code(path)),
|
||||
);
|
||||
|
||||
describe("consolidation passes hold the account against a restore", () => {
|
||||
it("finds the passes", () => {
|
||||
// Step, mean, per-sample cumulative, dense intraday retention.
|
||||
expect(passes.length).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
|
||||
it.each(passes.map((path) => [relative(ROOT, path), path]))(
|
||||
"%s opens every transaction with holdAccountAgainstRestore",
|
||||
(_label, path) => {
|
||||
const text = code(path);
|
||||
const opened = [
|
||||
...text.matchAll(
|
||||
/\$transaction\(\s*async\s*\(\s*(\w+)\s*\)\s*=>\s*\{/g,
|
||||
),
|
||||
];
|
||||
expect(opened.length).toBeGreaterThan(0);
|
||||
for (const match of opened) {
|
||||
const tx = match[1];
|
||||
const after = text.slice(match.index! + match[0].length).trimStart();
|
||||
expect(after.startsWith(`await holdAccountAgainstRestore(${tx},`)).toBe(
|
||||
true,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
storedBackupRefusal,
|
||||
type StoredBackupRef,
|
||||
} from "@/lib/export/stored-backup";
|
||||
import { takeRestoreLock } from "@/lib/export/restore-lock";
|
||||
import {
|
||||
insertMeasurementRows,
|
||||
type MeasurementInsertRow,
|
||||
@@ -683,12 +684,6 @@ export async function restoreBackup(
|
||||
report("clearing");
|
||||
outcome = await prisma.$transaction(
|
||||
async (tx) => {
|
||||
// One restore of an account at a time, whatever let a second one
|
||||
// start: the job table admits one queued or running job per account,
|
||||
// and this lock holds even if a worker that was thought gone is
|
||||
// still writing. A second transaction waits here for the first to
|
||||
// commit or roll back, then replaces the account again from its own
|
||||
// file, which is what it was asked to do.
|
||||
// Every connection carries the request limits from `src/lib/db.ts`
|
||||
// (60 s per statement, 60 s idle inside a transaction). A restore is
|
||||
// not a request: deleting two million readings is one statement, and
|
||||
@@ -697,7 +692,18 @@ export async function restoreBackup(
|
||||
// has its own limit; SET LOCAL gives each statement the same one, and
|
||||
// ends with the transaction.
|
||||
await tx.$executeRaw`SELECT set_config('statement_timeout', ${String(transactionTimeoutMs)}, true), set_config('idle_in_transaction_session_timeout', ${String(transactionTimeoutMs)}, true)`;
|
||||
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext(${`backup-restore:${ownerId}`}))`;
|
||||
// One restore of an account at a time, whatever let a second one
|
||||
// start: the job table admits one queued or running job per account,
|
||||
// and this lock holds even if a worker that was thought gone is
|
||||
// still writing. A second transaction waits here for the first to
|
||||
// commit or roll back, then replaces the account again from its own
|
||||
// file, which is what it was asked to do.
|
||||
//
|
||||
// The same lock orders the restore against the background passes that
|
||||
// rewrite readings a day at a time (`restore-lock.ts`): they hold it
|
||||
// shared for each day, so the delete below never meets a day half
|
||||
// folded, which is where the two used to deadlock.
|
||||
await takeRestoreLock(tx, ownerId);
|
||||
// Every reference this restore writes stays inside the account: the
|
||||
// references that leave it are counted now and again after the last
|
||||
// write, and any that grew rolls the whole transaction back
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* The account lock a restore holds, and the shared side of it for the passes
|
||||
* that rewrite an account's readings in the background.
|
||||
*
|
||||
* A restore deletes every reading of the account in one statement, inside
|
||||
* one long transaction. The consolidation passes (step and mean
|
||||
* consolidation, the per-sample cumulative drain, the dense intraday
|
||||
* retention) fold a day at a time: each day's transaction updates or mints
|
||||
* the day's total and then soft-deletes or deletes the day's per-sample
|
||||
* rows. Both take row locks on the same readings, each in its own order, and
|
||||
* when the restore reached a day's samples before its total while a pass was
|
||||
* folding that day, each held what the other needed next. Postgres broke the
|
||||
* cycle by cancelling one of them (`40P01`, #1031), and on a freshly booted
|
||||
* instance, where the step consolidation runs over every account with legacy
|
||||
* rows, that was the restore.
|
||||
*
|
||||
* So the two are ordered by one advisory lock before either touches a row.
|
||||
* The restore takes it exclusively as the first act of its transaction
|
||||
* ({@link takeRestoreLock}); every day transaction of those passes takes it in
|
||||
* shared mode as its first act ({@link holdAccountAgainstRestore}). Passes
|
||||
* share the lock with each other, so nothing about how they run together
|
||||
* changes. A restore that starts while a day is being folded waits for that
|
||||
* one day to commit, milliseconds. A day that would start while a restore is
|
||||
* running does not wait for it, which could take many minutes: it is refused
|
||||
* with {@link AccountRestoreInProgressError}, and the pass leaves the account
|
||||
* for its next run, which reads the restored rows rather than the ones it had
|
||||
* scanned before the restore replaced them.
|
||||
*/
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
|
||||
type LockClient = Pick<Prisma.TransactionClient, "$queryRaw">;
|
||||
|
||||
/** The lock's key. One per account; the restore has always used this one. */
|
||||
function restoreLockKey(userId: string): string {
|
||||
return `backup-restore:${userId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Take the account's restore lock exclusively, for the rest of the calling
|
||||
* transaction. Waits for any day transaction of a background pass to commit,
|
||||
* and for another restore of the same account to finish.
|
||||
*/
|
||||
export async function takeRestoreLock(
|
||||
tx: LockClient,
|
||||
userId: string,
|
||||
): Promise<void> {
|
||||
// `pg_advisory_xact_lock` returns void, which the client cannot read as a
|
||||
// column; selecting from it yields a plain row.
|
||||
await tx.$queryRaw`
|
||||
SELECT 1 AS locked
|
||||
FROM pg_advisory_xact_lock(hashtext(${restoreLockKey(userId)}))
|
||||
`;
|
||||
}
|
||||
|
||||
/** A restore of the account is running; the pass should leave it for now. */
|
||||
export class AccountRestoreInProgressError extends Error {
|
||||
readonly userId: string;
|
||||
constructor(userId: string) {
|
||||
super("A restore of this account is running");
|
||||
this.name = "AccountRestoreInProgressError";
|
||||
this.userId = userId;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hold the account against a restore for the rest of the calling
|
||||
* transaction: the shared side of {@link takeRestoreLock}. Call it before the
|
||||
* transaction's first read or write of the account's readings. Throws
|
||||
* {@link AccountRestoreInProgressError} at once when a restore holds the
|
||||
* lock, rather than waiting for it.
|
||||
*/
|
||||
export async function holdAccountAgainstRestore(
|
||||
tx: LockClient,
|
||||
userId: string,
|
||||
): Promise<void> {
|
||||
const [row] = await tx.$queryRaw<Array<{ held: boolean }>>`
|
||||
SELECT pg_try_advisory_xact_lock_shared(hashtext(${restoreLockKey(userId)})) AS held
|
||||
`;
|
||||
if (!row?.held) throw new AccountRestoreInProgressError(userId);
|
||||
}
|
||||
@@ -16,6 +16,14 @@ import { canonicalDailyTimestamp } from "../consolidation-tz";
|
||||
import type { PerSampleRow } from "../drain-per-sample-cumulative";
|
||||
import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
|
||||
|
||||
// The restore lock is a Postgres advisory lock; the mocked client has no
|
||||
// database behind it. The lock itself is pinned against a real one in
|
||||
// `tests/integration/restore-consolidation-deadlock.test.ts`.
|
||||
vi.mock("@/lib/export/restore-lock", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/lib/export/restore-lock")>()),
|
||||
holdAccountAgainstRestore: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
function row(
|
||||
id: string,
|
||||
value: number,
|
||||
|
||||
@@ -35,6 +35,14 @@ import type { PerSampleRow } from "../drain-per-sample-cumulative";
|
||||
import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
|
||||
import { candidateLookup, createManyVia } from "./hourly-mint-mock";
|
||||
|
||||
// The restore lock is a Postgres advisory lock; the mocked client has no
|
||||
// database behind it. The lock itself is pinned against a real one in
|
||||
// `tests/integration/restore-consolidation-deadlock.test.ts`.
|
||||
vi.mock("@/lib/export/restore-lock", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/lib/export/restore-lock")>()),
|
||||
holdAccountAgainstRestore: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
function pulseRow(id: string, value: number, iso: string): PerSampleRow {
|
||||
return {
|
||||
id,
|
||||
|
||||
@@ -34,6 +34,14 @@ import {
|
||||
import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
|
||||
import { candidateLookup, createManyVia } from "./hourly-mint-mock";
|
||||
|
||||
// The restore lock is a Postgres advisory lock; the mocked client has no
|
||||
// database behind it. The lock itself is pinned against a real one in
|
||||
// `tests/integration/restore-consolidation-deadlock.test.ts`.
|
||||
vi.mock("@/lib/export/restore-lock", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/lib/export/restore-lock")>()),
|
||||
holdAccountAgainstRestore: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
/** A day well outside the retention window, so the fold definitely runs. */
|
||||
const FOLD_DAY = new Date(
|
||||
Date.now() - (DENSE_INTRADAY_RETENTION_DAYS + 10) * 24 * 60 * 60 * 1000,
|
||||
|
||||
@@ -27,6 +27,14 @@ import type { PerSampleRow } from "../drain-per-sample-cumulative";
|
||||
import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
|
||||
import { candidateLookup, createManyVia } from "./hourly-mint-mock";
|
||||
|
||||
// The restore lock is a Postgres advisory lock; the mocked client has no
|
||||
// database behind it. The lock itself is pinned against a real one in
|
||||
// `tests/integration/restore-consolidation-deadlock.test.ts`.
|
||||
vi.mock("@/lib/export/restore-lock", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/lib/export/restore-lock")>()),
|
||||
holdAccountAgainstRestore: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
function row(
|
||||
id: string,
|
||||
value: number,
|
||||
|
||||
@@ -31,6 +31,14 @@ import type { PerSampleRow } from "../drain-per-sample-cumulative";
|
||||
import type { MeasurementType, PrismaClient } from "@/generated/prisma/client";
|
||||
import { candidateLookup, createManyVia } from "./hourly-mint-mock";
|
||||
|
||||
// The restore lock is a Postgres advisory lock; the mocked client has no
|
||||
// database behind it. The lock itself is pinned against a real one in
|
||||
// `tests/integration/restore-consolidation-deadlock.test.ts`.
|
||||
vi.mock("@/lib/export/restore-lock", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/lib/export/restore-lock")>()),
|
||||
holdAccountAgainstRestore: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
function spo2Row(id: string, value: number, iso: string): PerSampleRow {
|
||||
return {
|
||||
id,
|
||||
|
||||
@@ -22,6 +22,14 @@ import {
|
||||
} from "@/lib/tz/local-day";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
|
||||
// The restore lock is a Postgres advisory lock; the mocked client has no
|
||||
// database behind it. The lock itself is pinned against a real one in
|
||||
// `tests/integration/restore-consolidation-deadlock.test.ts`.
|
||||
vi.mock("@/lib/export/restore-lock", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@/lib/export/restore-lock")>()),
|
||||
holdAccountAgainstRestore: vi.fn(async () => {}),
|
||||
}));
|
||||
|
||||
describe("dayKeyForUserTz", () => {
|
||||
it("anchors the calendar day to the user's IANA zone", () => {
|
||||
// 23:45 NZST on 2026-05-16 is still 11:45 UTC on the same day, but
|
||||
|
||||
@@ -45,6 +45,7 @@
|
||||
* production standalone image strips `tsx`. Modelled on the
|
||||
* `step-consolidation` boot-time converging-backfill pattern.
|
||||
*/
|
||||
import { holdAccountAgainstRestore } from "@/lib/export/restore-lock";
|
||||
import type {
|
||||
MeasurementType,
|
||||
Prisma,
|
||||
@@ -273,6 +274,8 @@ export async function consolidateDailyMean(
|
||||
const unit = dayRows[0]?.unit ?? "unknown";
|
||||
let removed = 0;
|
||||
await pc.$transaction(async (tx) => {
|
||||
// First, before any reading is touched: see `restore-lock.ts`.
|
||||
await holdAccountAgainstRestore(tx, userId);
|
||||
// The upsert below arbiters on (userId, type, source, externalId),
|
||||
// but the Measurement model carries a SECOND full unique —
|
||||
// (userId, type, measuredAt, source, sleepStage), NULLS NOT
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
* production standalone image strips `tsx`. Modelled on the
|
||||
* `rollup-full-backfill` boot-time converging-backfill pattern.
|
||||
*/
|
||||
import { holdAccountAgainstRestore } from "@/lib/export/restore-lock";
|
||||
import type {
|
||||
MeasurementSource,
|
||||
PrismaClient,
|
||||
@@ -318,6 +319,8 @@ export async function consolidateLegacySteps(
|
||||
try {
|
||||
let removed = 0;
|
||||
await pc.$transaction(async (tx) => {
|
||||
// First, before any reading is touched: see `restore-lock.ts`.
|
||||
await holdAccountAgainstRestore(tx, userId);
|
||||
if (shouldMint) {
|
||||
// Mint the canonical daily-total row. The unique index
|
||||
// (userId, type, source, externalId) makes the upsert
|
||||
|
||||
@@ -26,6 +26,8 @@
|
||||
* is no value-level cycle between the two — a cycle would otherwise trip
|
||||
* a module-init TDZ in the production bundle.
|
||||
*/
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { AccountRestoreInProgressError } from "@/lib/export/restore-lock";
|
||||
import type {
|
||||
MeasurementType,
|
||||
PrismaClient,
|
||||
@@ -425,6 +427,8 @@ export async function runConsolidation<TType extends MeasurementType>(
|
||||
dryRun: boolean;
|
||||
daysFailed: number;
|
||||
stoppedEarly: boolean;
|
||||
/** Accounts left for the next run because a restore of them was running. */
|
||||
usersDeferredForRestore: number;
|
||||
}> {
|
||||
const { prismaClient, options } = params;
|
||||
const dryRun = options.dryRun ?? false;
|
||||
@@ -434,6 +438,7 @@ export async function runConsolidation<TType extends MeasurementType>(
|
||||
let daysFailed = 0;
|
||||
let daysWalked = 0;
|
||||
let stoppedEarly = false;
|
||||
let usersDeferredForRestore = 0;
|
||||
|
||||
const users = await loadConsolidationUsers(prismaClient, options.userId);
|
||||
|
||||
@@ -533,6 +538,20 @@ export async function runConsolidation<TType extends MeasurementType>(
|
||||
outcome,
|
||||
});
|
||||
} catch (err) {
|
||||
// A restore of this account started while the pass was walking
|
||||
// it. The rows scanned so far are about to be replaced, so the
|
||||
// account is left for the next run, which reads the restored ones
|
||||
// (`restore-lock.ts`). Not a failed day, and no reason to stop the
|
||||
// other accounts.
|
||||
if (err instanceof AccountRestoreInProgressError) {
|
||||
usersDeferredForRestore += 1;
|
||||
annotate({
|
||||
meta: {
|
||||
consolidation_deferred_for_restore: usersDeferredForRestore,
|
||||
},
|
||||
});
|
||||
continue walk;
|
||||
}
|
||||
// No boundary supplied → preserve the historical abort-the-run
|
||||
// behaviour for the drains that classify errors in `writeDay`.
|
||||
if (!params.onBucketError) throw err;
|
||||
@@ -561,5 +580,11 @@ export async function runConsolidation<TType extends MeasurementType>(
|
||||
params.onUserComplete?.({ userId: user.id, tz, dryRun });
|
||||
}
|
||||
|
||||
return { usersScanned: users.length, dryRun, daysFailed, stoppedEarly };
|
||||
return {
|
||||
usersScanned: users.length,
|
||||
dryRun,
|
||||
daysFailed,
|
||||
stoppedEarly,
|
||||
usersDeferredForRestore,
|
||||
};
|
||||
}
|
||||
@@ -47,6 +47,7 @@
|
||||
* The one-shot history rebuild that converts ALREADY-folded daily rows to
|
||||
* the hourly grain lives in `dense-intraday-hourly-rebuild.ts`.
|
||||
*/
|
||||
import { holdAccountAgainstRestore } from "@/lib/export/restore-lock";
|
||||
import type {
|
||||
MeasurementType,
|
||||
Prisma,
|
||||
@@ -617,6 +618,8 @@ export async function runDenseIntradayRetention(
|
||||
let removed = 0;
|
||||
let retiredDaily = false;
|
||||
await pc.$transaction(async (tx) => {
|
||||
// First, before any reading is touched: see `restore-lock.ts`.
|
||||
await holdAccountAgainstRestore(tx, userId);
|
||||
const canonicalRowIds = await adoptOrMintHourlyRows(tx, {
|
||||
userId,
|
||||
type,
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
* touching the DB so the operator can inspect what would change before
|
||||
* committing.
|
||||
*/
|
||||
import { holdAccountAgainstRestore } from "@/lib/export/restore-lock";
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
import type {
|
||||
MeasurementAggregationProvenance,
|
||||
@@ -291,6 +292,8 @@ export async function drainPerSampleCumulative(
|
||||
const foldOnce = async (): Promise<number> => {
|
||||
let removed = 0;
|
||||
await pc.$transaction(async (tx) => {
|
||||
// First, before any reading is touched: see `restore-lock.ts`.
|
||||
await holdAccountAgainstRestore(tx, userId);
|
||||
// Single-flight the per-(user, type, day) fold. Two overlapping
|
||||
// drains — a rolling-deploy nightly overlap, or an operator's manual
|
||||
// `POST /api/admin/drain-per-sample-cumulative` racing the cron —
|
||||
|
||||
Reference in new issue
Block a user