fix(jobs): let the nightly tombstone purge step around a running restore

A restore clears every row of the account, tombstones included, inside one
long transaction under the account's exclusive restore lock. The nightly
purge deleted expired tombstones across accounts, one statement per batch,
without that lock: on an account under restore it waited on the restore's
row locks for as long as the restore ran, past its statement timeout, or
deadlocked with it once each side held rows the other needed.

Each batch is now deleted one account at a time, in a transaction that
first takes the restore lock in shared mode, the same order the
consolidation passes keep. An account under restore is refused at once,
kept out of the rest of the run, and purged the next night. Still one
delete per account per batch, never one per row. The job reports how many
accounts it deferred.

The integration test holds a stand-in restore on one account while the
purge runs: the purge returns without waiting, leaves that account alone
and purges the other. With the lock call removed it waits out the hold.

Refs #1031

tests/integration/sync-changes.test.ts

@@ -399,7 +399,7 @@ describe("GET /api/sync/changes (real Postgres)", () => {
     // pruned; the recent reachable tombstone survives. `drained` says the
     // batched walk reached the end of the backlog rather than stopping at
     // its per-run cap.
-    expect(pruned).toEqual({ deleted: 1, drained: true });
+    expect(pruned).toEqual({ deleted: 1, drained: true, deferredAccounts: 0 });
     const survivors = await prisma.measurement.findMany({
       where: { userId: TEST_USER_ID },
       select: { externalId: true },

tests/integration/tombstone-cleanup-restore-lock.test.ts

@@ -0,0 +1,152 @@
+/**
+ * #1031 — the nightly tombstone purge against a running restore.
+ *
+ * A restore clears every row of the account, tombstones included, inside one
+ * long transaction that holds the account's restore lock exclusively. The
+ * purge used to delete expired tombstones across accounts in one statement
+ * per batch, without the lock: on an account under restore it waited on the
+ * restore's row locks for as long as the restore ran, or deadlocked with it.
+ *
+ * Here a transaction stands in for the restore: it takes the lock, deletes the
+ * account's rows, and holds for three seconds before rolling back. The purge
+ * runs while it holds. What must hold: the purge does not wait, leaves the
+ * account under restore alone and says so, and still purges every other
+ * account.
+ *
+ * Mutation check: drop the `holdAccountAgainstRestore` call from
+ * `purgeTombstonesByAccount` and the purge waits out the hold, then deletes
+ * the restoring account's tombstones once the stand-in rolls back.
+ */
+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+const RESTORING = "tombstone-restoring";
+const OTHER = "tombstone-other";
+const HOLD_MS = 3_000;
+const LONG_AGO = new Date("2020-01-01T00:00:00.000Z");
+
+async function seed() {
+  const prisma = getPrismaClient();
+  for (const id of [RESTORING, OTHER]) {
+    await prisma.user.create({
+      data: { id, username: id, role: "USER", timezone: "UTC" },
+    });
+    for (let i = 0; i < 3; i++) {
+      await prisma.measurement.create({
+        data: {
+          id: `${id}-tomb-${i}`,
+          userId: id,
+          type: "WEIGHT",
+          value: 70 + i,
+          unit: "kg",
+          source: "MANUAL",
+          measuredAt: new Date(`2020-01-0${i + 1}T08:00:00.000Z`),
+          deletedAt: LONG_AGO,
+        },
+      });
+    }
+    await prisma.moodEntry.create({
+      data: {
+        id: `${id}-mood-tomb`,
+        userId: id,
+        date: "2020-01-01",
+        mood: "OKAY",
+        score: 3,
+        moodLoggedAt: new Date("2020-01-01T08:00:00.000Z"),
+        deletedAt: LONG_AGO,
+      },
+    });
+  }
+}
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+});
+
+afterAll(async () => {
+  await truncateAllTables(getPrismaClient());
+});
+
+describe("tombstone purge against a running restore (#1031)", () => {
+  it("leaves the account under restore for the next run without waiting on it", async () => {
+    const prisma = getPrismaClient();
+    await seed();
+    const { takeRestoreLock } = await import("@/lib/export/restore-lock");
+    const {
+      cleanupExpiredMeasurementTombstones,
+      cleanupExpiredMoodTombstones,
+    } = await import("@/lib/jobs/measurement-tombstone-cleanup");
+
+    let holding!: () => void;
+    const isHolding = new Promise<void>((resolve) => (holding = resolve));
+    const restore = prisma
+      .$transaction(
+        async (tx) => {
+          await takeRestoreLock(tx, RESTORING);
+          await tx.measurement.deleteMany({ where: { userId: RESTORING } });
+          await tx.moodEntry.deleteMany({ where: { userId: RESTORING } });
+          holding();
+          await new Promise((resolve) => setTimeout(resolve, HOLD_MS));
+          throw new Error("rolled back");
+        },
+        { timeout: 30_000 },
+      )
+      .catch(() => undefined);
+    await isHolding;
+
+    const started = Date.now();
+    const measurements = await cleanupExpiredMeasurementTombstones(prisma);
+    const moods = await cleanupExpiredMoodTombstones(prisma);
+    const elapsed = Date.now() - started;
+    await restore;
+
+    expect(elapsed).toBeLessThan(HOLD_MS / 2);
+    expect(measurements).toEqual({
+      deleted: 3,
+      drained: true,
+      deferredAccounts: 1,
+    });
+    expect(moods).toEqual({ deleted: 1, drained: true, deferredAccounts: 1 });
+    // The stand-in rolled back, so the restoring account still has its
+    // tombstones; the next run purges them.
+    expect(
+      await prisma.measurement.count({ where: { userId: RESTORING } }),
+    ).toBe(3);
+    expect(await prisma.measurement.count({ where: { userId: OTHER } })).toBe(
+      0,
+    );
+    expect(await prisma.moodEntry.count({ where: { userId: OTHER } })).toBe(0);
+
+    const next = await cleanupExpiredMeasurementTombstones(prisma);
+    expect(next).toEqual({ deleted: 3, drained: true, deferredAccounts: 0 });
+  }, 60_000);
+
+  it("a restore that starts during the purge waits only for the account's batch", async () => {
+    const prisma = getPrismaClient();
+    await seed();
+    const { takeRestoreLock } = await import("@/lib/export/restore-lock");
+    const { cleanupExpiredMeasurementTombstones } =
+      await import("@/lib/jobs/measurement-tombstone-cleanup");
+
+    const purge = cleanupExpiredMeasurementTombstones(prisma);
+    const restore = prisma.$transaction(async (tx) => {
+      await takeRestoreLock(tx, RESTORING);
+      return tx.measurement.deleteMany({ where: { userId: RESTORING } });
+    });
+    const [purged] = await Promise.all([purge, restore]);
+
+    // Whichever went first, both finished and nothing is left.
+    expect(purged.deferredAccounts).toBeLessThanOrEqual(1);
+    expect(await prisma.measurement.count({ where: { userId: OTHER } })).toBe(
+      0,
+    );
+    expect(
+      await prisma.measurement.count({ where: { userId: RESTORING } }),
+    ).toBe(0);
+  }, 60_000);
+});