Files
HealthLog/tests/integration/tombstone-cleanup-restore-lock.test.ts
T

160 lines
5.4 KiB
TypeScript

/**
* #1031 — the nightly tombstone purge against a running restore.
*
* A restore clears every row of the account, tombstones included, inside one
* long transaction that holds the account's restore lock exclusively. The
* purge used to delete expired tombstones across accounts in one statement
* per batch, without the lock: on an account under restore it waited on the
* restore's row locks for as long as the restore ran, or deadlocked with it.
*
* Here a transaction stands in for the restore: it takes the lock, deletes the
* account's rows, and holds for three seconds before rolling back. The purge
* runs while it holds. What must hold: the purge does not wait, leaves the
* account under restore alone and says so, and still purges every other
* account.
*
* Mutation check: drop the `holdAccountAgainstRestore` call from
* `purgeTombstonesByAccount` and the purge waits out the hold, then deletes
* the restoring account's tombstones once the stand-in rolls back.
*/
import { afterAll, beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
const RESTORING = "tombstone-restoring";
const OTHER = "tombstone-other";
const HOLD_MS = 3_000;
const LONG_AGO = new Date("2020-01-01T00:00:00.000Z");
async function seed() {
const prisma = getPrismaClient();
for (const id of [RESTORING, OTHER]) {
await prisma.user.create({
data: { id, username: id, role: "USER", timezone: "UTC" },
});
for (let i = 0; i < 3; i++) {
await prisma.measurement.create({
data: {
id: `${id}-tomb-${i}`,
userId: id,
type: "WEIGHT",
value: 70 + i,
unit: "kg",
source: "MANUAL",
measuredAt: new Date(`2020-01-0${i + 1}T08:00:00.000Z`),
deletedAt: LONG_AGO,
},
});
}
await prisma.moodEntry.create({
data: {
id: `${id}-mood-tomb`,
userId: id,
date: "2020-01-01",
mood: "OKAY",
score: 3,
moodLoggedAt: new Date("2020-01-01T08:00:00.000Z"),
deletedAt: LONG_AGO,
},
});
}
}
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
});
afterAll(async () => {
await truncateAllTables(getPrismaClient());
});
describe("tombstone purge against a running restore (#1031)", () => {
it("leaves the account under restore for the next run without waiting on it", async () => {
const prisma = getPrismaClient();
await seed();
const { takeRestoreLock } = await import("@/lib/export/restore-lock");
const {
cleanupExpiredMeasurementTombstones,
cleanupExpiredMoodTombstones,
} = await import("@/lib/jobs/measurement-tombstone-cleanup");
let holding!: () => void;
const isHolding = new Promise<void>((resolve) => (holding = resolve));
const restore = prisma
.$transaction(
async (tx) => {
await takeRestoreLock(tx, RESTORING);
await tx.measurement.deleteMany({ where: { userId: RESTORING } });
await tx.moodEntry.deleteMany({ where: { userId: RESTORING } });
holding();
await new Promise((resolve) => setTimeout(resolve, HOLD_MS));
throw new Error("rolled back");
},
{ timeout: 30_000 },
)
.catch(() => undefined);
await isHolding;
const started = Date.now();
const measurements = await cleanupExpiredMeasurementTombstones(prisma);
const moods = await cleanupExpiredMoodTombstones(prisma);
const elapsed = Date.now() - started;
await restore;
expect(elapsed).toBeLessThan(HOLD_MS / 2);
expect(measurements).toEqual({
deleted: 3,
drained: true,
deferredAccounts: 1,
deferredUserIds: [RESTORING],
});
expect(moods).toEqual({
deleted: 1,
drained: true,
deferredAccounts: 1,
deferredUserIds: [RESTORING],
});
// The stand-in rolled back, so the restoring account still has its
// tombstones; the next run purges them.
expect(
await prisma.measurement.count({ where: { userId: RESTORING } }),
).toBe(3);
expect(await prisma.measurement.count({ where: { userId: OTHER } })).toBe(
0,
);
expect(await prisma.moodEntry.count({ where: { userId: OTHER } })).toBe(0);
const next = await cleanupExpiredMeasurementTombstones(prisma);
expect(next).toEqual({
deleted: 3,
drained: true,
deferredAccounts: 0,
deferredUserIds: [],
});
}, 60_000);
it("a restore that starts during the purge waits only for the account's batch", async () => {
const prisma = getPrismaClient();
await seed();
const { takeRestoreLock } = await import("@/lib/export/restore-lock");
const { cleanupExpiredMeasurementTombstones } =
await import("@/lib/jobs/measurement-tombstone-cleanup");
const purge = cleanupExpiredMeasurementTombstones(prisma);
const restore = prisma.$transaction(async (tx) => {
await takeRestoreLock(tx, RESTORING);
return tx.measurement.deleteMany({ where: { userId: RESTORING } });
});
const [purged] = await Promise.all([purge, restore]);
// Whichever went first, both finished and nothing is left.
expect(purged.deferredAccounts).toBeLessThanOrEqual(1);
expect(await prisma.measurement.count({ where: { userId: OTHER } })).toBe(
0,
);
expect(
await prisma.measurement.count({ where: { userId: RESTORING } }),
).toBe(0);
}, 60_000);
});