diff --git a/docs/ops/measurement-maintenance.md b/docs/ops/measurement-maintenance.md index 8429f15f8..d6a7865eb 100644 --- a/docs/ops/measurement-maintenance.md +++ b/docs/ops/measurement-maintenance.md @@ -137,8 +137,8 @@ it. ## Reading the result The run's wide event `job.measurement_maintenance` carries -`maintenance_outcome` (`completed`, `nothing_due`, `refused_purge_running`, -`failed`), the live and dead rows the run started from, the +`maintenance_outcome` (`completed`, `nothing_due` for the nightly check, +`refused_purge_running`, `failed`), the live and dead rows the run started from, the table and index sizes before and after, and `maintenance_steps`, one entry per step with its target, size before and after, and duration. The same sizes from `psql`: @@ -151,17 +151,22 @@ SELECT pg_size_pretty(pg_total_relation_size('measurements')) AS total, ## The nightly check Since v1.43.7 the same queue also runs by itself at 03:40 (Europe/Berlin). -That run names no steps: it reads the table's live and dead rows and runs a -plain `VACUUM (ANALYZE)` only when there are at least 10,000 dead rows and -more than one for every ten live ones. It never rebuilds an index, and a -plain vacuum stops neither reads nor writes. It shares the singleton key with -the run you start, so the two never overlap. The admin card keeps showing -only the runs you started. +That run names no steps and changes nothing: vacuuming the table is +autovacuum's job, which migration 0412 starts at 5 % dead rows instead of +Postgres's 20 %. The check reads the table's live and dead rows and warns +(`maintenance_autovacuum_behind` on the wide event) when there are at least +10,000 dead rows and more than one for every five live ones, four times the +point at which autovacuum should already have run. That means autovacuum is +not keeping up: look for a transaction left open for hours +(`pg_stat_activity`, `state = 'idle in transaction'`) or for autovacuum +workers busy elsewhere. The check never rebuilds an index. It shares the +singleton key with the run you start, so the two never overlap. The admin +card keeps showing only the runs you started. The check also measures how far the table file has grown past what its live rows need (`maintenance_bloat_factor` on the wide event, `bloat_factor` in the -job result). Past 3 it adds a warning to the event: a plain vacuum makes the -space reusable but does not shrink the file, and only `VACUUM FULL` does (see +job result). Past 3 it adds a warning to the event: a vacuum makes the space +reusable but does not shrink the file, and only `VACUUM FULL` does (see below). Why the table grew in the first place: until v1.43.7 an update that changed diff --git a/src/lib/jobs/__tests__/measurement-maintenance-schedule.test.ts b/src/lib/jobs/__tests__/measurement-maintenance-schedule.test.ts index 2a0eb28dc..da6ce0bbd 100644 --- a/src/lib/jobs/__tests__/measurement-maintenance-schedule.test.ts +++ b/src/lib/jobs/__tests__/measurement-maintenance-schedule.test.ts @@ -1,8 +1,11 @@ /** * The nightly measurements check: what it decides from the table's - * statistics. It vacuums only when dead rows pile up and never rebuilds; the + * statistics. It never vacuums and never rebuilds, since autovacuum owns the + * vacuuming (migration 0412); it warns when autovacuum falls behind. The * operator's run keeps the steps it names. */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; vi.mock("@/lib/db", () => ({ @@ -11,10 +14,11 @@ vi.mock("@/lib/db", () => ({ })); import { - DEAD_ROW_VACUUM_FLOOR, + AUTOVACUUM_SCALE_FACTOR, + DEAD_ROW_WARN_FLOOR, + autovacuumBehind, resolveSteps, tableHealth, - vacuumDue, type TableHealth, } from "@/lib/jobs/measurement-maintenance"; @@ -26,40 +30,53 @@ const health = (over: Partial = {}): TableHealth => ({ ...over, }); -describe("vacuumDue", () => { - it("vacuums once dead rows pass a tenth of the live ones", () => { - expect(vacuumDue(health({ deadRows: 100_001 }))).toBe(true); - expect(vacuumDue(health({ deadRows: 99_999 }))).toBe(false); +describe("autovacuumBehind", () => { + it("matches the threshold migration 0412 sets on the table", () => { + const migration = readFileSync( + join( + process.cwd(), + "prisma/migrations/0412_measurement_hot_updates/migration.sql", + ), + "utf8", + ); + expect(migration).toContain( + `autovacuum_vacuum_scale_factor = ${AUTOVACUUM_SCALE_FACTOR}`, + ); + }); + + it("warns once dead rows pass four times autovacuum's threshold", () => { + expect(autovacuumBehind(health({ deadRows: 200_001 }))).toBe(true); + expect(autovacuumBehind(health({ deadRows: 199_999 }))).toBe(false); }); it("leaves a small table alone below the floor", () => { expect( - vacuumDue(health({ liveRows: 100, deadRows: DEAD_ROW_VACUUM_FLOOR - 1 })), + autovacuumBehind( + health({ liveRows: 100, deadRows: DEAD_ROW_WARN_FLOOR - 1 }), + ), ).toBe(false); }); }); describe("resolveSteps", () => { - it("gives the nightly tick a vacuum when one is due and never a rebuild", () => { - expect(resolveSteps({}, health({ deadRows: 500_000 }))).toEqual({ - vacuum: true, - reindex: false, - }); - expect(resolveSteps(undefined, health())).toEqual({ - vacuum: false, - reindex: false, - }); + it("gives the nightly tick no step at all, whatever the dead rows", () => { + expect(resolveSteps({})).toEqual({ vacuum: false, reindex: false }); + expect(resolveSteps(undefined)).toEqual({ vacuum: false, reindex: false }); }); - it("keeps the operator's steps whatever the statistics say", () => { - expect(resolveSteps({ vacuum: false, reindex: true }, health())).toEqual({ + it("keeps the operator's steps", () => { + expect(resolveSteps({ vacuum: false, reindex: true })).toEqual({ vacuum: false, reindex: true, }); - expect(resolveSteps({ vacuum: true, reindex: false }, health())).toEqual({ + expect(resolveSteps({ vacuum: true, reindex: false })).toEqual({ vacuum: true, reindex: false, }); + expect(resolveSteps({ vacuum: true })).toEqual({ + vacuum: true, + reindex: true, + }); }); }); diff --git a/src/lib/jobs/measurement-maintenance.ts b/src/lib/jobs/measurement-maintenance.ts index b48ad7b96..a5e4efa0f 100644 --- a/src/lib/jobs/measurement-maintenance.ts +++ b/src/lib/jobs/measurement-maintenance.ts @@ -11,12 +11,16 @@ * (cookie-only admin), with the steps named; the rebuild only ever runs * this way, in a window the operator picks. * - The nightly check (`MEASUREMENT_MAINTENANCE_CRON`, a tick without - * steps). It reads the table's dead and live rows and vacuums only when - * the dead ones pass `vacuumDue`. A plain VACUUM takes no lock that stops - * reads or writes. It also measures how far the heap has grown past its - * live rows (`TableHealth.bloatFactor`); only a rewrite (`VACUUM FULL`, - * which locks the table) gives that space back, so past - * `TABLE_BLOAT_WARN_FACTOR` the run warns instead of acting. + * steps). It measures and never acts. Vacuuming the table is + * autovacuum's job, at the 5 % dead-row threshold migration 0412 gives + * the table; a second trigger here at 10 % never fired before autovacuum + * had already run, and two mechanisms for one task only hide which of + * them is doing it. The check warns when the dead rows pass + * `AUTOVACUUM_BEHIND_SHARE`, which means autovacuum is not keeping up + * (a long transaction holding it back, too few workers), and when the + * heap has grown past its live rows (`TableHealth.bloatFactor`) further + * than `TABLE_BLOAT_WARN_FACTOR`, which only a rewrite (`VACUUM FULL`, + * which locks the table) gives back. * * Why the indexes: after the purge the heap is small, but a B-tree does not * give back the pages its deleted entries held. VACUUM marks the heap space @@ -84,11 +88,17 @@ export interface MaintenanceSteps { /** 03:40 Europe/Berlin, inside the nightly maintenance band. */ export const MEASUREMENT_MAINTENANCE_CRON = "40 3 * * *"; -/** Dead rows, as a share of live ones, past which the nightly check vacuums. */ -export const DEAD_ROW_VACUUM_SHARE = 0.1; +/** The dead-row share at which autovacuum starts on the table (0412). */ +export const AUTOVACUUM_SCALE_FACTOR = 0.05; -/** Fewer dead rows than this are never worth a vacuum of their own. */ -export const DEAD_ROW_VACUUM_FLOOR = 10_000; +/** + * Dead rows, as a share of live ones, past which the nightly check warns + * that autovacuum is behind: four times its own threshold. + */ +export const AUTOVACUUM_BEHIND_SHARE = 4 * AUTOVACUUM_SCALE_FACTOR; + +/** Fewer dead rows than this are never worth a warning. */ +export const DEAD_ROW_WARN_FLOOR = 10_000; /** * Heap size against what the live rows need, past which the run warns that @@ -230,26 +240,25 @@ export async function tableHealth( }; } -/** Whether the dead rows are worth a vacuum of their own tonight. */ -export function vacuumDue(health: TableHealth): boolean { +/** Whether the dead rows say autovacuum is not keeping up with the table. */ +export function autovacuumBehind(health: TableHealth): boolean { return ( - health.deadRows >= DEAD_ROW_VACUUM_FLOOR && - health.deadRows > health.liveRows * DEAD_ROW_VACUUM_SHARE + health.deadRows >= DEAD_ROW_WARN_FLOOR && + health.deadRows > health.liveRows * AUTOVACUUM_BEHIND_SHARE ); } /** - * The steps a delivery takes: the operator's as asked, the nightly tick's a - * vacuum when one is due and never a rebuild. + * The steps a delivery takes: the operator's as asked, none for the nightly + * tick, which only measures. */ export function resolveSteps( payload: MeasurementMaintenancePayload | undefined, - health: TableHealth, ): MaintenanceSteps { const named = typeof payload?.vacuum === "boolean" || typeof payload?.reindex === "boolean"; - if (!named) return { vacuum: vacuumDue(health), reindex: false }; + if (!named) return { vacuum: false, reindex: false }; return { vacuum: payload?.vacuum ?? true, reindex: payload?.reindex ?? true }; } @@ -459,7 +468,7 @@ export async function handleMeasurementMaintenance( } } const health = await tableHealth(client); - const steps = resolveSteps(job?.data, health); + const steps = resolveSteps(job?.data); const summary = await runMeasurementMaintenance(client, steps); return { summary, health, steps }; } finally { @@ -486,6 +495,12 @@ export async function handleMeasurementMaintenance( evt.addMeta("maintenance_live_rows", health.liveRows); evt.addMeta("maintenance_dead_rows", health.deadRows); evt.addMeta("maintenance_heap_bytes", health.heapBytes); + if (autovacuumBehind(health)) { + evt.addMeta("maintenance_autovacuum_behind", true); + evt.addWarning( + `measurements holds ${health.deadRows} dead rows against ${health.liveRows} live; autovacuum starts at ${AUTOVACUUM_SCALE_FACTOR * 100} % and is not keeping up (look for a long-open transaction or too few autovacuum workers)`, + ); + } if (health.bloatFactor !== null) { evt.addMeta("maintenance_bloat_factor", health.bloatFactor); if (health.bloatFactor > TABLE_BLOAT_WARN_FACTOR) {