fix(maintenance): leave vacuuming measurements to autovacuum

Migration 0412 starts autovacuum on the table at 5 % dead rows, and the
nightly check vacuumed at 10 %, a point autovacuum had always passed first.
Two triggers for one task only hid which of them did it. The nightly check
now measures and changes nothing: it warns when the dead rows reach four
times autovacuum's threshold, which means autovacuum is not keeping up, and
when the heap has outgrown its live rows. The operator's run keeps its
steps.

docs/ops/measurement-maintenance.md

@@ -137,8 +137,8 @@ it.
 ## Reading the result
 
 The run's wide event `job.measurement_maintenance` carries
-`maintenance_outcome` (`completed`, `nothing_due`, `refused_purge_running`,
-`failed`), the live and dead rows the run started from, the
+`maintenance_outcome` (`completed`, `nothing_due` for the nightly check,
+`refused_purge_running`, `failed`), the live and dead rows the run started from, the
 table and index sizes before and after, and `maintenance_steps`, one entry
 per step with its target, size before and after, and duration. The same
 sizes from `psql`:
@@ -151,17 +151,22 @@ SELECT pg_size_pretty(pg_total_relation_size('measurements')) AS total,
 ## The nightly check
 
 Since v1.43.7 the same queue also runs by itself at 03:40 (Europe/Berlin).
-That run names no steps: it reads the table's live and dead rows and runs a
-plain `VACUUM (ANALYZE)` only when there are at least 10,000 dead rows and
-more than one for every ten live ones. It never rebuilds an index, and a
-plain vacuum stops neither reads nor writes. It shares the singleton key with
-the run you start, so the two never overlap. The admin card keeps showing
-only the runs you started.
+That run names no steps and changes nothing: vacuuming the table is
+autovacuum's job, which migration 0412 starts at 5 % dead rows instead of
+Postgres's 20 %. The check reads the table's live and dead rows and warns
+(`maintenance_autovacuum_behind` on the wide event) when there are at least
+10,000 dead rows and more than one for every five live ones, four times the
+point at which autovacuum should already have run. That means autovacuum is
+not keeping up: look for a transaction left open for hours
+(`pg_stat_activity`, `state = 'idle in transaction'`) or for autovacuum
+workers busy elsewhere. The check never rebuilds an index. It shares the
+singleton key with the run you start, so the two never overlap. The admin
+card keeps showing only the runs you started.
 
 The check also measures how far the table file has grown past what its live
 rows need (`maintenance_bloat_factor` on the wide event, `bloat_factor` in the
-job result). Past 3 it adds a warning to the event: a plain vacuum makes the
-space reusable but does not shrink the file, and only `VACUUM FULL` does (see
+job result). Past 3 it adds a warning to the event: a vacuum makes the space
+reusable but does not shrink the file, and only `VACUUM FULL` does (see
 below).
 
 Why the table grew in the first place: until v1.43.7 an update that changed

src/lib/jobs/__tests__/measurement-maintenance-schedule.test.ts

@@ -1,8 +1,11 @@
 /**
  * The nightly measurements check: what it decides from the table's
- * statistics. It vacuums only when dead rows pile up and never rebuilds; the
+ * statistics. It never vacuums and never rebuilds, since autovacuum owns the
+ * vacuuming (migration 0412); it warns when autovacuum falls behind. The
  * operator's run keeps the steps it names.
  */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
 import { describe, expect, it, vi } from "vitest";
 
 vi.mock("@/lib/db", () => ({
@@ -11,10 +14,11 @@ vi.mock("@/lib/db", () => ({
 }));
 
 import {
-  DEAD_ROW_VACUUM_FLOOR,
+  AUTOVACUUM_SCALE_FACTOR,
+  DEAD_ROW_WARN_FLOOR,
+  autovacuumBehind,
   resolveSteps,
   tableHealth,
-  vacuumDue,
   type TableHealth,
 } from "@/lib/jobs/measurement-maintenance";
 
@@ -26,40 +30,53 @@ const health = (over: Partial<TableHealth> = {}): TableHealth => ({
   ...over,
 });
 
-describe("vacuumDue", () => {
-  it("vacuums once dead rows pass a tenth of the live ones", () => {
-    expect(vacuumDue(health({ deadRows: 100_001 }))).toBe(true);
-    expect(vacuumDue(health({ deadRows: 99_999 }))).toBe(false);
+describe("autovacuumBehind", () => {
+  it("matches the threshold migration 0412 sets on the table", () => {
+    const migration = readFileSync(
+      join(
+        process.cwd(),
+        "prisma/migrations/0412_measurement_hot_updates/migration.sql",
+      ),
+      "utf8",
+    );
+    expect(migration).toContain(
+      `autovacuum_vacuum_scale_factor = ${AUTOVACUUM_SCALE_FACTOR}`,
+    );
+  });
+
+  it("warns once dead rows pass four times autovacuum's threshold", () => {
+    expect(autovacuumBehind(health({ deadRows: 200_001 }))).toBe(true);
+    expect(autovacuumBehind(health({ deadRows: 199_999 }))).toBe(false);
   });
 
   it("leaves a small table alone below the floor", () => {
     expect(
-      vacuumDue(health({ liveRows: 100, deadRows: DEAD_ROW_VACUUM_FLOOR - 1 })),
+      autovacuumBehind(
+        health({ liveRows: 100, deadRows: DEAD_ROW_WARN_FLOOR - 1 }),
+      ),
     ).toBe(false);
   });
 });
 
 describe("resolveSteps", () => {
-  it("gives the nightly tick a vacuum when one is due and never a rebuild", () => {
-    expect(resolveSteps({}, health({ deadRows: 500_000 }))).toEqual({
-      vacuum: true,
-      reindex: false,
-    });
-    expect(resolveSteps(undefined, health())).toEqual({
-      vacuum: false,
-      reindex: false,
-    });
+  it("gives the nightly tick no step at all, whatever the dead rows", () => {
+    expect(resolveSteps({})).toEqual({ vacuum: false, reindex: false });
+    expect(resolveSteps(undefined)).toEqual({ vacuum: false, reindex: false });
   });
 
-  it("keeps the operator's steps whatever the statistics say", () => {
-    expect(resolveSteps({ vacuum: false, reindex: true }, health())).toEqual({
+  it("keeps the operator's steps", () => {
+    expect(resolveSteps({ vacuum: false, reindex: true })).toEqual({
       vacuum: false,
       reindex: true,
     });
-    expect(resolveSteps({ vacuum: true, reindex: false }, health())).toEqual({
+    expect(resolveSteps({ vacuum: true, reindex: false })).toEqual({
       vacuum: true,
       reindex: false,
     });
+    expect(resolveSteps({ vacuum: true })).toEqual({
+      vacuum: true,
+      reindex: true,
+    });
   });
 });
 

src/lib/jobs/measurement-maintenance.ts

@@ -11,12 +11,16 @@
  *     (cookie-only admin), with the steps named; the rebuild only ever runs
  *     this way, in a window the operator picks.
  *   - The nightly check (`MEASUREMENT_MAINTENANCE_CRON`, a tick without
- *     steps). It reads the table's dead and live rows and vacuums only when
- *     the dead ones pass `vacuumDue`. A plain VACUUM takes no lock that stops
- *     reads or writes. It also measures how far the heap has grown past its
- *     live rows (`TableHealth.bloatFactor`); only a rewrite (`VACUUM FULL`,
- *     which locks the table) gives that space back, so past
- *     `TABLE_BLOAT_WARN_FACTOR` the run warns instead of acting.
+ *     steps). It measures and never acts. Vacuuming the table is
+ *     autovacuum's job, at the 5 % dead-row threshold migration 0412 gives
+ *     the table; a second trigger here at 10 % never fired before autovacuum
+ *     had already run, and two mechanisms for one task only hide which of
+ *     them is doing it. The check warns when the dead rows pass
+ *     `AUTOVACUUM_BEHIND_SHARE`, which means autovacuum is not keeping up
+ *     (a long transaction holding it back, too few workers), and when the
+ *     heap has grown past its live rows (`TableHealth.bloatFactor`) further
+ *     than `TABLE_BLOAT_WARN_FACTOR`, which only a rewrite (`VACUUM FULL`,
+ *     which locks the table) gives back.
  *
  * Why the indexes: after the purge the heap is small, but a B-tree does not
  * give back the pages its deleted entries held. VACUUM marks the heap space
@@ -84,11 +88,17 @@ export interface MaintenanceSteps {
 /** 03:40 Europe/Berlin, inside the nightly maintenance band. */
 export const MEASUREMENT_MAINTENANCE_CRON = "40 3 * * *";
 
-/** Dead rows, as a share of live ones, past which the nightly check vacuums. */
-export const DEAD_ROW_VACUUM_SHARE = 0.1;
+/** The dead-row share at which autovacuum starts on the table (0412). */
+export const AUTOVACUUM_SCALE_FACTOR = 0.05;
 
-/** Fewer dead rows than this are never worth a vacuum of their own. */
-export const DEAD_ROW_VACUUM_FLOOR = 10_000;
+/**
+ * Dead rows, as a share of live ones, past which the nightly check warns
+ * that autovacuum is behind: four times its own threshold.
+ */
+export const AUTOVACUUM_BEHIND_SHARE = 4 * AUTOVACUUM_SCALE_FACTOR;
+
+/** Fewer dead rows than this are never worth a warning. */
+export const DEAD_ROW_WARN_FLOOR = 10_000;
 
 /**
  * Heap size against what the live rows need, past which the run warns that
@@ -230,26 +240,25 @@ export async function tableHealth(
   };
 }
 
-/** Whether the dead rows are worth a vacuum of their own tonight. */
-export function vacuumDue(health: TableHealth): boolean {
+/** Whether the dead rows say autovacuum is not keeping up with the table. */
+export function autovacuumBehind(health: TableHealth): boolean {
   return (
-    health.deadRows >= DEAD_ROW_VACUUM_FLOOR &&
-    health.deadRows > health.liveRows * DEAD_ROW_VACUUM_SHARE
+    health.deadRows >= DEAD_ROW_WARN_FLOOR &&
+    health.deadRows > health.liveRows * AUTOVACUUM_BEHIND_SHARE
   );
 }
 
 /**
- * The steps a delivery takes: the operator's as asked, the nightly tick's a
- * vacuum when one is due and never a rebuild.
+ * The steps a delivery takes: the operator's as asked, none for the nightly
+ * tick, which only measures.
  */
 export function resolveSteps(
   payload: MeasurementMaintenancePayload | undefined,
-  health: TableHealth,
 ): MaintenanceSteps {
   const named =
     typeof payload?.vacuum === "boolean" ||
     typeof payload?.reindex === "boolean";
-  if (!named) return { vacuum: vacuumDue(health), reindex: false };
+  if (!named) return { vacuum: false, reindex: false };
   return { vacuum: payload?.vacuum ?? true, reindex: payload?.reindex ?? true };
 }
 
@@ -459,7 +468,7 @@ export async function handleMeasurementMaintenance(
             }
           }
           const health = await tableHealth(client);
-          const steps = resolveSteps(job?.data, health);
+          const steps = resolveSteps(job?.data);
           const summary = await runMeasurementMaintenance(client, steps);
           return { summary, health, steps };
         } finally {
@@ -486,6 +495,12 @@ export async function handleMeasurementMaintenance(
     evt.addMeta("maintenance_live_rows", health.liveRows);
     evt.addMeta("maintenance_dead_rows", health.deadRows);
     evt.addMeta("maintenance_heap_bytes", health.heapBytes);
+    if (autovacuumBehind(health)) {
+      evt.addMeta("maintenance_autovacuum_behind", true);
+      evt.addWarning(
+        `measurements holds ${health.deadRows} dead rows against ${health.liveRows} live; autovacuum starts at ${AUTOVACUUM_SCALE_FACTOR * 100} % and is not keeping up (look for a long-open transaction or too few autovacuum workers)`,
+      );
+    }
     if (health.bloatFactor !== null) {
       evt.addMeta("maintenance_bloat_factor", health.bloatFactor);
       if (health.bloatFactor > TABLE_BLOAT_WARN_FACTOR) {