diff --git a/docker-compose.yml b/docker-compose.yml index 02a2b3257..eaacd11d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -554,6 +554,10 @@ services: - shared_preload_libraries=pg_stat_statements - -c - pg_stat_statements.track=top + # Docker gives a container 64 MB of /dev/shm by default. Postgres keeps + # the shared memory of parallel queries and parallel maintenance there, + # and a larger plan fails with "could not resize shared memory segment". + shm_size: 256mb # cap_drop: [ALL] is deliberately NOT applied here (unlike `app`): the # official Postgres entrypoint starts as root, chowns the data # directory, and drops privileges to the `postgres` user itself, which diff --git a/docs/ops/measurement-maintenance.md b/docs/ops/measurement-maintenance.md index aee952f08..2576b3fcd 100644 --- a/docs/ops/measurement-maintenance.md +++ b/docs/ops/measurement-maintenance.md @@ -147,6 +147,16 @@ SELECT pg_size_pretty(pg_total_relation_size('measurements')) AS total, pg_size_pretty(pg_indexes_size('measurements')) AS indexes; ``` +## "could not resize shared memory segment" + +Up to v1.42.0 the run could stop at once with `could not resize shared memory +segment ... No space left on device`. Postgres ran the vacuum in parallel and +kept its working memory in `/dev/shm`, which Docker limits to 64 MB per +container unless `shm_size` is set. From v1.42.1 the maintenance connection +runs single-process and needs no shared segment, and the bundled +`docker-compose.yml` gives the database `shm_size: 256mb`. On v1.42.0, add +`shm_size: 256mb` to the `db` service and restart the database container. + ## Not part of this `VACUUM FULL` would shrink the table file as well, but it locks the table for diff --git a/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts b/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts index e2b6ffe9c..93ba20002 100644 --- a/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts +++ b/src/lib/jobs/__tests__/measurement-maintenance-session.test.ts @@ -24,7 +24,13 @@ describe("maintenance session settings behind a pooler", () => { "SET statement_timeout = '0'", "SET idle_in_transaction_session_timeout = '0'", "SET maintenance_work_mem = '128MB'", + "SET max_parallel_maintenance_workers = '0'", "SET lock_timeout = '300000'", ]); }); + it("keeps maintenance in one process so it needs no /dev/shm segment", () => { + expect(MAINTENANCE_SESSION_OPTIONS).toContain( + "-c max_parallel_maintenance_workers=0", + ); + }); }); diff --git a/src/lib/jobs/measurement-maintenance.ts b/src/lib/jobs/measurement-maintenance.ts index fa5305baf..72a7c6a03 100644 --- a/src/lib/jobs/measurement-maintenance.ts +++ b/src/lib/jobs/measurement-maintenance.ts @@ -65,9 +65,19 @@ export interface MeasurementMaintenancePayload { reindex: boolean; } -/** Session settings of the maintenance connection. */ +/** + * Session settings of the maintenance connection. + * + * `max_parallel_maintenance_workers=0` keeps VACUUM and the index rebuild in + * one process. A parallel run shares its `maintenance_work_mem` through + * dynamic shared memory in `/dev/shm`, which Docker caps at 64 MB unless the + * container sets `shm_size`; a 128 MB budget then fails at once with + * "could not resize shared memory segment ... No space left on device". + * One process needs no shared segment, so the run works on any host, + * whatever its container settings. + */ export const MAINTENANCE_SESSION_OPTIONS = - "-c statement_timeout=0 -c idle_in_transaction_session_timeout=0 -c maintenance_work_mem=128MB -c lock_timeout=300000"; + "-c statement_timeout=0 -c idle_in_transaction_session_timeout=0 -c maintenance_work_mem=128MB -c max_parallel_maintenance_workers=0 -c lock_timeout=300000"; /** * The same settings as `SET` statements, for a host behind a connection