From d8fcf814b29d8b4a4bd0ef1fad844a204b9bd2df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 22:03:48 +0200 Subject: [PATCH] fix(docker): a boot storm cannot flip the healthcheck to unhealthy /api/health gives the database two seconds and answers 503 past that. With three retries, a pool saturated for a minute after the start period could mark the container unhealthy and take it out of the proxy's routing. The compose healthcheck and the image's HEALTHCHECK now allow five misses (two and a half minutes) and end wget after four seconds, inside Docker's five, so a stalled request is one clean miss. A test holds both to these bounds and to each other. --- Dockerfile | 7 +- docker-compose.yml | 12 +- docs/self-hosting/truenas.md | 2 +- src/__tests__/healthcheck-boot-storm.test.ts | 113 +++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 src/__tests__/healthcheck-boot-storm.test.ts diff --git a/Dockerfile b/Dockerfile index 6f58628aa..338364c89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -320,8 +320,11 @@ ENV HOSTNAME="0.0.0.0" # Start period: the entrypoint migrates before the server listens, and a # migration waiting out a held lock retries for up to about eight minutes. -HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=3 \ - CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1 +# Five retries: a pool saturated for a minute during a boot storm makes the +# probe's 2 s database check miss once or twice without marking the container +# unhealthy (docker-compose.yml explains the numbers). +HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=5 \ + CMD wget --no-verbose --tries=1 --timeout=4 --spider http://127.0.0.1:3000/api/health || exit 1 ENTRYPOINT ["./docker-entrypoint.sh"] CMD ["node", "server.js"] diff --git a/docker-compose.yml b/docker-compose.yml index c3aee3175..dad7cf80d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -512,11 +512,19 @@ services: test: [ "CMD-SHELL", - "wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1", + "wget --no-verbose --tries=1 --timeout=4 --spider http://127.0.0.1:3000/api/health || exit 1", ] interval: 30s timeout: 5s - retries: 3 + # Five misses in a row, two and a half minutes, before "unhealthy". The + # probe gives the database two seconds to answer SELECT 1 and says 503 + # past that, so a boot storm that saturates the pool for a minute after + # the start period (backfills, a restore, another tenant's import) costs + # a probe or two, not the container's health and the proxy's routing. + # wget's own 4 s timeout ends the request inside Docker's 5 s, so a + # stalled connection reads as one clean miss rather than a killed probe. + # A database that stays away for 2.5 minutes is a real outage and shows. + retries: 5 # The entrypoint applies migrations before the server listens. A # migration that waits out a held lock retries for up to about eight # minutes (docs/ops/deploy.md), and failed probes in that time must not diff --git a/docs/self-hosting/truenas.md b/docs/self-hosting/truenas.md index 0b34fe4eb..3f7f6abcb 100644 --- a/docs/self-hosting/truenas.md +++ b/docs/self-hosting/truenas.md @@ -13,7 +13,7 @@ one value you must keep yourself: the encryption key. | Port | `3000` (HTTP) | | User | runs as uid `1001`; no extra capabilities, `no-new-privileges` is fine | | Health check | `GET /api/health`, `200` when ready | -| Start period | at least 60 s, with more retries than the image's 3 × 30 s: migrations run at every boot | +| Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot | | Database | PostgreSQL 16 (17 and later are not a drop-in), no extensions | | Volumes | none for the app; the database gets its own dataset | diff --git a/src/__tests__/healthcheck-boot-storm.test.ts b/src/__tests__/healthcheck-boot-storm.test.ts new file mode 100644 index 000000000..e91b7401b --- /dev/null +++ b/src/__tests__/healthcheck-boot-storm.test.ts @@ -0,0 +1,113 @@ +/** + * The container healthcheck cannot flap during a boot storm. + * + * `/api/health` gives the database two seconds to answer `SELECT 1` and says + * 503 past that. After the start period a saturated pool (backfills, a + * restore, a large import) can make one or two probes miss; the healthcheck + * has to absorb that and still report an outage that lasts. The compose file + * and the image's own HEALTHCHECK must agree, since the published image runs + * outside compose too. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const ROOT = process.cwd(); + +function seconds(value: string): number { + const m = /^(\d+)(ms|s|m)$/.exec(value.trim()); + if (!m) throw new Error(`unreadable duration ${value}`); + const n = Number(m[1]); + return m[2] === "ms" ? n / 1000 : m[2] === "m" ? n * 60 : n; +} + +const probeTimeoutSeconds = (() => { + const src = readFileSync(join(ROOT, "src/app/api/health/route.ts"), "utf8"); + const m = /DB_PROBE_TIMEOUT_MS = ([\d_]+);/.exec(src); + if (!m) throw new Error("DB_PROBE_TIMEOUT_MS not found"); + return Number(m[1]!.replace(/_/g, "")) / 1000; +})(); + +interface Check { + interval: number; + timeout: number; + retries: number; + command: string; +} + +function composeCheck(): Check { + const compose = parse( + readFileSync(join(ROOT, "docker-compose.yml"), "utf8"), + ) as { + services: { + app: { + healthcheck: { + test: string[]; + interval: string; + timeout: string; + retries: number; + }; + }; + }; + }; + const hc = compose.services.app.healthcheck; + return { + interval: seconds(hc.interval), + timeout: seconds(hc.timeout), + retries: hc.retries, + command: hc.test.join(" "), + }; +} + +function imageCheck(): Check { + const dockerfile = readFileSync(join(ROOT, "Dockerfile"), "utf8"); + const m = /HEALTHCHECK ([^\n]*\\\n[^\n]*)/.exec(dockerfile); + if (!m) throw new Error("no HEALTHCHECK in the Dockerfile"); + const line = m[1]!; + const flag = (name: string) => { + const f = new RegExp(`--${name}=(\\S+)`).exec(line); + if (!f) throw new Error(`HEALTHCHECK has no --${name}`); + return f[1]!; + }; + return { + interval: seconds(flag("interval")), + timeout: seconds(flag("timeout")), + retries: Number(flag("retries")), + command: line.slice(line.indexOf("CMD")), + }; +} + +function wgetTimeout(command: string): number { + const m = /wget[^|]*--timeout=(\d+)/.exec(command); + if (!m) throw new Error(`no wget timeout in: ${command}`); + return Number(m[1]); +} + +describe.each([ + ["docker-compose.yml", composeCheck], + ["Dockerfile", imageCheck], +])("%s healthcheck", (_name, read) => { + it("rides out two minutes of slow probes before calling the app unhealthy", () => { + const check = read(); + expect(check.interval * check.retries).toBeGreaterThanOrEqual(120); + }); + + it("ends the request inside Docker's timeout, after the database probe's", () => { + const check = read(); + const wget = wgetTimeout(check.command); + expect(wget).toBeGreaterThan(probeTimeoutSeconds); + expect(wget).toBeLessThan(check.timeout); + }); +}); + +it("compose and the image agree", () => { + const a = composeCheck(); + const b = imageCheck(); + expect([a.interval, a.timeout, a.retries]).toEqual([ + b.interval, + b.timeout, + b.retries, + ]); +});