diff --git a/Dockerfile b/Dockerfile index 6f58628aa..338364c89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -320,8 +320,11 @@ ENV HOSTNAME="0.0.0.0" # Start period: the entrypoint migrates before the server listens, and a # migration waiting out a held lock retries for up to about eight minutes. -HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=3 \ - CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1 +# Five retries: a pool saturated for a minute during a boot storm makes the +# probe's 2 s database check miss once or twice without marking the container +# unhealthy (docker-compose.yml explains the numbers). +HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=5 \ + CMD wget --no-verbose --tries=1 --timeout=4 --spider http://127.0.0.1:3000/api/health || exit 1 ENTRYPOINT ["./docker-entrypoint.sh"] CMD ["node", "server.js"] diff --git a/docker-compose.yml b/docker-compose.yml index c3aee3175..dad7cf80d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -512,11 +512,19 @@ services: test: [ "CMD-SHELL", - "wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1", + "wget --no-verbose --tries=1 --timeout=4 --spider http://127.0.0.1:3000/api/health || exit 1", ] interval: 30s timeout: 5s - retries: 3 + # Five misses in a row, two and a half minutes, before "unhealthy". The + # probe gives the database two seconds to answer SELECT 1 and says 503 + # past that, so a boot storm that saturates the pool for a minute after + # the start period (backfills, a restore, another tenant's import) costs + # a probe or two, not the container's health and the proxy's routing. + # wget's own 4 s timeout ends the request inside Docker's 5 s, so a + # stalled connection reads as one clean miss rather than a killed probe. + # A database that stays away for 2.5 minutes is a real outage and shows. + retries: 5 # The entrypoint applies migrations before the server listens. A # migration that waits out a held lock retries for up to about eight # minutes (docs/ops/deploy.md), and failed probes in that time must not diff --git a/docs/self-hosting/truenas.md b/docs/self-hosting/truenas.md index 0b34fe4eb..3f7f6abcb 100644 --- a/docs/self-hosting/truenas.md +++ b/docs/self-hosting/truenas.md @@ -13,7 +13,7 @@ one value you must keep yourself: the encryption key. | Port | `3000` (HTTP) | | User | runs as uid `1001`; no extra capabilities, `no-new-privileges` is fine | | Health check | `GET /api/health`, `200` when ready | -| Start period | at least 60 s, with more retries than the image's 3 × 30 s: migrations run at every boot | +| Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot | | Database | PostgreSQL 16 (17 and later are not a drop-in), no extensions | | Volumes | none for the app; the database gets its own dataset | diff --git a/src/__tests__/healthcheck-boot-storm.test.ts b/src/__tests__/healthcheck-boot-storm.test.ts new file mode 100644 index 000000000..e91b7401b --- /dev/null +++ b/src/__tests__/healthcheck-boot-storm.test.ts @@ -0,0 +1,113 @@ +/** + * The container healthcheck cannot flap during a boot storm. + * + * `/api/health` gives the database two seconds to answer `SELECT 1` and says + * 503 past that. After the start period a saturated pool (backfills, a + * restore, a large import) can make one or two probes miss; the healthcheck + * has to absorb that and still report an outage that lasts. The compose file + * and the image's own HEALTHCHECK must agree, since the published image runs + * outside compose too. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const ROOT = process.cwd(); + +function seconds(value: string): number { + const m = /^(\d+)(ms|s|m)$/.exec(value.trim()); + if (!m) throw new Error(`unreadable duration ${value}`); + const n = Number(m[1]); + return m[2] === "ms" ? n / 1000 : m[2] === "m" ? n * 60 : n; +} + +const probeTimeoutSeconds = (() => { + const src = readFileSync(join(ROOT, "src/app/api/health/route.ts"), "utf8"); + const m = /DB_PROBE_TIMEOUT_MS = ([\d_]+);/.exec(src); + if (!m) throw new Error("DB_PROBE_TIMEOUT_MS not found"); + return Number(m[1]!.replace(/_/g, "")) / 1000; +})(); + +interface Check { + interval: number; + timeout: number; + retries: number; + command: string; +} + +function composeCheck(): Check { + const compose = parse( + readFileSync(join(ROOT, "docker-compose.yml"), "utf8"), + ) as { + services: { + app: { + healthcheck: { + test: string[]; + interval: string; + timeout: string; + retries: number; + }; + }; + }; + }; + const hc = compose.services.app.healthcheck; + return { + interval: seconds(hc.interval), + timeout: seconds(hc.timeout), + retries: hc.retries, + command: hc.test.join(" "), + }; +} + +function imageCheck(): Check { + const dockerfile = readFileSync(join(ROOT, "Dockerfile"), "utf8"); + const m = /HEALTHCHECK ([^\n]*\\\n[^\n]*)/.exec(dockerfile); + if (!m) throw new Error("no HEALTHCHECK in the Dockerfile"); + const line = m[1]!; + const flag = (name: string) => { + const f = new RegExp(`--${name}=(\\S+)`).exec(line); + if (!f) throw new Error(`HEALTHCHECK has no --${name}`); + return f[1]!; + }; + return { + interval: seconds(flag("interval")), + timeout: seconds(flag("timeout")), + retries: Number(flag("retries")), + command: line.slice(line.indexOf("CMD")), + }; +} + +function wgetTimeout(command: string): number { + const m = /wget[^|]*--timeout=(\d+)/.exec(command); + if (!m) throw new Error(`no wget timeout in: ${command}`); + return Number(m[1]); +} + +describe.each([ + ["docker-compose.yml", composeCheck], + ["Dockerfile", imageCheck], +])("%s healthcheck", (_name, read) => { + it("rides out two minutes of slow probes before calling the app unhealthy", () => { + const check = read(); + expect(check.interval * check.retries).toBeGreaterThanOrEqual(120); + }); + + it("ends the request inside Docker's timeout, after the database probe's", () => { + const check = read(); + const wget = wgetTimeout(check.command); + expect(wget).toBeGreaterThan(probeTimeoutSeconds); + expect(wget).toBeLessThan(check.timeout); + }); +}); + +it("compose and the image agree", () => { + const a = composeCheck(); + const b = imageCheck(); + expect([a.interval, a.timeout, a.retries]).toEqual([ + b.interval, + b.timeout, + b.retries, + ]); +});