fix(docker): a boot storm cannot flip the healthcheck to unhealthy

/api/health gives the database two seconds and answers 503 past that.
With three retries, a pool saturated for a minute after the start period
could mark the container unhealthy and take it out of the proxy's
routing. The compose healthcheck and the image's HEALTHCHECK now allow
five misses (two and a half minutes) and end wget after four seconds,
inside Docker's five, so a stalled request is one clean miss. A test
holds both to these bounds and to each other.

Dockerfile

@@ -320,8 +320,11 @@ ENV HOSTNAME="0.0.0.0"
 
 # Start period: the entrypoint migrates before the server listens, and a
 # migration waiting out a held lock retries for up to about eight minutes.
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=3 \
-  CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1
+# Five retries: a pool saturated for a minute during a boot storm makes the
+# probe's 2 s database check miss once or twice without marking the container
+# unhealthy (docker-compose.yml explains the numbers).
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=5 \
+  CMD wget --no-verbose --tries=1 --timeout=4 --spider http://127.0.0.1:3000/api/health || exit 1
 
 ENTRYPOINT ["./docker-entrypoint.sh"]
 CMD ["node", "server.js"]

docker-compose.yml

@@ -512,11 +512,19 @@ services:
       test:
         [
           "CMD-SHELL",
-          "wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1",
+          "wget --no-verbose --tries=1 --timeout=4 --spider http://127.0.0.1:3000/api/health || exit 1",
         ]
       interval: 30s
       timeout: 5s
-      retries: 3
+      # Five misses in a row, two and a half minutes, before "unhealthy". The
+      # probe gives the database two seconds to answer SELECT 1 and says 503
+      # past that, so a boot storm that saturates the pool for a minute after
+      # the start period (backfills, a restore, another tenant's import) costs
+      # a probe or two, not the container's health and the proxy's routing.
+      # wget's own 4 s timeout ends the request inside Docker's 5 s, so a
+      # stalled connection reads as one clean miss rather than a killed probe.
+      # A database that stays away for 2.5 minutes is a real outage and shows.
+      retries: 5
       # The entrypoint applies migrations before the server listens. A
       # migration that waits out a held lock retries for up to about eight
       # minutes (docs/ops/deploy.md), and failed probes in that time must not

docs/self-hosting/truenas.md

@@ -13,7 +13,7 @@ one value you must keep yourself: the encryption key.
 | Port         | `3000` (HTTP)                                                                            |
 | User         | runs as uid `1001`; no extra capabilities, `no-new-privileges` is fine                   |
 | Health check | `GET /api/health`, `200` when ready                                                      |
-| Start period | at least 60 s, with more retries than the image's 3 × 30 s: migrations run at every boot |
+| Start period | at least 60 s, with more retries than the image's 5 × 30 s: migrations run at every boot |
 | Database     | PostgreSQL 16 (17 and later are not a drop-in), no extensions                            |
 | Volumes      | none for the app; the database gets its own dataset                                      |
 

src/__tests__/healthcheck-boot-storm.test.ts

@@ -0,0 +1,113 @@
+/**
+ * The container healthcheck cannot flap during a boot storm.
+ *
+ * `/api/health` gives the database two seconds to answer `SELECT 1` and says
+ * 503 past that. After the start period a saturated pool (backfills, a
+ * restore, a large import) can make one or two probes miss; the healthcheck
+ * has to absorb that and still report an outage that lasts. The compose file
+ * and the image's own HEALTHCHECK must agree, since the published image runs
+ * outside compose too.
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+import { parse } from "yaml";
+
+const ROOT = process.cwd();
+
+function seconds(value: string): number {
+  const m = /^(\d+)(ms|s|m)$/.exec(value.trim());
+  if (!m) throw new Error(`unreadable duration ${value}`);
+  const n = Number(m[1]);
+  return m[2] === "ms" ? n / 1000 : m[2] === "m" ? n * 60 : n;
+}
+
+const probeTimeoutSeconds = (() => {
+  const src = readFileSync(join(ROOT, "src/app/api/health/route.ts"), "utf8");
+  const m = /DB_PROBE_TIMEOUT_MS = ([\d_]+);/.exec(src);
+  if (!m) throw new Error("DB_PROBE_TIMEOUT_MS not found");
+  return Number(m[1]!.replace(/_/g, "")) / 1000;
+})();
+
+interface Check {
+  interval: number;
+  timeout: number;
+  retries: number;
+  command: string;
+}
+
+function composeCheck(): Check {
+  const compose = parse(
+    readFileSync(join(ROOT, "docker-compose.yml"), "utf8"),
+  ) as {
+    services: {
+      app: {
+        healthcheck: {
+          test: string[];
+          interval: string;
+          timeout: string;
+          retries: number;
+        };
+      };
+    };
+  };
+  const hc = compose.services.app.healthcheck;
+  return {
+    interval: seconds(hc.interval),
+    timeout: seconds(hc.timeout),
+    retries: hc.retries,
+    command: hc.test.join(" "),
+  };
+}
+
+function imageCheck(): Check {
+  const dockerfile = readFileSync(join(ROOT, "Dockerfile"), "utf8");
+  const m = /HEALTHCHECK ([^\n]*\\\n[^\n]*)/.exec(dockerfile);
+  if (!m) throw new Error("no HEALTHCHECK in the Dockerfile");
+  const line = m[1]!;
+  const flag = (name: string) => {
+    const f = new RegExp(`--${name}=(\\S+)`).exec(line);
+    if (!f) throw new Error(`HEALTHCHECK has no --${name}`);
+    return f[1]!;
+  };
+  return {
+    interval: seconds(flag("interval")),
+    timeout: seconds(flag("timeout")),
+    retries: Number(flag("retries")),
+    command: line.slice(line.indexOf("CMD")),
+  };
+}
+
+function wgetTimeout(command: string): number {
+  const m = /wget[^|]*--timeout=(\d+)/.exec(command);
+  if (!m) throw new Error(`no wget timeout in: ${command}`);
+  return Number(m[1]);
+}
+
+describe.each([
+  ["docker-compose.yml", composeCheck],
+  ["Dockerfile", imageCheck],
+])("%s healthcheck", (_name, read) => {
+  it("rides out two minutes of slow probes before calling the app unhealthy", () => {
+    const check = read();
+    expect(check.interval * check.retries).toBeGreaterThanOrEqual(120);
+  });
+
+  it("ends the request inside Docker's timeout, after the database probe's", () => {
+    const check = read();
+    const wget = wgetTimeout(check.command);
+    expect(wget).toBeGreaterThan(probeTimeoutSeconds);
+    expect(wget).toBeLessThan(check.timeout);
+  });
+});
+
+it("compose and the image agree", () => {
+  const a = composeCheck();
+  const b = imageCheck();
+  expect([a.interval, a.timeout, a.retries]).toEqual([
+    b.interval,
+    b.timeout,
+    b.retries,
+  ]);
+});