diff --git a/Dockerfile b/Dockerfile index 625feff6d..6f58628aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -318,7 +318,9 @@ EXPOSE 3000 ENV PORT=3000 ENV HOSTNAME="0.0.0.0" -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ +# Start period: the entrypoint migrates before the server listens, and a +# migration waiting out a held lock retries for up to about eight minutes. +HEALTHCHECK --interval=30s --timeout=5s --start-period=10m --retries=3 \ CMD wget --no-verbose --tries=1 --spider http://127.0.0.1:3000/api/health || exit 1 ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index 72a75c0c3..fb778bee4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -516,7 +516,13 @@ services: interval: 30s timeout: 5s retries: 3 - start_period: 30s + # The entrypoint applies migrations before the server listens. A + # migration that waits out a held lock retries for up to about eight + # minutes (docs/ops/deploy.md), and failed probes in that time must not + # read as a dead container. The first probe that answers still ends + # the start period, so a normal boot is healthy within 30 seconds of + # listening. + start_period: 10m # Optional dedicated worker container — uncomment + set # HEALTHLOG_PROCESS_TYPE=web on `app` to split the deployment. diff --git a/docs/ops/deploy.md b/docs/ops/deploy.md index 36e78047e..d4a828b9e 100644 --- a/docs/ops/deploy.md +++ b/docs/ops/deploy.md @@ -79,6 +79,12 @@ does the same. Nothing else is retried: any other failure, and a lock timeout in a migration that does not carry the marker, stops the boot exactly as before. +The retries fit inside the container's health check: its start period is ten +minutes (`docker-compose.yml` and the image's own `HEALTHCHECK`), so failed +probes while the entrypoint is still migrating do not mark the container +unhealthy. The first probe that answers ends the start period, so a normal +boot is reported healthy within half a minute of the server listening. + If all five retries time out, the log ends with `HealthLog: ERROR — migration … timed out on a lock 6 times in a row; giving up.` and the container exits. Something has held the table for about eight