diff --git a/docs/ops/env-check.md b/docs/ops/env-check.md new file mode 100644 index 000000000..3dd76f928 --- /dev/null +++ b/docs/ops/env-check.md @@ -0,0 +1,99 @@ +# Pre-deploy env-var sanity check + +`pnpm check-env` reads `scripts/env-manifest.json` and verifies every +variable declared `required: true` is present + non-empty in the +runtime environment. Optional groups are surfaced too, so an operator +can spot the "3 out of 4 vars set" pattern that silently disables a +feature. + +This is the answer to the v1.4.40 AP-2 silent-disable: the `.p8` file +went missing during a Coolify migration, three of four `APNS_*` vars +remained set, the app booted cleanly, and APNs notifications stopped +working — undetected for three days because the dispatcher's fallback +chain hid the gap behind the Telegram → ntfy → Web Push path. + +## Usage + +### Against the live shell + +```sh +pnpm check-env +``` + +Runs against `process.env` (after the local `.env` / `.env.local` load, +if any). Exit code 0 on green, 1 on at least one required variable +missing, 2 on a malformed manifest. + +### Against a Coolify export / .env.production + +```sh +pnpm check-env --file /path/to/.env.production +``` + +The file is parsed locally (no shell execution, no variable expansion) +and checked against the manifest. Useful for spot-checking a Coolify +export without polluting the local shell. + +## Output format + +Output is one row per declared variable, grouped by section. Three +states: + +| Marker | Meaning | +| --------------------- | ---------------------------------------------------------------------------------- | +| `[OK]` | Variable is set and non-empty. | +| `[MISSING-REQUIRED]` | Variable is required for boot. The script exits 1. | +| `[missing-optional]` | Variable is optional. The script reports it but does not fail. | + +The grep-friendly format lets a CI pipeline do `grep -q +'\[MISSING-REQUIRED\]'` to gate a deploy. The CI integration itself +is deferred to v1.4.43 — this release ships the CLI tool only. + +## All-or-none groups + +Some groups (e.g. off-host backups) are marked `allOrNone: true` in +the manifest. When such a group is partially populated — some vars +set, some missing — the script emits a synthetic `` row +with severity REQUIRED, even if the surrounding group is `required: +false`. This is the v1.4.40 AP-2 detection pattern. + +Example output for a partial APNs config: + +``` +# APNs (iOS push) + [OK] APNS_KEY_ID + [OK] APNS_TEAM_ID + [OK] APNS_BUNDLE_ID + [missing-optional] APNS_KEY — Satisfied by any of: APNS_KEY, APNS_KEY_FILE +``` + +A reviewer immediately sees that `APNS_KEY` (or its `APNS_KEY_FILE` +alternative) is missing while the other three APNs vars are set — the +exact configuration that silently disables iOS push. + +## Editing the manifest + +The manifest lives at `scripts/env-manifest.json` and is git-tracked +so changes go through a PR. To add a new variable: + +1. Pick the right group (or add a new one if the variable doesn't + belong to any existing category). +2. Decide whether the variable is required for boot, optional, or + part of an all-or-none feature group. +3. Document the variable's `purpose` in one sentence — the operator + reads this text when the var shows up as missing. +4. Run `pnpm check-env` locally against your live shell to confirm + the manifest still passes (or fails for the right reason). +5. Add a regression test under `scripts/__tests__/check-env.test.ts` + if the manifest change introduces a new classification branch + (anyOf, allOrNone, …). + +## Future: CI gate + +Deferred to v1.4.43: + +- GitHub Actions workflow that runs `pnpm check-env --file + .env.production.example` on every PR, blocking changes that + introduce new required vars without updating the example file. +- Coolify pre-deploy hook that runs `pnpm check-env` inside the + target container and aborts the deploy on exit code 1. diff --git a/package.json b/package.json index 46c3439f3..a6c51111e 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "e2e": "playwright test", "e2e:ui": "playwright test --ui", "openapi:generate": "npx tsx scripts/generate-openapi.ts", - "openapi:check": "npx tsx scripts/check-openapi.ts" + "openapi:check": "npx tsx scripts/check-openapi.ts", + "check-env": "npx tsx scripts/check-env.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1048.0", diff --git a/scripts/__tests__/check-env.test.ts b/scripts/__tests__/check-env.test.ts new file mode 100644 index 000000000..5d880cb71 --- /dev/null +++ b/scripts/__tests__/check-env.test.ts @@ -0,0 +1,206 @@ +/** + * v1.4.42 W6 — env-check sanity tests. + * + * The script's two pure helpers (`parseEnvFile` + `checkEnv`) are + * pinned here. The renderResults / main() entrypoint is intentionally + * not unit-tested — it's a thin shell over the pure helpers and the + * exit-code behaviour is best left to a smoke test in CI (deferred to + * v1.4.43). + */ +import { describe, expect, it } from "vitest"; +import { checkEnv, parseEnvFile } from "../check-env"; + +describe("parseEnvFile", () => { + it("parses simple KEY=VALUE pairs", () => { + expect(parseEnvFile("FOO=bar\nBAZ=qux")).toEqual({ + FOO: "bar", + BAZ: "qux", + }); + }); + + it("strips surrounding double quotes", () => { + expect(parseEnvFile('FOO="bar baz"')).toEqual({ FOO: "bar baz" }); + }); + + it("strips surrounding single quotes", () => { + expect(parseEnvFile("FOO='bar baz'")).toEqual({ FOO: "bar baz" }); + }); + + it("skips comments and blank lines", () => { + const src = `# header comment +FOO=bar + +# another +BAZ=qux +`; + expect(parseEnvFile(src)).toEqual({ FOO: "bar", BAZ: "qux" }); + }); + + it("preserves an empty value (the v1.4.40 .p8 gap pattern)", () => { + // An empty string MUST round-trip as "" so the presence-check + // downstream classifies it as missing. + expect(parseEnvFile("APNS_KEY=")).toEqual({ APNS_KEY: "" }); + }); + + it("handles CRLF line endings (Coolify exports use \\r\\n)", () => { + expect(parseEnvFile("FOO=bar\r\nBAZ=qux")).toEqual({ + FOO: "bar", + BAZ: "qux", + }); + }); +}); + +const SAMPLE_MANIFEST = { + description: "test manifest", + groups: [ + { + name: "Core", + description: "required boot vars", + required: true, + variables: [ + { name: "DATABASE_URL", purpose: "db" }, + { name: "ENCRYPTION_KEY", purpose: "crypto" }, + ], + }, + { + name: "APNs", + description: "iOS push", + required: false, + variables: [ + { name: "APNS_KEY_ID", purpose: "key id" }, + { name: "APNS_TEAM_ID", purpose: "team id" }, + { + name: "APNS_KEY", + purpose: "key body or file", + anyOf: ["APNS_KEY", "APNS_KEY_FILE"], + }, + ], + }, + { + name: "Backups", + description: "all-or-none", + required: false, + allOrNone: true, + variables: [ + { name: "BACKUP_S3_BUCKET", purpose: "bucket" }, + { name: "BACKUP_S3_ACCESS_KEY", purpose: "key" }, + ], + }, + ], +}; + +describe("checkEnv — required group", () => { + it("flags every missing required var", () => { + const out = checkEnv(SAMPLE_MANIFEST, {}); + const required = out.filter((r) => r.required && !r.present); + expect(required.map((r) => r.variable)).toEqual([ + "DATABASE_URL", + "ENCRYPTION_KEY", + ]); + }); + + it("classifies whitespace-only values as missing (deploy-failure prevention)", () => { + // A `DATABASE_URL=" "` would silently boot with a useless value; + // the deployer expects to see a clear MISSING-REQUIRED row here. + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: " ", + ENCRYPTION_KEY: "ok", + }); + const row = out.find((r) => r.variable === "DATABASE_URL")!; + expect(row.present).toBe(false); + expect(row.required).toBe(true); + }); + + it("greenlights when every required var is non-empty", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "postgresql://x", + ENCRYPTION_KEY: "deadbeef".repeat(8), + }); + const required = out.filter((r) => r.required); + expect(required.every((r) => r.present)).toBe(true); + }); +}); + +describe("checkEnv — anyOf alternatives", () => { + it("treats APNS_KEY as satisfied when APNS_KEY_FILE is set instead", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + APNS_KEY_ID: "abc", + APNS_TEAM_ID: "def", + APNS_KEY_FILE: "/secrets/apns.p8", + }); + const apnsKey = out.find((r) => r.variable === "APNS_KEY")!; + expect(apnsKey.present).toBe(true); + }); + + it("treats APNS_KEY as satisfied when APNS_KEY is set (the 12-factor variant)", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + APNS_KEY_ID: "abc", + APNS_TEAM_ID: "def", + APNS_KEY: "-----BEGIN PRIVATE KEY-----...", + }); + expect(out.find((r) => r.variable === "APNS_KEY")?.present).toBe(true); + }); + + it("marks APNS_KEY missing when BOTH alternatives are absent (the v1.4.40 AP-2 gap)", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + APNS_KEY_ID: "abc", + APNS_TEAM_ID: "def", + }); + expect(out.find((r) => r.variable === "APNS_KEY")?.present).toBe(false); + }); +}); + +describe("checkEnv — all-or-none groups", () => { + it("emits a synthetic required row when a group is half-populated", () => { + // The v1.4.40 AP-2 silent-disable pattern: 3 of 4 APNS_* set, .p8 + // missing → app silently falls back to Telegram. The all-or-none + // synthetic row catches this even when the surrounding group is + // `required: false`. + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + BACKUP_S3_BUCKET: "my-bucket", + // BACKUP_S3_ACCESS_KEY deliberately absent + }); + const synth = out.find((r) => r.variable === "")!; + expect(synth).toBeDefined(); + expect(synth.required).toBe(true); + expect(synth.note).toContain("1/2"); + }); + + it("does NOT emit the synthetic row when the group is fully set", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + BACKUP_S3_BUCKET: "my-bucket", + BACKUP_S3_ACCESS_KEY: "key", + }); + expect(out.find((r) => r.variable === "")).toBeUndefined(); + }); + + it("does NOT emit the synthetic row when the group is fully empty", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + }); + expect(out.find((r) => r.variable === "")).toBeUndefined(); + }); +}); + +describe("checkEnv — optional groups (informational)", () => { + it("reports optional vars without setting required: true", () => { + const out = checkEnv(SAMPLE_MANIFEST, { + DATABASE_URL: "x", + ENCRYPTION_KEY: "y", + }); + const apnsKeyId = out.find((r) => r.variable === "APNS_KEY_ID")!; + expect(apnsKeyId.present).toBe(false); + expect(apnsKeyId.required).toBe(false); + }); +}); diff --git a/scripts/check-env.ts b/scripts/check-env.ts new file mode 100644 index 000000000..061d22e13 --- /dev/null +++ b/scripts/check-env.ts @@ -0,0 +1,267 @@ +#!/usr/bin/env tsx +/** + * Pre-deploy env-var sanity check. + * + * Reads `scripts/env-manifest.json` and verifies every variable + * declared `required: true` (group-level) is present + non-empty in + * the runtime environment. Optional groups are reported as + * informational so an operator can spot misconfigurations (e.g. three + * out of four APNS_* vars set — the silent-disable case that bit us + * for 3 days in v1.4.40). + * + * Designed to run in two modes: + * + * 1. Live env (default): inspects `process.env` after dotenv has + * loaded `.env` / `.env.production`. This is what `pnpm + * check-env` does locally and what CI will run against the + * pre-deploy environment. + * + * 2. File mode: `pnpm check-env --file .env.production` reads a + * KEY=VALUE file directly without loading it into `process.env`. + * Useful for checking a Coolify export against the manifest + * without polluting the local shell. + * + * Exit codes: + * 0 — all required variables present + non-empty. + * 1 — at least one required variable missing/empty. + * 2 — manifest file unreadable or malformed. + * + * Future (v1.4.43): CI integration via a GitHub Action that runs this + * against a `.env.production.example`-style file in the repo, + * blocking PRs that introduce new required vars without updating the + * manifest. + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { argv, exit, env as processEnv } from "node:process"; + +const MANIFEST_PATH = resolve(__dirname, "env-manifest.json"); + +interface ManifestVariable { + name: string; + purpose: string; + example?: string; + /** + * Alternative names — if at least one is present, the variable is + * considered satisfied. Used for APNS_KEY vs APNS_KEY_FILE. + */ + anyOf?: string[]; +} + +interface ManifestGroup { + name: string; + description: string; + required: boolean; + allOrNone?: boolean; + variables: ManifestVariable[]; +} + +interface Manifest { + description: string; + groups: ManifestGroup[]; +} + +interface CheckResult { + group: string; + variable: string; + present: boolean; + required: boolean; + note?: string; +} + +/** + * Parse a KEY=VALUE file into an env-lookup object. Mirrors dotenv's + * minimal behaviour (no variable expansion, no escape sequences) so a + * Coolify export file or .env.production can be inspected without + * pulling in the full dotenv parse path. + * + * - Lines beginning with `#` are comments. + * - Empty lines are skipped. + * - Quoted values (`"..."` or `'...'`) have the quotes stripped. + * - Whitespace around `=` is preserved (Coolify never adds any). + */ +export function parseEnvFile(content: string): Record { + const out: Record = {}; + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq === -1) continue; + const key = line.slice(0, eq).trim(); + let value = line.slice(eq + 1).trim(); + // Strip surrounding quotes — dotenv accepts both flavours. + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + out[key] = value; + } + return out; +} + +/** + * A variable counts as "present" only when the value is defined AND + * non-empty (whitespace-only counts as empty). Pre-v1.4.42 the v1.4.40 + * AP-2 gap shipped because the .p8 file simply did not exist — but a + * `""` value would have passed an `existsSync`-style check too. + */ +function isPresent(value: string | undefined): boolean { + return typeof value === "string" && value.trim().length > 0; +} + +/** + * Pure classifier — exported so unit tests can drive it without + * hitting the filesystem. Given a manifest and an env lookup, return + * one result row per declared variable plus any group-level + * all-or-none violations. + */ +export function checkEnv( + manifest: Manifest, + env: Record, +): CheckResult[] { + const results: CheckResult[] = []; + + for (const group of manifest.groups) { + let presentInGroup = 0; + let absentInGroup = 0; + + for (const v of group.variables) { + const names = v.anyOf ?? [v.name]; + const present = names.some((n) => isPresent(env[n])); + const note = v.anyOf + ? `Satisfied by any of: ${v.anyOf.join(", ")}` + : undefined; + results.push({ + group: group.name, + variable: v.name, + present, + required: group.required, + note, + }); + if (present) presentInGroup++; + else absentInGroup++; + } + + // All-or-none groups: surface a synthetic row when the group is + // partially populated. This catches the v1.4.40 AP-2 pattern even + // when the group itself is `required: false`. + if ( + group.allOrNone && + presentInGroup > 0 && + absentInGroup > 0 + ) { + results.push({ + group: group.name, + variable: "", + present: false, + // Promote to required severity — a partial all-or-none group + // is always wrong, no matter the surrounding group flag. + required: true, + note: `Group is all-or-none but only ${presentInGroup}/${ + presentInGroup + absentInGroup + } variables are set. Either set all or unset all.`, + }); + } + } + + return results; +} + +/** + * Render a one-row-per-variable summary to stdout. The output is + * intentionally grep-friendly so a CI pipeline can pattern-match for + * `[MISSING-REQUIRED]` to fail the build. + */ +function renderResults(results: CheckResult[]): { + missingRequired: number; + missingOptional: number; +} { + let missingRequired = 0; + let missingOptional = 0; + let currentGroup = ""; + + for (const r of results) { + if (r.group !== currentGroup) { + console.log(`\n# ${r.group}`); + currentGroup = r.group; + } + if (r.present) { + console.log(` [OK] ${r.variable}`); + } else if (r.required) { + console.log( + ` [MISSING-REQUIRED] ${r.variable}` + + (r.note ? ` — ${r.note}` : ""), + ); + missingRequired++; + } else { + console.log( + ` [missing-optional] ${r.variable}` + (r.note ? ` — ${r.note}` : ""), + ); + missingOptional++; + } + } + + return { missingRequired, missingOptional }; +} + +/** + * Entry point — kept as a named export so a test harness can call it + * with a synthetic argv without spawning a child process. + */ +export function main(args: string[]): number { + const fileFlagIdx = args.indexOf("--file"); + let env: Record; + let envLabel: string; + + if (fileFlagIdx !== -1 && args[fileFlagIdx + 1]) { + const filePath = resolve(args[fileFlagIdx + 1]!); + try { + env = parseEnvFile(readFileSync(filePath, "utf8")); + envLabel = filePath; + } catch (err) { + console.error(`Failed to read env file at ${filePath}: ${err}`); + return 2; + } + } else { + env = processEnv; + envLabel = "process.env"; + } + + let manifest: Manifest; + try { + manifest = JSON.parse(readFileSync(MANIFEST_PATH, "utf8")) as Manifest; + } catch (err) { + console.error(`Failed to read manifest at ${MANIFEST_PATH}: ${err}`); + return 2; + } + + console.log(`Checking ${envLabel} against ${MANIFEST_PATH}`); + const results = checkEnv(manifest, env); + const { missingRequired, missingOptional } = renderResults(results); + + console.log(""); + console.log( + `Summary: ${results.filter((r) => r.present).length}/${results.length} variables present`, + ); + if (missingRequired > 0) { + console.log( + ` ${missingRequired} REQUIRED variable(s) missing — deploy WILL FAIL.`, + ); + } + if (missingOptional > 0) { + console.log( + ` ${missingOptional} optional variable(s) missing — feature will silently disable.`, + ); + } + + return missingRequired > 0 ? 1 : 0; +} + +// Only run when invoked directly (`pnpm check-env` / `tsx scripts/check-env.ts`). +// `require.main === module` survives the bundler-style ESM->CJS shim +// that tsx uses. +if (require.main === module) { + exit(main(argv.slice(2))); +} diff --git a/scripts/env-manifest.json b/scripts/env-manifest.json new file mode 100644 index 000000000..def38748d --- /dev/null +++ b/scripts/env-manifest.json @@ -0,0 +1,106 @@ +{ + "$schema": "./env-manifest.schema.json", + "description": "Manifest of env vars required by a HealthLog production deployment. Used by `pnpm check-env` to catch missing/empty vars BEFORE a deploy ships. v1.4.42 W6 — born out of the v1.4.40 AP-2 .p8 install gap that shipped silently for 3 days.", + "groups": [ + { + "name": "Core", + "description": "Database, app URL, encryption keys. The app refuses to boot without these.", + "required": true, + "variables": [ + { + "name": "DATABASE_URL", + "purpose": "Postgres connection string. App boot fails if missing.", + "example": "postgresql://healthlog:@db:5432/healthlog?schema=public" + }, + { + "name": "POSTGRES_PASSWORD", + "purpose": "Database password for the bundled `db` service in docker-compose.yml.", + "example": "" + }, + { + "name": "NEXT_PUBLIC_APP_URL", + "purpose": "Public URL embedded in OAuth redirects, webhook callbacks, APNs deep-links." + }, + { + "name": "APP_URL", + "purpose": "Server-side mirror of NEXT_PUBLIC_APP_URL; used by background workers." + }, + { + "name": "ENCRYPTION_KEY", + "purpose": "AES-256-GCM key for at-rest encryption of provider tokens. 64 hex chars (32 bytes). Required unless ENCRYPTION_KEYS is set." + }, + { + "name": "API_TOKEN_HMAC_KEY", + "purpose": "HMAC-SHA256 key for hashing external Bearer tokens. 64 hex chars." + } + ] + }, + { + "name": "Withings OAuth", + "description": "Required to enable the Withings integration. App boots without these but Withings sync stays disabled.", + "required": false, + "variables": [ + { + "name": "WITHINGS_CLIENT_ID", + "purpose": "Withings OAuth client_id from https://developer.withings.com." + }, + { + "name": "WITHINGS_CLIENT_SECRET", + "purpose": "Withings OAuth client_secret." + }, + { + "name": "WITHINGS_WEBHOOK_SECRET", + "purpose": "Random secret carried as a path segment in the Withings webhook callback URL. Generate with `openssl rand -hex 32`." + } + ] + }, + { + "name": "APNs (iOS push)", + "description": "All four core APNS_* vars together. Missing any disables APNs silently — the v1.4.40 AP-2 gap that motivated this manifest.", + "required": false, + "variables": [ + { + "name": "APNS_KEY_ID", + "purpose": "10-char Key ID from Apple Developer." + }, + { + "name": "APNS_TEAM_ID", + "purpose": "10-char Team ID from Apple Developer." + }, + { + "name": "APNS_BUNDLE_ID", + "purpose": "iOS app bundle id, e.g. dev.healthlog.ios." + }, + { + "name": "APNS_KEY", + "purpose": "APNs auth key (.p8 contents). Use APNS_KEY for 12-factor env-block deploys; APNS_KEY_FILE is the filesystem alternative — at least one of the two must be set.", + "anyOf": ["APNS_KEY", "APNS_KEY_FILE"] + } + ] + }, + { + "name": "Deploy webhook", + "description": "Coolify -> HealthLog deploy-result webhook. Without this, the auto-deploy success/failure feedback loop is broken.", + "required": false, + "variables": [ + { + "name": "DEPLOY_WEBHOOK_SECRET", + "purpose": "Shared secret for the X-Deploy-Webhook-Secret header on POST /api/internal/deploy-webhook. 64 hex chars." + } + ] + }, + { + "name": "Off-host backups", + "description": "Required ALL-OR-NONE: any of these without the rest disables backups silently.", + "required": false, + "allOrNone": true, + "variables": [ + { "name": "BACKUP_S3_ENDPOINT", "purpose": "S3-compatible endpoint URL." }, + { "name": "BACKUP_S3_BUCKET", "purpose": "Bucket name." }, + { "name": "BACKUP_S3_ACCESS_KEY", "purpose": "S3 access key." }, + { "name": "BACKUP_S3_SECRET_KEY", "purpose": "S3 secret key." }, + { "name": "BACKUP_ENCRYPTION_KEY", "purpose": "Separate AES key for the backup body — MUST NOT match ENCRYPTION_KEY." } + ] + } + ] +}