diff --git a/Dockerfile b/Dockerfile index 24982844a..37f80524a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -179,8 +179,16 @@ RUN mkdir -p /opt/prisma-cli && \ # # Attribution (CC BY-SA 4.0): see `docs/audit/v1427-summary.md` and # `/about` in the running app. -RUN mkdir -p /opt/geolite2 -COPY assets/geolite2/ /opt/geolite2/ +# +# v1.37.7 (issue #659) — the directory is owned by the unprivileged `nextjs` +# user so the runtime worker can WRITE it: a self-hoster who sets +# MAXMIND_LICENSE_KEY on the published image has the databases fetched into +# this directory at runtime (src/lib/geo/geolite2-fetch.ts) with no rebuild or +# manual mount. `tar`/`gzip` for the extraction ship in the busybox base. A +# read-only bind mount at this path (the bring-your-own-database route) still +# works — the runtime fetch just fails soft and the mounted files stand. +RUN mkdir -p /opt/geolite2 && chown nextjs:nodejs /opt/geolite2 +COPY --chown=nextjs:nodejs assets/geolite2/ /opt/geolite2/ # ISO-8601 build timestamp — /api/version returns it as `builtAt`. # Declared this late in the stage on purpose: the value differs on diff --git a/docker-compose.yml b/docker-compose.yml index 345c19d88..7add1e61d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,6 +88,16 @@ x-healthlog-env: &healthlog_env # value in .env reaches the container (the `environment:` block is a # whitelist). See docs/self-hosting/geolite2.md. GEOLITE2_DIR: "${GEOLITE2_DIR:-/opt/geolite2}" + # MaxMind licence key for the RUNTIME GeoLite2 fetch (issue #659). Set this + # and the worker downloads GeoLite2-City + GeoLite2-ASN itself into + # GEOLITE2_DIR on the next boot and monthly thereafter — no image rebuild, + # no manual mount. Get a free key at + # https://www.maxmind.com/en/geolite2/signup (My Account -> Manage License + # Keys). Leave it unset and nothing is fetched (mounting your own databases + # or IP_GEO_LOOKUP_DISABLED=1 both still work). Listed here so an operator + # value in .env reaches the container (the `environment:` block is a + # whitelist). See docs/self-hosting/geolite2.md. + MAXMIND_LICENSE_KEY: "${MAXMIND_LICENSE_KEY:-}" # all = bundle web + worker in this container (default, single-host) # web = serve HTTP only; pair with a separate app-worker service HEALTHLOG_PROCESS_TYPE: "${HEALTHLOG_PROCESS_TYPE:-all}" diff --git a/src/lib/geo.ts b/src/lib/geo.ts index 7813c32c9..3efa3cd5a 100644 --- a/src/lib/geo.ts +++ b/src/lib/geo.ts @@ -169,7 +169,7 @@ export function resolveGeoProviderHost(): string { * as unset, otherwise a stack that merely lists the variable would read * the process working directory and silently lose the offline tier. */ -function geoLiteDir(): string { +export function geoLiteDir(): string { const configured = process.env.GEOLITE2_DIR?.trim(); return configured ? configured : "/opt/geolite2"; } @@ -252,15 +252,28 @@ function getAsnReader(): MmdbReader | null { } /** - * Test-only — reset the lazy reader cache so a test can swap the - * `GEOLITE2_DIR` between cases without leaking the previous Reader. - * Not exported from the public surface; the tests import via the - * module-level alias. + * Reset the lazy MMDB reader cache so the next lookup re-reads the files + * from `geoLiteDir()`. The runtime GeoLite2 fetch (`src/lib/geo/geolite2-fetch.ts`) + * calls this after it places freshly downloaded databases, so a running worker + * picks them up without a restart — the readers latch `null` on the first miss, + * so without this a process that booted before the files landed would keep + * resolving online forever. Also clears the one-shot "offline unavailable" + * admin-notification latch, since the offline tier may now be present. */ -export function __resetGeoLite2CacheForTests(): void { +export function resetGeoLite2ReaderCache(): void { cache.city = undefined; cache.asn = undefined; notifiedThisProcess = false; +} + +/** + * Test-only — reset the lazy reader cache so a test can swap the + * `GEOLITE2_DIR` between cases without leaking the previous Reader. Extends + * the runtime reset with the per-IP location cache, which production has no + * reason to drop. + */ +export function __resetGeoLite2CacheForTests(): void { + resetGeoLite2ReaderCache(); GEO_CACHE.clear(); } diff --git a/src/lib/geo/__tests__/geolite2-fetch.test.ts b/src/lib/geo/__tests__/geolite2-fetch.test.ts new file mode 100644 index 000000000..c3958f911 --- /dev/null +++ b/src/lib/geo/__tests__/geolite2-fetch.test.ts @@ -0,0 +1,264 @@ +/** + * Runtime GeoLite2 fetch (issue #659) — unit coverage. + * + * The network is mocked (safeFetch), but the filesystem + `tar` extraction run + * for real against a tar.gz fixture built the same way MaxMind ships one (the + * mmdb nested under a date-stamped directory). That exercises the actual + * temp-then-rename place path, not a stub of it. + */ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { SafeFetchError } from "@/lib/safe-fetch"; + +const { safeFetchMock } = vi.hoisted(() => ({ safeFetchMock: vi.fn() })); +vi.mock("@/lib/safe-fetch", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, safeFetch: safeFetchMock }; +}); + +const { resetCacheSpy } = vi.hoisted(() => ({ resetCacheSpy: vi.fn() })); +vi.mock("@/lib/geo", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, resetGeoLite2ReaderCache: resetCacheSpy }; +}); + +import { + buildMaxmindDownloadUrl, + fetchGeoLite2Databases, + MAXMIND_DOWNLOAD_BASE, +} from "@/lib/geo/geolite2-fetch"; + +const CITY_BYTES = Buffer.from("CITY-MMDB-CONTENT-"); +const ASN_BYTES = Buffer.from("ASN-MMDB-CONTENT-"); + +/** Build a MaxMind-shaped tar.gz: `_/.mmdb`. */ +function buildEditionTarball( + editionId: string, + mmdbBytes: Buffer, +): ArrayBuffer { + const stage = fs.mkdtempSync(path.join(os.tmpdir(), "geolite2-fixture-")); + try { + const inner = `${editionId}_20260808`; + fs.mkdirSync(path.join(stage, inner)); + fs.writeFileSync(path.join(stage, inner, `${editionId}.mmdb`), mmdbBytes); + const tarball = path.join(stage, `${editionId}.tar.gz`); + execFileSync("tar", ["-czf", tarball, "-C", stage, inner]); + const buf = fs.readFileSync(tarball); + return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + } finally { + fs.rmSync(stage, { recursive: true, force: true }); + } +} + +function okResponse(body: ArrayBuffer): Response { + return { + ok: true, + status: 200, + arrayBuffer: async () => body, + } as unknown as Response; +} + +let geoDir: string; +const ORIGINAL_KEY = process.env.MAXMIND_LICENSE_KEY; +const ORIGINAL_DISABLED = process.env.IP_GEO_LOOKUP_DISABLED; +const ORIGINAL_DIR = process.env.GEOLITE2_DIR; + +beforeEach(() => { + safeFetchMock.mockReset(); + resetCacheSpy.mockReset(); + geoDir = fs.mkdtempSync(path.join(os.tmpdir(), "geolite2-target-")); + process.env.GEOLITE2_DIR = geoDir; + process.env.MAXMIND_LICENSE_KEY = "test-licence-key"; + delete process.env.IP_GEO_LOOKUP_DISABLED; +}); + +afterEach(() => { + fs.rmSync(geoDir, { recursive: true, force: true }); + if (ORIGINAL_KEY === undefined) delete process.env.MAXMIND_LICENSE_KEY; + else process.env.MAXMIND_LICENSE_KEY = ORIGINAL_KEY; + if (ORIGINAL_DISABLED === undefined) + delete process.env.IP_GEO_LOOKUP_DISABLED; + else process.env.IP_GEO_LOOKUP_DISABLED = ORIGINAL_DISABLED; + if (ORIGINAL_DIR === undefined) delete process.env.GEOLITE2_DIR; + else process.env.GEOLITE2_DIR = ORIGINAL_DIR; +}); + +/** Route each edition's mocked download to its fixture tarball. */ +function wireTarballDownloads(): void { + const city = buildEditionTarball("GeoLite2-City", CITY_BYTES); + const asn = buildEditionTarball("GeoLite2-ASN", ASN_BYTES); + safeFetchMock.mockImplementation(async (url: string) => { + if (url.includes("GeoLite2-City")) return okResponse(city); + if (url.includes("GeoLite2-ASN")) return okResponse(asn); + throw new Error(`unexpected url ${url}`); + }); +} + +describe("buildMaxmindDownloadUrl", () => { + it("builds the permalink URL per edition with the key + tar.gz suffix", () => { + const url = buildMaxmindDownloadUrl("GeoLite2-City", "abc123"); + expect(url.startsWith(`${MAXMIND_DOWNLOAD_BASE}?`)).toBe(true); + const params = new URL(url).searchParams; + expect(params.get("edition_id")).toBe("GeoLite2-City"); + expect(params.get("license_key")).toBe("abc123"); + expect(params.get("suffix")).toBe("tar.gz"); + }); +}); + +describe("fetchGeoLite2Databases — success", () => { + it("downloads both editions and places the mmdbs atomically", async () => { + wireTarballDownloads(); + fs.writeFileSync(path.join(geoDir, ".empty"), ""); // stale keyless marker + + const result = await fetchGeoLite2Databases(); + + expect(result.status).toBe("fetched"); + expect(result.editions).toEqual(["GeoLite2-City", "GeoLite2-ASN"]); + + // Both databases landed with the exact fixture bytes. + expect(fs.readFileSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toEqual( + CITY_BYTES, + ); + expect(fs.readFileSync(path.join(geoDir, "GeoLite2-ASN.mmdb"))).toEqual( + ASN_BYTES, + ); + + // The stale `.empty` marker is gone and no temp/work residue remains. + expect(fs.existsSync(path.join(geoDir, ".empty"))).toBe(false); + const leftovers = fs + .readdirSync(geoDir) + .filter((n) => n.startsWith(".geolite2-") || n.includes(".tmp-")); + expect(leftovers).toEqual([]); + + // The running worker's reader cache is reset so it re-reads the new files. + expect(resetCacheSpy).toHaveBeenCalledTimes(1); + }); + + it("requests each edition with redirect-following + the public-host pin", async () => { + wireTarballDownloads(); + + await fetchGeoLite2Databases(); + + expect(safeFetchMock).toHaveBeenCalledTimes(2); + for (const call of safeFetchMock.mock.calls) { + const [url, , opts] = call; + expect(url).toContain("license_key=test-licence-key"); + expect(url).toContain("suffix=tar.gz"); + expect(opts).toMatchObject({ + followRedirects: true, + requirePublicHost: true, + }); + expect(typeof opts.timeoutMs).toBe("number"); + } + const urls = safeFetchMock.mock.calls.map((c) => c[0] as string); + expect(urls.some((u) => u.includes("edition_id=GeoLite2-City"))).toBe(true); + expect(urls.some((u) => u.includes("edition_id=GeoLite2-ASN"))).toBe(true); + }); +}); + +describe("fetchGeoLite2Databases — fail-soft", () => { + it("leaves an existing database untouched and writes no partial file on a download error", async () => { + const existing = Buffer.from("EXISTING-GOOD-CITY-DB"); + fs.writeFileSync(path.join(geoDir, "GeoLite2-City.mmdb"), existing); + + safeFetchMock.mockRejectedValue(new SafeFetchError("boom", "network")); + + const result = await fetchGeoLite2Databases(); + + expect(result.status).toBe("failed"); + // The pre-existing database is byte-for-byte intact. + expect(fs.readFileSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toEqual( + existing, + ); + // No partial temp file, no orphaned work directory. + const residue = fs + .readdirSync(geoDir) + .filter((n) => n.startsWith(".geolite2-") || n.includes(".tmp-")); + expect(residue).toEqual([]); + // A failed fetch must NOT reset the reader cache. + expect(resetCacheSpy).not.toHaveBeenCalled(); + }); + + it("promotes nothing when the first edition succeeds but the second fails", async () => { + // The atomicity contract: City downloads + extracts fine, ASN 401s. Since + // not every edition was staged, NOTHING is promoted — no GeoLite2-City.mmdb + // appears — and the City temp file is cleaned up. + const city = buildEditionTarball("GeoLite2-City", CITY_BYTES); + safeFetchMock.mockImplementation(async (url: string) => { + if (url.includes("GeoLite2-City")) return okResponse(city); + return { + ok: false, + status: 401, + arrayBuffer: async () => new ArrayBuffer(0), + } as unknown as Response; + }); + + const result = await fetchGeoLite2Databases(); + + expect(result.status).toBe("failed"); + // City must NOT have been promoted — the place is all-or-nothing. + expect(fs.existsSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toBe(false); + // And its staged temp file must be gone. + const residue = fs + .readdirSync(geoDir) + .filter((n) => n.startsWith(".geolite2-") || n.includes(".tmp-")); + expect(residue).toEqual([]); + expect(resetCacheSpy).not.toHaveBeenCalled(); + }); + + it("fails soft when a non-ok HTTP status comes back", async () => { + safeFetchMock.mockResolvedValue({ + ok: false, + status: 401, + arrayBuffer: async () => new ArrayBuffer(0), + } as unknown as Response); + + const result = await fetchGeoLite2Databases(); + + expect(result.status).toBe("failed"); + expect(result.error).toContain("401"); + expect(fs.existsSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toBe(false); + }); +}); + +describe("fetchGeoLite2Databases — gating", () => { + it("skips when IP_GEO_LOOKUP_DISABLED=1", async () => { + process.env.IP_GEO_LOOKUP_DISABLED = "1"; + const result = await fetchGeoLite2Databases(); + expect(result.status).toBe("skipped_disabled"); + expect(safeFetchMock).not.toHaveBeenCalled(); + }); + + it("skips when no MAXMIND_LICENSE_KEY is set", async () => { + delete process.env.MAXMIND_LICENSE_KEY; + const result = await fetchGeoLite2Databases(); + expect(result.status).toBe("skipped_no_key"); + expect(safeFetchMock).not.toHaveBeenCalled(); + }); + + it("skips a boot-mode fetch when the offline tier is already present", async () => { + fs.writeFileSync( + path.join(geoDir, "GeoLite2-City.mmdb"), + Buffer.from("already-here"), + ); + const result = await fetchGeoLite2Databases({ skipIfPresent: true }); + expect(result.status).toBe("skipped_present"); + expect(safeFetchMock).not.toHaveBeenCalled(); + }); + + it("fetches anyway (refresh) when present but skipIfPresent is not set", async () => { + wireTarballDownloads(); + fs.writeFileSync( + path.join(geoDir, "GeoLite2-City.mmdb"), + Buffer.from("old-city"), + ); + const result = await fetchGeoLite2Databases(); + expect(result.status).toBe("fetched"); + expect(fs.readFileSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toEqual( + CITY_BYTES, + ); + }); +}); diff --git a/src/lib/geo/geolite2-fetch.ts b/src/lib/geo/geolite2-fetch.ts new file mode 100644 index 000000000..25bd4f2f8 --- /dev/null +++ b/src/lib/geo/geolite2-fetch.ts @@ -0,0 +1,287 @@ +/** + * Runtime GeoLite2 database fetch (issue #659). + * + * A self-hoster running the published image cannot set a build secret, so the + * two pre-existing ways to get the offline MMDBs — bake them at build time via + * `scripts/fetch-geolite2.sh`, or mount your own and point `GEOLITE2_DIR` at + * them — both need something the operator of a stock `:latest` pull does not + * have. This is the third way: given `MAXMIND_LICENSE_KEY` at runtime, the + * worker downloads GeoLite2-City + GeoLite2-ASN itself, extracts each, and + * places them in `geoLiteDir()`. No rebuild, no manual mount. + * + * Fail-soft is the whole contract. Every download extracts to a temp file in + * the SAME directory as the target, and a completed edition is promoted with an + * atomic `rename` only after ALL editions are in hand — so a mid-fetch error + * (bad key, MaxMind 5xx, a truncated body) leaves the previous databases exactly + * as they were and never writes a partial `.mmdb`. The offline tier is optional; + * a failure just means the online provider keeps answering, which is the + * keyless default anyway. + * + * The MaxMind permalink (`download.maxmind.com/app/geoip_download`) 302-redirects + * to a signed CDN URL, so the fetch opts into `followRedirects: true` on top of + * the `requirePublicHost: true` DNS-rebinding pin. + * + * Extraction is pure Node: `zlib.gunzipSync` + a minimal in-process tar reader + * (`extractMmdbFromTarGz`), NOT a shell-out to `tar`. The runtime image does + * carry busybox `tar`+`gzip`, but this module is reachable from the worker-boot + * import chain (`instrumentation.ts` → `reminder-worker` → this file), and a + * `child_process` import there makes Next's file tracer give up and trace the + * whole project into the standalone output. `node:zlib` keeps the trace bounded, + * removes the busybox runtime dependency entirely, and is trivially unit-testable + * — the MaxMind tarball nests a single `.mmdb` under one date-stamped directory, + * a shape a ~40-line reader handles without a dependency. + */ +import zlib from "node:zlib"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { getEvent } from "@/lib/logging/context"; +import { safeFetch } from "@/lib/safe-fetch"; +import { + geoLiteDir, + offlineGeoReady, + resetGeoLite2ReaderCache, +} from "@/lib/geo"; + +/** + * The MaxMind permalink download endpoint. Same URL shape the build-time + * `scripts/fetch-geolite2.sh` uses: `?edition_id=…&license_key=…&suffix=tar.gz`. + */ +export const MAXMIND_DOWNLOAD_BASE = + "https://download.maxmind.com/app/geoip_download"; + +export interface Geolite2Edition { + /** MaxMind edition id, e.g. `GeoLite2-City`. */ + editionId: string; + /** The `.mmdb` basename the resolver in `geo.ts` reads. */ + mmdb: string; +} + +/** + * The two editions the resolver reads: City for location, ASN for the + * authoritative carrier + AS number. Both must land for the fetch to count as + * a success, matching the build-time script's all-or-nothing contract. + */ +export const GEOLITE2_EDITIONS: readonly Geolite2Edition[] = [ + { editionId: "GeoLite2-City", mmdb: "GeoLite2-City.mmdb" }, + { editionId: "GeoLite2-ASN", mmdb: "GeoLite2-ASN.mmdb" }, +]; + +// A GeoLite2 tarball is ~40-60 MB; give the download well clear of the 15 s +// safeFetch default so a slow self-host link does not abort a good fetch. The +// signal covers body streaming too (undici aborts the read on the same signal). +const DOWNLOAD_TIMEOUT_MS = 120_000; + +export type Geolite2FetchStatus = + | "skipped_disabled" + | "skipped_no_key" + | "skipped_present" + | "fetched" + | "failed"; + +export interface Geolite2FetchResult { + status: Geolite2FetchStatus; + /** Edition ids that were placed on a `"fetched"` result; empty otherwise. */ + editions: string[]; + /** Human-readable cause on a `"failed"` result. */ + error?: string; +} + +/** + * Build the MaxMind permalink download URL for one edition. Exported so the + * unit test can assert the exact wire URL per edition without a live request. + */ +export function buildMaxmindDownloadUrl( + editionId: string, + licenseKey: string, +): string { + const params = new URLSearchParams({ + edition_id: editionId, + license_key: licenseKey, + suffix: "tar.gz", + }); + return `${MAXMIND_DOWNLOAD_BASE}?${params.toString()}`; +} + +function fail(message: string, cause: unknown): Geolite2FetchResult { + const detail = cause instanceof Error ? cause.message : String(cause); + getEvent()?.addWarning(`geolite2-fetch: ${message}: ${detail}`); + return { status: "failed", editions: [], error: `${message}: ${detail}` }; +} + +const TAR_BLOCK = 512; + +/** Read a NUL-terminated ASCII field from a tar header block. */ +function readTarField(header: Buffer, offset: number, length: number): string { + const raw = header.subarray(offset, offset + length); + const end = raw.indexOf(0); + return raw.toString("ascii", 0, end === -1 ? raw.length : end).trim(); +} + +/** Parse a tar octal size field. GeoLite2 mmdbs fit octal, so no base-256 path. */ +function readTarOctal(header: Buffer, offset: number, length: number): number { + const text = readTarField(header, offset, length); + if (!text) return 0; + const n = parseInt(text, 8); + return Number.isFinite(n) ? n : 0; +} + +/** + * Extract one `.mmdb` from a gzipped tar. The MaxMind tarball nests a single + * database under a date-stamped directory (`GeoLite2-City_YYYYMMDD/…mmdb`), so + * we walk the ustar entries and return the first regular file whose basename + * matches. Non-file records (directories, PAX/GNU extension headers) never + * match and are stepped over by their declared size, so the reader stays + * correct against both GNU tar and bsdtar output. Returns null if absent. + */ +export function extractMmdbFromTarGz( + gz: Buffer, + mmdbBasename: string, +): Buffer | null { + const tar = zlib.gunzipSync(gz); + let offset = 0; + while (offset + TAR_BLOCK <= tar.length) { + const header = tar.subarray(offset, offset + TAR_BLOCK); + // Two all-zero blocks mark end-of-archive; one is enough to stop on. + if (header.every((b) => b === 0)) break; + + const name = readTarField(header, 0, 100); + const size = readTarOctal(header, 124, 12); + const typeflag = header[156]; + const dataStart = offset + TAR_BLOCK; + + // Regular file: typeflag '0' (0x30) or the legacy NUL (0x00). + const isFile = typeflag === 0x30 || typeflag === 0x00; + if (isFile && name.split("/").pop() === mmdbBasename) { + return Buffer.from(tar.subarray(dataStart, dataStart + size)); + } + + // Advance past the data, padded up to the next 512-byte boundary. + offset = dataStart + Math.ceil(size / TAR_BLOCK) * TAR_BLOCK; + } + return null; +} + +/** + * Download one edition and extract its `.mmdb` to a temp file in `destDir`. + * Returns the temp path — the caller promotes it with an atomic rename once + * every edition has been staged. Throws on any failure so the caller can + * discard all partial temp files and leave the existing databases untouched. + */ +async function downloadAndExtractEdition( + edition: Geolite2Edition, + licenseKey: string, + destDir: string, +): Promise { + const url = buildMaxmindDownloadUrl(edition.editionId, licenseKey); + const res = await safeFetch( + url, + { headers: { Accept: "application/gzip" } }, + { + // The permalink 302s to a signed CDN, so follow redirects; keep the + // connect-time public-host pin so a rebinding cannot swing the hop onto + // a private/metadata address. + followRedirects: true, + requirePublicHost: true, + timeoutMs: DOWNLOAD_TIMEOUT_MS, + }, + ); + if (!res.ok) { + // 401 = wrong key / unsigned EULA, 403 = throttle, 5xx = MaxMind-side. + throw new Error( + `${edition.editionId} download returned HTTP ${res.status}`, + ); + } + + const gz = Buffer.from(await res.arrayBuffer()); + const mmdb = extractMmdbFromTarGz(gz, edition.mmdb); + if (!mmdb) { + throw new Error( + `${edition.mmdb} not found inside the ${edition.editionId} tarball`, + ); + } + if (mmdb.length === 0) { + throw new Error(`${edition.mmdb} extracted empty`); + } + + // Stage under destDir (same filesystem) so the promote is an atomic rename. + // A `.tmp-` suffix keeps it out of the resolver's eye — `offlineGeoReady()` + // only accepts the exact `GeoLite2-City.mmdb`. + const tmpDest = path.join( + destDir, + `${edition.mmdb}.tmp-${process.pid}-${Date.now()}`, + ); + await fsp.writeFile(tmpDest, mmdb); + return tmpDest; +} + +/** + * Fetch both GeoLite2 databases and place them in `geoLiteDir()`. + * + * - `IP_GEO_LOOKUP_DISABLED=1` → `skipped_disabled` (the whole tier is off). + * - No `MAXMIND_LICENSE_KEY` → `skipped_no_key`. + * - `skipIfPresent` and the offline tier already ready → `skipped_present` + * (the boot path: only fetch when there is no database yet). + * + * On success the reader cache is reset so a running worker reads the new + * databases without a restart, and any stale keyless-build `.empty` marker is + * removed. Never throws. + */ +export async function fetchGeoLite2Databases( + opts: { skipIfPresent?: boolean } = {}, +): Promise { + if (process.env.IP_GEO_LOOKUP_DISABLED === "1") { + return { status: "skipped_disabled", editions: [] }; + } + const licenseKey = process.env.MAXMIND_LICENSE_KEY?.trim(); + if (!licenseKey) { + return { status: "skipped_no_key", editions: [] }; + } + if (opts.skipIfPresent && offlineGeoReady()) { + return { status: "skipped_present", editions: [] }; + } + + const dir = geoLiteDir(); + try { + await fsp.mkdir(dir, { recursive: true }); + } catch (err) { + return fail(`geolite2 directory not writable (${dir})`, err); + } + + const staged: { mmdb: string; tmp: string }[] = []; + try { + for (const edition of GEOLITE2_EDITIONS) { + const tmp = await downloadAndExtractEdition(edition, licenseKey, dir); + staged.push({ mmdb: edition.mmdb, tmp }); + } + } catch (err) { + // Fail-soft: drop every partial temp file, leave the existing databases. + await Promise.all( + staged.map((s) => fsp.rm(s.tmp, { force: true }).catch(() => {})), + ); + return fail("geolite2 download/extract failed", err); + } + + // Every edition is staged; promote them all with atomic renames. + try { + for (const s of staged) { + await fsp.rename(s.tmp, path.join(dir, s.mmdb)); + } + // A freshly populated directory must not keep a stale keyless-build marker + // (the resolver reads `.empty` as "offline tier deliberately absent"). + await fsp.rm(path.join(dir, ".empty"), { force: true }).catch(() => {}); + } catch (err) { + await Promise.all( + staged.map((s) => fsp.rm(s.tmp, { force: true }).catch(() => {})), + ); + return fail("geolite2 promote failed", err); + } + + // The running worker latched its readers before the files existed; reset so + // the next lookup reads them. + resetGeoLite2ReaderCache(); + + return { + status: "fetched", + editions: GEOLITE2_EDITIONS.map((e) => e.editionId), + }; +} diff --git a/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts b/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts new file mode 100644 index 000000000..7b65fe798 --- /dev/null +++ b/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts @@ -0,0 +1,137 @@ +/** + * Runtime GeoLite2 fetch (issue #659) — queue-wiring guard + boot gating. + * + * The source-text-grep half mirrors the other dead-queue guards: assert the + * queue is imported, in `allQueues`, scheduled, and bound to a `createAndWork` + * handler that runs the fetch — without booting pg-boss. An unregistered queue + * silently never drains (the recurring v1.4.37 dead-queue bug), which here + * would mean a keyed self-host never gets its databases fetched. + * + * The second half exercises the boot-discovery gate directly: it must send a + * job only when a key is set, egress is not disabled, and the offline tier is + * absent. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const REGISTRAR_PATH = join( + __dirname, + "..", + "reminder", + "register-maintenance.ts", +); +const workerSource = + readFileSync(REGISTRAR_PATH, "utf8") + + readFileSync(join(__dirname, "..", "reminder", "ops-handlers.ts"), "utf8"); + +describe("reminder-worker — geolite2-fetch wiring", () => { + it("imports the queue symbols from the geolite2-fetch module", () => { + expect(workerSource).toMatch(/from\s*["']@\/lib\/jobs\/geolite2-fetch["']/); + expect(workerSource).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/); + expect(workerSource).toMatch(/\bGEOLITE2_FETCH_CRON\b/); + expect(workerSource).toMatch(/\benqueueGeolite2FetchBootDiscovery\b/); + }); + + it("registers the queue in the allQueues loop", () => { + const allQueuesMatch = workerSource.match( + /const allQueues\s*=\s*\[([\s\S]*?)\];/, + ); + expect(allQueuesMatch).not.toBeNull(); + expect(allQueuesMatch![1]).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/); + }); + + it("schedules the monthly cron", () => { + expect(workerSource).toMatch( + /\[GEOLITE2_FETCH_QUEUE,\s*GEOLITE2_FETCH_CRON\]/, + ); + }); + + it("registers a createAndWork handler for the queue", () => { + expect(workerSource).toMatch( + /createAndWork[\s\S]{0,200}GEOLITE2_FETCH_QUEUE[\s\S]{0,200}handleGeolite2Fetch/, + ); + }); + + it("runs the fetch inside the handler", () => { + expect(workerSource).toMatch( + /handleGeolite2Fetch[\s\S]{0,500}fetchGeoLite2Databases/, + ); + }); + + it("fires the boot discovery from enqueueMaintenanceBootDiscovery", () => { + expect(workerSource).toMatch( + /enqueueMaintenanceBootDiscovery[\s\S]*enqueueGeolite2FetchBootDiscovery/, + ); + }); +}); + +// ── Boot-discovery gate ────────────────────────────────────────────── +const { sendMock, bossMock, offlineGeoReadyMock } = vi.hoisted(() => { + const sendMock = vi.fn().mockResolvedValue("job-1"); + return { + sendMock, + bossMock: { send: sendMock }, + offlineGeoReadyMock: vi.fn(), + }; +}); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: () => bossMock, +})); +vi.mock("@/lib/geo", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, offlineGeoReady: offlineGeoReadyMock }; +}); + +import { enqueueGeolite2FetchBootDiscovery } from "@/lib/jobs/geolite2-fetch"; + +const ORIGINAL_KEY = process.env.MAXMIND_LICENSE_KEY; +const ORIGINAL_DISABLED = process.env.IP_GEO_LOOKUP_DISABLED; + +describe("enqueueGeolite2FetchBootDiscovery — gating", () => { + beforeEach(() => { + sendMock.mockClear(); + sendMock.mockResolvedValue("job-1"); + offlineGeoReadyMock.mockReturnValue(false); + process.env.MAXMIND_LICENSE_KEY = "test-key"; + delete process.env.IP_GEO_LOOKUP_DISABLED; + }); + + afterEach(() => { + if (ORIGINAL_KEY === undefined) delete process.env.MAXMIND_LICENSE_KEY; + else process.env.MAXMIND_LICENSE_KEY = ORIGINAL_KEY; + if (ORIGINAL_DISABLED === undefined) + delete process.env.IP_GEO_LOOKUP_DISABLED; + else process.env.IP_GEO_LOOKUP_DISABLED = ORIGINAL_DISABLED; + }); + + it("enqueues a boot-mode fetch when keyed and no database is present", async () => { + const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); + expect(enqueued).toBe(1); + expect(sendMock).toHaveBeenCalledTimes(1); + const [queue, payload] = sendMock.mock.calls[0]; + expect(queue).toBe("geolite2-fetch"); + expect(payload).toMatchObject({ mode: "boot" }); + }); + + it("skips when no MAXMIND_LICENSE_KEY is set", async () => { + delete process.env.MAXMIND_LICENSE_KEY; + const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); + expect(enqueued).toBe(0); + expect(sendMock).not.toHaveBeenCalled(); + }); + + it("skips when IP_GEO_LOOKUP_DISABLED=1", async () => { + process.env.IP_GEO_LOOKUP_DISABLED = "1"; + const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); + expect(enqueued).toBe(0); + expect(sendMock).not.toHaveBeenCalled(); + }); + + it("skips when the offline tier is already present", async () => { + offlineGeoReadyMock.mockReturnValue(true); + const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); + expect(enqueued).toBe(0); + expect(sendMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/geolite2-fetch.ts b/src/lib/jobs/geolite2-fetch.ts new file mode 100644 index 000000000..f3c11eb6e --- /dev/null +++ b/src/lib/jobs/geolite2-fetch.ts @@ -0,0 +1,78 @@ +/** + * Runtime GeoLite2 fetch — pg-boss queue wiring (issue #659). + * + * The actual download + extract + place logic lives in + * `src/lib/geo/geolite2-fetch.ts`; this module owns the queue name, the cron, + * and the boot-discovery enqueue, mirroring the geo-backfill split + * (`geo-backfill.ts` holds `runGeoBackfill`, `ops-handlers.ts` holds the + * `boss.work` handler). Worker-only by construction — nothing here runs in the + * web request path. + * + * Two triggers: + * + * - BOOT: `enqueueGeolite2FetchBootDiscovery()` fires once at worker boot, + * only when a key is set and the offline tier is not already present. The + * first-run acquisition — pull `:latest`, set the key, restart, the + * databases appear without a manual mount. + * - CRON: monthly. MaxMind reissues GeoLite2 on the first Tuesday of each + * month; the 8th is safely past that, so a monthly refresh keeps a keyed + * host current. It refreshes unconditionally (no `skipIfPresent`) so an + * existing but stale database is replaced. + */ +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { offlineGeoReady } from "@/lib/geo"; +import { annotate } from "@/lib/logging/context"; + +export const GEOLITE2_FETCH_QUEUE = "geolite2-fetch"; + +// Monthly on the 8th at 04:00 Europe/Berlin — past the first-Tuesday reissue +// window, inside the existing 02:xx-04:xx maintenance band. NOT `cronIsTheRetry`: +// a monthly cadence is far too long to be the retry, so the queue keeps +// pg-boss's default retries for a transient MaxMind hiccup. +export const GEOLITE2_FETCH_CRON = "0 4 8 * *"; + +export interface Geolite2FetchPayload { + triggeredAt?: string; + /** `"boot"` fetches only when no database is present yet; `"refresh"` always fetches. */ + mode?: "boot" | "refresh"; +} + +/** + * Fire-and-forget boot discovery. Enqueues a single boot-mode fetch only when a + * key is configured, egress is not disabled, and the offline tier is absent — + * so a host that already has the databases (baked, mounted, or fetched on a + * prior boot) enqueues nothing. The handler re-checks all three; gating here + * just avoids a no-op job on the common keyless boot. + */ +export async function enqueueGeolite2FetchBootDiscovery(): Promise<{ + enqueued: number; +}> { + const boss = getGlobalBoss(); + if (!boss) return { enqueued: 0 }; + if (!process.env.MAXMIND_LICENSE_KEY?.trim()) return { enqueued: 0 }; + if (process.env.IP_GEO_LOOKUP_DISABLED === "1") return { enqueued: 0 }; + if (offlineGeoReady()) return { enqueued: 0 }; + + try { + const payload: Geolite2FetchPayload = { + mode: "boot", + triggeredAt: new Date().toISOString(), + }; + const jobId = await boss.send(GEOLITE2_FETCH_QUEUE, payload, { + retryLimit: 3, + retryDelay: 60, + retryBackoff: true, + singletonKey: "geolite2-fetch|boot", + }); + const enqueued = jobId ? 1 : 0; + annotate({ + action: { name: "geo.database.fetch.boot" }, + meta: { enqueued }, + }); + return { enqueued }; + } catch { + // A transient send failure is a no-op — the monthly cron and the next boot + // both re-attempt. + return { enqueued: 0 }; + } +} diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 2c255d692..72e390f2b 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -96,6 +96,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "geo_backfill_scanned", "geo_backfill_skipped", "geo_backfill_still_unresolved", + "geolite2_fetch_status", "host_metric_pruned", "hourly_rows_upserted", "imported", diff --git a/src/lib/jobs/reminder/ops-handlers.ts b/src/lib/jobs/reminder/ops-handlers.ts index a8774641e..7560e70f5 100644 --- a/src/lib/jobs/reminder/ops-handlers.ts +++ b/src/lib/jobs/reminder/ops-handlers.ts @@ -6,6 +6,8 @@ */ import { type Job } from "pg-boss"; import { runGeoBackfill } from "@/lib/jobs/geo-backfill"; +import { fetchGeoLite2Databases } from "@/lib/geo/geolite2-fetch"; +import { type Geolite2FetchPayload } from "@/lib/jobs/geolite2-fetch"; import { runTlsPinMonitor } from "@/lib/jobs/tls-pin-monitor"; import { type PrDetectionPayload } from "@/lib/jobs/pr-detection"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; @@ -138,6 +140,52 @@ export async function handleGeoBackfill( }); } +/** + * Runtime GeoLite2 fetch worker (issue #659). Downloads + places the offline + * MMDBs when a `MAXMIND_LICENSE_KEY` is set, so a self-hoster on the published + * image gets the offline geo tier without a rebuild or a manual mount. The + * fetch is fail-soft (a failure leaves the previous databases untouched); the + * boot mode only fetches when no database is present yet, the cron mode always + * refreshes. Skips (no key, egress disabled, already present) complete the job; + * a genuine download/extract failure fails it so pg-boss's default retries get + * a few attempts before the monthly cron catches the next window. + */ +export async function handleGeolite2Fetch( + jobs: Job[], +): Promise { + return withBackgroundEvent("job.geolite2_fetch", async (evt) => { + // Batched worker mode always hands us an array; the queue is single-flight + // in practice, so this loop is ~always one job. + let lastStatus = "none"; + let failure: JobOutcome | null = null; + for (const job of jobs) { + const mode = job.data?.mode === "boot" ? "boot" : "refresh"; + const result = await fetchGeoLite2Databases({ + skipIfPresent: mode === "boot", + }); + lastStatus = result.status; + evt.addMeta("geolite2_fetch_mode", mode); + evt.addMeta("geolite2_fetch_status", result.status); + if (result.status === "fetched") { + evt.addMeta("geolite2_fetch_editions", result.editions.join(",")); + } + if (result.status === "failed" && failure === null) { + evt.addWarning( + `geolite2-fetch failed: ${result.error ?? "unknown error"}`, + ); + failure = jobFailed( + "geolite2 fetch failed", + new Error(result.error ?? "unknown error"), + ); + } + } + return ( + failure ?? + jobDone({ geolite2_fetch_status: lastStatus, jobs: jobs.length }) + ); + }); +} + export async function handleTlsPinMonitor( jobs: Job[], ): Promise { diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 6eee62d3b..4a815dc03 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -17,6 +17,12 @@ */ import { PgBoss } from "pg-boss"; import { GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON } from "@/lib/jobs/geo-backfill"; +import { + GEOLITE2_FETCH_QUEUE, + GEOLITE2_FETCH_CRON, + enqueueGeolite2FetchBootDiscovery, + type Geolite2FetchPayload, +} from "@/lib/jobs/geolite2-fetch"; import { TLS_PIN_MONITOR_QUEUE, TLS_PIN_MONITOR_CRON, @@ -176,6 +182,7 @@ import { handleHostMetricSample, handleFeedbackAggregator, handleGeoBackfill, + handleGeolite2Fetch, handleTlsPinMonitor, handlePrDetection, } from "./ops-handlers"; @@ -335,6 +342,12 @@ const allQueues = [ HOST_METRIC_QUEUE, FEEDBACK_AGGREGATOR_QUEUE, GEO_BACKFILL_QUEUE, + // v1.37.7 (issue #659) — runtime GeoLite2 fetch. A self-hoster who sets + // MAXMIND_LICENSE_KEY on the published image gets the offline geo tier + // without a rebuild or a manual mount: boot discovery fetches when no + // database is present, a monthly cron keeps a keyed host current. Without + // this entry pg-boss never provisions the queue and both silently no-op. + GEOLITE2_FETCH_QUEUE, // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. The iOS client // pins the served leaf certificate; this queue probes it every 6 h and // alarms when the served SPKI leaves the operator's known-good set. @@ -472,6 +485,11 @@ const schedules: ScheduleEntry[] = [ // long tail of audit rows that landed with the offline MMDB // missing or the online provider unreachable. [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry], + // v1.37.7 (issue #659) — monthly runtime GeoLite2 refresh (8th, 04:00 + // Europe/Berlin, past MaxMind's first-Tuesday reissue). Deliberately NOT + // cronIsTheRetry: a month between ticks is far too long to be the retry, so + // the queue keeps pg-boss's default retries for a transient MaxMind hiccup. + [GEOLITE2_FETCH_QUEUE, GEOLITE2_FETCH_CRON], // v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the // hourly sync crons). Surfaces a pinned-leaf rotation well inside the // ≥11-day re-pin window the iOS release owner needs. @@ -715,6 +733,14 @@ export async function registerMaintenanceQueues( { localConcurrency: 1 }, handleGeoBackfill, ); + // v1.37.7 (issue #659) — runtime GeoLite2 fetch. Single-flight: one download + // at a time, and two ticks racing the same atomic place is wasted bandwidth. + await createAndWork( + boss, + GEOLITE2_FETCH_QUEUE, + { localConcurrency: 1 }, + handleGeolite2Fetch, + ); // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one // short outbound TLS handshake per tick, no benefit to overlapping ticks. await createAndWork( @@ -1475,4 +1501,20 @@ export async function enqueueMaintenanceBootDiscovery(): Promise { err, ); } + + // v1.37.7 (issue #659) — runtime GeoLite2 fetch. Only enqueues when a + // MAXMIND_LICENSE_KEY is set, egress is not disabled, and the offline tier + // is not already present — so a self-hoster who sets the key on the + // published image gets the databases fetched on the next worker boot without + // a rebuild or a manual mount. + try { + const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); + workerLog("info", `[geolite2-fetch] boot discovery: enqueued=${enqueued}`); + } catch (err) { + workerLog( + "error", + "[geolite2-fetch] boot discovery threw an unexpected error", + err, + ); + } } diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index 89dd1f05f..e04088d14 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -104,6 +104,7 @@ const subsystemSurface = { "host-metric-sample": { audience: "system" }, "feedback-aggregator": { audience: "system" }, "geo-backfill": { audience: "system" }, + "geolite2-fetch": { audience: "system" }, "tls-pin-monitor": { audience: "system" }, "pr-detection": { audience: "system" }, "medication-inventory-expire": { audience: "system" },