mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 13:22:14 +00:00
Merge pull request #766 from MBombeck/feat/geolite2-runtime
feat/geolite2-runtime — runtime GeoLite2 fetch (#659)
This commit is contained in:
21 files changed
+1036
-31
No files matched your search
@@ -264,3 +264,6 @@ API_TOKEN_HMAC_KEY="REPLACE-WITH-OUTPUT-OF-openssl-rand-hex-32"
|
||||
#
|
||||
# Directory the offline GeoLite2 databases are read from; default /opt/geolite2 (the path the image bakes its own copy into). Point it at a directory you mount yourself to bring your own MMDBs — see docs/self-hosting/geolite2.md.
|
||||
# GEOLITE2_DIR="/var/lib/geolite2"
|
||||
#
|
||||
# MaxMind licence key for the runtime GeoLite2 fetch (issue #659): set it and the worker downloads both databases into GEOLITE2_DIR itself, no rebuild or manual mount. Free key at https://www.maxmind.com/en/geolite2/signup — see docs/self-hosting/geolite2.md.
|
||||
# MAXMIND_LICENSE_KEY="your-maxmind-licence-key"
|
||||
+20
-7
@@ -328,14 +328,27 @@ API_TOKEN_HMAC_KEY="<openssl rand -hex 32>"
|
||||
# a stock `:latest` the directory is usually empty and every login IP goes
|
||||
# to the online provider instead.
|
||||
#
|
||||
# To resolve locations on your own host: create a free MaxMind account,
|
||||
# pull both databases with `geoipupdate`, mount the directory into the
|
||||
# container read-only (the commented `volumes:` block in docker-compose.yml
|
||||
# shows the line), and point this at the container path. Pair it with
|
||||
# IP_GEO_LOOKUP_DISABLED="1" above for zero IP egress. Refreshing the files
|
||||
# on the host is enough -- the reader picks them up on the next container
|
||||
# restart. See docs/self-hosting/geolite2.md.
|
||||
# Three ways to get the databases:
|
||||
#
|
||||
# 1. RUNTIME FETCH (issue #659) -- set MAXMIND_LICENSE_KEY below and the
|
||||
# worker downloads both databases into GEOLITE2_DIR itself on the next
|
||||
# boot and monthly thereafter. No image rebuild, no manual mount. This is
|
||||
# the way for a stock `:latest` pull.
|
||||
# 2. Bring your own -- create a free MaxMind account, pull both databases
|
||||
# with `geoipupdate`, mount the directory into the container read-only
|
||||
# (the commented `volumes:` block in docker-compose.yml shows the line),
|
||||
# and point GEOLITE2_DIR at the container path. Refreshing the files on
|
||||
# the host is enough -- the reader picks them up on the next restart.
|
||||
# 3. Bake them at build time (build your own image with a licence key).
|
||||
#
|
||||
# Pair any of them with IP_GEO_LOOKUP_DISABLED="1" above for zero IP egress.
|
||||
# See docs/self-hosting/geolite2.md.
|
||||
# GEOLITE2_DIR="/var/lib/geolite2"
|
||||
#
|
||||
# MaxMind licence key for the runtime fetch (option 1 above). Free: sign up at
|
||||
# https://www.maxmind.com/en/geolite2/signup, then My Account -> Manage
|
||||
# License Keys. Unset, nothing is fetched and the online provider answers.
|
||||
# MAXMIND_LICENSE_KEY="your-maxmind-licence-key"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Environmental-context module (Open-Meteo) -- optional
|
||||
|
||||
+9
-1
@@ -1,6 +1,14 @@
|
||||
# Changelog
|
||||
|
||||
## [1.37.6] — 2026-08-09
|
||||
## [1.37.7] — 2026-08-09
|
||||
|
||||
### Added
|
||||
|
||||
- A self-hoster running the published image can now set MAXMIND_LICENSE_KEY as a runtime environment variable and the app fetches the GeoLite2 City and ASN databases itself, so login locations resolve offline without a rebuild or a manual database mount. It fetches once on worker boot when the key is set and the databases are absent, refreshes monthly, and never blocks anything: any download error leaves the previous databases in place and the online lookup keeps answering. A read-only mounted database still wins. Documented as the third way to supply GeoLite2 in the self-hosting guide. Refs #659.
|
||||
|
||||
### Security
|
||||
|
||||
- The MaxMind licence key no longer appears in a log line when the runtime GeoLite2 fetch fails. The download URL carries the key in its query string, and the failure path used to print the full URL; it now reports only the edition and the kind of error, and the central log redactor also masks a license_key parameter wherever one might otherwise slip through.
|
||||
|
||||
### Security
|
||||
|
||||
|
||||
+10
-2
@@ -179,8 +179,16 @@ RUN mkdir -p /opt/prisma-cli && \
|
||||
#
|
||||
# Attribution (CC BY-SA 4.0): see `docs/audit/v1427-summary.md` and
|
||||
# `/about` in the running app.
|
||||
RUN mkdir -p /opt/geolite2
|
||||
COPY assets/geolite2/ /opt/geolite2/
|
||||
#
|
||||
# v1.37.7 (issue #659) — the directory is owned by the unprivileged `nextjs`
|
||||
# user so the runtime worker can WRITE it: a self-hoster who sets
|
||||
# MAXMIND_LICENSE_KEY on the published image has the databases fetched into
|
||||
# this directory at runtime (src/lib/geo/geolite2-fetch.ts) with no rebuild or
|
||||
# manual mount. `tar`/`gzip` for the extraction ship in the busybox base. A
|
||||
# read-only bind mount at this path (the bring-your-own-database route) still
|
||||
# works — the runtime fetch just fails soft and the mounted files stand.
|
||||
RUN mkdir -p /opt/geolite2 && chown nextjs:nodejs /opt/geolite2
|
||||
COPY --chown=nextjs:nodejs assets/geolite2/ /opt/geolite2/
|
||||
|
||||
# ISO-8601 build timestamp — /api/version returns it as `builtAt`.
|
||||
# Declared this late in the stage on purpose: the value differs on
|
||||
|
||||
@@ -88,6 +88,16 @@ x-healthlog-env: &healthlog_env
|
||||
# value in .env reaches the container (the `environment:` block is a
|
||||
# whitelist). See docs/self-hosting/geolite2.md.
|
||||
GEOLITE2_DIR: "${GEOLITE2_DIR:-/opt/geolite2}"
|
||||
# MaxMind licence key for the RUNTIME GeoLite2 fetch (issue #659). Set this
|
||||
# and the worker downloads GeoLite2-City + GeoLite2-ASN itself into
|
||||
# GEOLITE2_DIR on the next boot and monthly thereafter — no image rebuild,
|
||||
# no manual mount. Get a free key at
|
||||
# https://www.maxmind.com/en/geolite2/signup (My Account -> Manage License
|
||||
# Keys). Leave it unset and nothing is fetched (mounting your own databases
|
||||
# or IP_GEO_LOOKUP_DISABLED=1 both still work). Listed here so an operator
|
||||
# value in .env reaches the container (the `environment:` block is a
|
||||
# whitelist). See docs/self-hosting/geolite2.md.
|
||||
MAXMIND_LICENSE_KEY: "${MAXMIND_LICENSE_KEY:-}"
|
||||
# all = bundle web + worker in this container (default, single-host)
|
||||
# web = serve HTTP only; pair with a separate app-worker service
|
||||
HEALTHLOG_PROCESS_TYPE: "${HEALTHLOG_PROCESS_TYPE:-all}"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: HealthLog API
|
||||
version: 1.37.6
|
||||
version: 1.37.7
|
||||
description: >-
|
||||
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
|
||||
|
||||
|
||||
@@ -16,11 +16,53 @@ next to every sign-in. Two tiers can produce that:
|
||||
The offline tier is optional and absent by default. The published image
|
||||
only carries the databases when the image build had a MaxMind licence key
|
||||
(`scripts/fetch-geolite2.sh`, run from the release workflow), which is a
|
||||
build-time decision you do not control when you pull `:latest`. Bringing
|
||||
your own copy is the way to get the offline tier on a self-host, and it
|
||||
is the subject of this page.
|
||||
build-time decision you do not control when you pull `:latest`. Two ways
|
||||
get the offline tier onto a stock `:latest` self-host, and both are on this
|
||||
page: let the app **fetch the databases itself** at runtime (simplest, one
|
||||
env var), or **bring your own** copy and mount it.
|
||||
|
||||
## What you need
|
||||
## Runtime fetch (one env var)
|
||||
|
||||
Set a MaxMind licence key and the app downloads the databases itself — no
|
||||
rebuild, no mount, no `geoipupdate` on the host.
|
||||
|
||||
1. Create a free MaxMind account at
|
||||
<https://www.maxmind.com/en/geolite2/signup>, then a licence key under
|
||||
**My Account → Manage License Keys**. GeoLite2 is free; the account and
|
||||
key exist because MaxMind requires them for the download.
|
||||
|
||||
2. Put the key in your `.env`:
|
||||
|
||||
```env
|
||||
MAXMIND_LICENSE_KEY="your-licence-key"
|
||||
```
|
||||
|
||||
`MAXMIND_LICENSE_KEY` is on the compose `environment:` whitelist, so a
|
||||
value in `.env` reaches the container. Leave `GEOLITE2_DIR` unset to use
|
||||
the default `/opt/geolite2`, which the image already owns as the app
|
||||
user for exactly this write.
|
||||
|
||||
3. `docker compose up -d`. On the next worker boot the app fetches
|
||||
`GeoLite2-City.mmdb` and `GeoLite2-ASN.mmdb` into `GEOLITE2_DIR` and
|
||||
starts reading them — no restart needed beyond the one that picks up the
|
||||
new env var. It refreshes once a month afterwards (MaxMind reissues on
|
||||
the first Tuesday), so a keyed host stays current on its own.
|
||||
|
||||
The fetch is fail-soft: if the key is wrong or MaxMind is unreachable, the
|
||||
previous state stands (the online provider keeps answering) and the worker
|
||||
log carries the reason. It never writes a partial database. Confirm it
|
||||
landed under **Admin → System status** (`Geo data: Offline (GeoLite2)`) or
|
||||
from the host: `docker compose exec app ls -l /opt/geolite2`.
|
||||
|
||||
If you would rather not have the app reach MaxMind on its own schedule —
|
||||
for example you keep a curated database copy — skip the key and use
|
||||
**bring your own** below instead. The two do not conflict: a read-only
|
||||
bind mount wins, because the runtime fetch cannot write over it and just
|
||||
fails soft.
|
||||
|
||||
## Bring your own (mount a copy)
|
||||
|
||||
### What you need
|
||||
|
||||
- A free MaxMind account. Sign up at
|
||||
<https://www.maxmind.com/en/geolite2/signup>, then create a licence key
|
||||
@@ -30,7 +72,7 @@ is the subject of this page.
|
||||
two `.mmdb` files onto the host. Most distributions package
|
||||
`geoipupdate`; there is also an official container image.
|
||||
|
||||
## Setup
|
||||
### Setup
|
||||
|
||||
1. Pick a directory on the host and fetch both databases into it. With
|
||||
`geoipupdate`, `/etc/GeoIP.conf` looks like this:
|
||||
@@ -88,12 +130,17 @@ health data is a reasonable place to make that trade.
|
||||
|
||||
MaxMind reissues GeoLite2 on its own schedule, and a stale city database
|
||||
is quietly wrong rather than loudly broken — it keeps answering, with
|
||||
last month's allocations. Run `geoipupdate` on a schedule (a weekly cron
|
||||
or systemd timer is enough) against the same directory.
|
||||
last month's allocations.
|
||||
|
||||
The reader loads each database once per worker process and holds it, so a
|
||||
refreshed file on the host is picked up at the next container restart,
|
||||
not immediately. A weekly `geoipupdate` followed by
|
||||
The **runtime fetch** handles this for you: it re-downloads both databases
|
||||
once a month and resets the reader in place, so a keyed host stays current
|
||||
with no cron of your own and no restart.
|
||||
|
||||
The **bring-your-own** path is yours to keep fresh. Run `geoipupdate` on a
|
||||
schedule (a weekly cron or systemd timer is enough) against the same
|
||||
directory. The reader loads each database once per worker process and holds
|
||||
it, so a refreshed file on the host is picked up at the next container
|
||||
restart, not immediately. A weekly `geoipupdate` followed by
|
||||
`docker compose restart app` keeps the two in step.
|
||||
|
||||
## Licence and attribution
|
||||
|
||||
+10
-1
@@ -117,7 +117,16 @@
|
||||
"@openai/codex-security",
|
||||
"postcss",
|
||||
],
|
||||
"ignoreBinaries": ["docker", "docker-compose", "pg_isready", "tsx", "cosign"],
|
||||
"ignoreBinaries": [
|
||||
"docker",
|
||||
"docker-compose",
|
||||
"pg_isready",
|
||||
"tsx",
|
||||
"cosign",
|
||||
// The runtime GeoLite2 fetch extracts in pure Node, but its test builds a
|
||||
// MaxMind-shaped tar.gz fixture with the POSIX `tar` (present in CI + dev).
|
||||
"tar",
|
||||
],
|
||||
// Scoped to a single file: an export used only inside its own module is not
|
||||
// a finding. It is also why the default pass cannot answer the question the
|
||||
// production pass answers — this setting is about exports, not reachability.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "healthlog",
|
||||
"version": "1.37.6",
|
||||
"version": "1.37.7",
|
||||
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
"homepage": "https://healthlog.dev",
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ try {
|
||||
// v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild.
|
||||
const CACHE_VERSION =
|
||||
(typeof self !== "undefined" && self.__APP_VERSION__) ||
|
||||
/* @sw-version-fallback */ "v1.37.6";
|
||||
/* @sw-version-fallback */ "v1.37.7";
|
||||
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
|
||||
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
|
||||
// v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`
|
||||
|
||||
@@ -140,6 +140,10 @@
|
||||
{
|
||||
"name": "GEOLITE2_DIR",
|
||||
"purpose": "Directory holding GeoLite2-City.mmdb + GeoLite2-ASN.mmdb. Point it at a read-only mount to bring your own databases. Default /opt/geolite2."
|
||||
},
|
||||
{
|
||||
"name": "MAXMIND_LICENSE_KEY",
|
||||
"purpose": "MaxMind licence key for the runtime GeoLite2 fetch (issue #659). Set it and the worker downloads both databases into GEOLITE2_DIR on boot and monthly, no rebuild or manual mount. Free at maxmind.com/en/geolite2/signup."
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
+19
-6
@@ -169,7 +169,7 @@ export function resolveGeoProviderHost(): string {
|
||||
* as unset, otherwise a stack that merely lists the variable would read
|
||||
* the process working directory and silently lose the offline tier.
|
||||
*/
|
||||
function geoLiteDir(): string {
|
||||
export function geoLiteDir(): string {
|
||||
const configured = process.env.GEOLITE2_DIR?.trim();
|
||||
return configured ? configured : "/opt/geolite2";
|
||||
}
|
||||
@@ -252,15 +252,28 @@ function getAsnReader(): MmdbReader<AsnResponse> | null {
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only — reset the lazy reader cache so a test can swap the
|
||||
* `GEOLITE2_DIR` between cases without leaking the previous Reader.
|
||||
* Not exported from the public surface; the tests import via the
|
||||
* module-level alias.
|
||||
* Reset the lazy MMDB reader cache so the next lookup re-reads the files
|
||||
* from `geoLiteDir()`. The runtime GeoLite2 fetch (`src/lib/geo/geolite2-fetch.ts`)
|
||||
* calls this after it places freshly downloaded databases, so a running worker
|
||||
* picks them up without a restart — the readers latch `null` on the first miss,
|
||||
* so without this a process that booted before the files landed would keep
|
||||
* resolving online forever. Also clears the one-shot "offline unavailable"
|
||||
* admin-notification latch, since the offline tier may now be present.
|
||||
*/
|
||||
export function __resetGeoLite2CacheForTests(): void {
|
||||
export function resetGeoLite2ReaderCache(): void {
|
||||
cache.city = undefined;
|
||||
cache.asn = undefined;
|
||||
notifiedThisProcess = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only — reset the lazy reader cache so a test can swap the
|
||||
* `GEOLITE2_DIR` between cases without leaking the previous Reader. Extends
|
||||
* the runtime reset with the per-IP location cache, which production has no
|
||||
* reason to drop.
|
||||
*/
|
||||
export function __resetGeoLite2CacheForTests(): void {
|
||||
resetGeoLite2ReaderCache();
|
||||
GEO_CACHE.clear();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Runtime GeoLite2 fetch (issue #659) — unit coverage.
|
||||
*
|
||||
* The network is mocked (safeFetch), but the filesystem + `tar` extraction run
|
||||
* for real against a tar.gz fixture built the same way MaxMind ships one (the
|
||||
* mmdb nested under a date-stamped directory). That exercises the actual
|
||||
* temp-then-rename place path, not a stub of it.
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { SafeFetchError } from "@/lib/safe-fetch";
|
||||
|
||||
const { safeFetchMock } = vi.hoisted(() => ({ safeFetchMock: vi.fn() }));
|
||||
vi.mock("@/lib/safe-fetch", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/safe-fetch")>();
|
||||
return { ...actual, safeFetch: safeFetchMock };
|
||||
});
|
||||
|
||||
const { resetCacheSpy } = vi.hoisted(() => ({ resetCacheSpy: vi.fn() }));
|
||||
vi.mock("@/lib/geo", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/geo")>();
|
||||
return { ...actual, resetGeoLite2ReaderCache: resetCacheSpy };
|
||||
});
|
||||
|
||||
import {
|
||||
buildMaxmindDownloadUrl,
|
||||
fetchGeoLite2Databases,
|
||||
MAXMIND_DOWNLOAD_BASE,
|
||||
} from "@/lib/geo/geolite2-fetch";
|
||||
|
||||
const CITY_BYTES = Buffer.from("CITY-MMDB-CONTENT-");
|
||||
const ASN_BYTES = Buffer.from("ASN-MMDB-CONTENT-");
|
||||
|
||||
/** Build a MaxMind-shaped tar.gz: `<Edition>_<date>/<Edition>.mmdb`. */
|
||||
function buildEditionTarball(
|
||||
editionId: string,
|
||||
mmdbBytes: Buffer,
|
||||
): ArrayBuffer {
|
||||
const stage = fs.mkdtempSync(path.join(os.tmpdir(), "geolite2-fixture-"));
|
||||
try {
|
||||
const inner = `${editionId}_20260808`;
|
||||
fs.mkdirSync(path.join(stage, inner));
|
||||
fs.writeFileSync(path.join(stage, inner, `${editionId}.mmdb`), mmdbBytes);
|
||||
const tarball = path.join(stage, `${editionId}.tar.gz`);
|
||||
execFileSync("tar", ["-czf", tarball, "-C", stage, inner]);
|
||||
const buf = fs.readFileSync(tarball);
|
||||
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
||||
} finally {
|
||||
fs.rmSync(stage, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function okResponse(body: ArrayBuffer): Response {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
arrayBuffer: async () => body,
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
let geoDir: string;
|
||||
const ORIGINAL_KEY = process.env.MAXMIND_LICENSE_KEY;
|
||||
const ORIGINAL_DISABLED = process.env.IP_GEO_LOOKUP_DISABLED;
|
||||
const ORIGINAL_DIR = process.env.GEOLITE2_DIR;
|
||||
|
||||
beforeEach(() => {
|
||||
safeFetchMock.mockReset();
|
||||
resetCacheSpy.mockReset();
|
||||
geoDir = fs.mkdtempSync(path.join(os.tmpdir(), "geolite2-target-"));
|
||||
process.env.GEOLITE2_DIR = geoDir;
|
||||
process.env.MAXMIND_LICENSE_KEY = "test-licence-key";
|
||||
delete process.env.IP_GEO_LOOKUP_DISABLED;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(geoDir, { recursive: true, force: true });
|
||||
if (ORIGINAL_KEY === undefined) delete process.env.MAXMIND_LICENSE_KEY;
|
||||
else process.env.MAXMIND_LICENSE_KEY = ORIGINAL_KEY;
|
||||
if (ORIGINAL_DISABLED === undefined)
|
||||
delete process.env.IP_GEO_LOOKUP_DISABLED;
|
||||
else process.env.IP_GEO_LOOKUP_DISABLED = ORIGINAL_DISABLED;
|
||||
if (ORIGINAL_DIR === undefined) delete process.env.GEOLITE2_DIR;
|
||||
else process.env.GEOLITE2_DIR = ORIGINAL_DIR;
|
||||
});
|
||||
|
||||
/** Route each edition's mocked download to its fixture tarball. */
|
||||
function wireTarballDownloads(): void {
|
||||
const city = buildEditionTarball("GeoLite2-City", CITY_BYTES);
|
||||
const asn = buildEditionTarball("GeoLite2-ASN", ASN_BYTES);
|
||||
safeFetchMock.mockImplementation(async (url: string) => {
|
||||
if (url.includes("GeoLite2-City")) return okResponse(city);
|
||||
if (url.includes("GeoLite2-ASN")) return okResponse(asn);
|
||||
throw new Error(`unexpected url ${url}`);
|
||||
});
|
||||
}
|
||||
|
||||
describe("buildMaxmindDownloadUrl", () => {
|
||||
it("builds the permalink URL per edition with the key + tar.gz suffix", () => {
|
||||
const url = buildMaxmindDownloadUrl("GeoLite2-City", "abc123");
|
||||
expect(url.startsWith(`${MAXMIND_DOWNLOAD_BASE}?`)).toBe(true);
|
||||
const params = new URL(url).searchParams;
|
||||
expect(params.get("edition_id")).toBe("GeoLite2-City");
|
||||
expect(params.get("license_key")).toBe("abc123");
|
||||
expect(params.get("suffix")).toBe("tar.gz");
|
||||
});
|
||||
});
|
||||
|
||||
describe("fetchGeoLite2Databases — success", () => {
|
||||
it("downloads both editions and places the mmdbs atomically", async () => {
|
||||
wireTarballDownloads();
|
||||
fs.writeFileSync(path.join(geoDir, ".empty"), ""); // stale keyless marker
|
||||
|
||||
const result = await fetchGeoLite2Databases();
|
||||
|
||||
expect(result.status).toBe("fetched");
|
||||
expect(result.editions).toEqual(["GeoLite2-City", "GeoLite2-ASN"]);
|
||||
|
||||
// Both databases landed with the exact fixture bytes.
|
||||
expect(fs.readFileSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toEqual(
|
||||
CITY_BYTES,
|
||||
);
|
||||
expect(fs.readFileSync(path.join(geoDir, "GeoLite2-ASN.mmdb"))).toEqual(
|
||||
ASN_BYTES,
|
||||
);
|
||||
|
||||
// The stale `.empty` marker is gone and no temp/work residue remains.
|
||||
expect(fs.existsSync(path.join(geoDir, ".empty"))).toBe(false);
|
||||
const leftovers = fs
|
||||
.readdirSync(geoDir)
|
||||
.filter((n) => n.startsWith(".geolite2-") || n.includes(".tmp-"));
|
||||
expect(leftovers).toEqual([]);
|
||||
|
||||
// The running worker's reader cache is reset so it re-reads the new files.
|
||||
expect(resetCacheSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("requests each edition with redirect-following + the public-host pin", async () => {
|
||||
wireTarballDownloads();
|
||||
|
||||
await fetchGeoLite2Databases();
|
||||
|
||||
expect(safeFetchMock).toHaveBeenCalledTimes(2);
|
||||
for (const call of safeFetchMock.mock.calls) {
|
||||
const [url, , opts] = call;
|
||||
expect(url).toContain("license_key=test-licence-key");
|
||||
expect(url).toContain("suffix=tar.gz");
|
||||
expect(opts).toMatchObject({
|
||||
followRedirects: true,
|
||||
requirePublicHost: true,
|
||||
});
|
||||
expect(typeof opts.timeoutMs).toBe("number");
|
||||
}
|
||||
const urls = safeFetchMock.mock.calls.map((c) => c[0] as string);
|
||||
expect(urls.some((u) => u.includes("edition_id=GeoLite2-City"))).toBe(true);
|
||||
expect(urls.some((u) => u.includes("edition_id=GeoLite2-ASN"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fetchGeoLite2Databases — fail-soft", () => {
|
||||
it("leaves an existing database untouched and writes no partial file on a download error", async () => {
|
||||
const existing = Buffer.from("EXISTING-GOOD-CITY-DB");
|
||||
fs.writeFileSync(path.join(geoDir, "GeoLite2-City.mmdb"), existing);
|
||||
|
||||
safeFetchMock.mockRejectedValue(new SafeFetchError("boom", "network"));
|
||||
|
||||
const result = await fetchGeoLite2Databases();
|
||||
|
||||
expect(result.status).toBe("failed");
|
||||
// The pre-existing database is byte-for-byte intact.
|
||||
expect(fs.readFileSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toEqual(
|
||||
existing,
|
||||
);
|
||||
// No partial temp file, no orphaned work directory.
|
||||
const residue = fs
|
||||
.readdirSync(geoDir)
|
||||
.filter((n) => n.startsWith(".geolite2-") || n.includes(".tmp-"));
|
||||
expect(residue).toEqual([]);
|
||||
// A failed fetch must NOT reset the reader cache.
|
||||
expect(resetCacheSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never leaks the licence key into the error on a fetch-layer throw", async () => {
|
||||
// safeFetch embeds the full target URL — which carries `license_key=<secret>`
|
||||
// — in its thrown message. The download timeout was raised to 120s for slow
|
||||
// self-host links, so timeouts are an expected path; the reported error must
|
||||
// name the failure without the URL, or the key lands on stdout / Loki.
|
||||
safeFetchMock.mockRejectedValue(
|
||||
new SafeFetchError(
|
||||
"safeFetch aborted (TimeoutError): https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=test-licence-key&suffix=tar.gz",
|
||||
"timeout",
|
||||
),
|
||||
);
|
||||
|
||||
const result = await fetchGeoLite2Databases();
|
||||
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.error).toBeDefined();
|
||||
expect(result.error).not.toContain("test-licence-key");
|
||||
expect(result.error).not.toContain("license_key");
|
||||
});
|
||||
|
||||
it("promotes nothing when the first edition succeeds but the second fails", async () => {
|
||||
// The atomicity contract: City downloads + extracts fine, ASN 401s. Since
|
||||
// not every edition was staged, NOTHING is promoted — no GeoLite2-City.mmdb
|
||||
// appears — and the City temp file is cleaned up.
|
||||
const city = buildEditionTarball("GeoLite2-City", CITY_BYTES);
|
||||
safeFetchMock.mockImplementation(async (url: string) => {
|
||||
if (url.includes("GeoLite2-City")) return okResponse(city);
|
||||
return {
|
||||
ok: false,
|
||||
status: 401,
|
||||
arrayBuffer: async () => new ArrayBuffer(0),
|
||||
} as unknown as Response;
|
||||
});
|
||||
|
||||
const result = await fetchGeoLite2Databases();
|
||||
|
||||
expect(result.status).toBe("failed");
|
||||
// City must NOT have been promoted — the place is all-or-nothing.
|
||||
expect(fs.existsSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toBe(false);
|
||||
// And its staged temp file must be gone.
|
||||
const residue = fs
|
||||
.readdirSync(geoDir)
|
||||
.filter((n) => n.startsWith(".geolite2-") || n.includes(".tmp-"));
|
||||
expect(residue).toEqual([]);
|
||||
expect(resetCacheSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails soft when a non-ok HTTP status comes back", async () => {
|
||||
safeFetchMock.mockResolvedValue({
|
||||
ok: false,
|
||||
status: 401,
|
||||
arrayBuffer: async () => new ArrayBuffer(0),
|
||||
} as unknown as Response);
|
||||
|
||||
const result = await fetchGeoLite2Databases();
|
||||
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.error).toContain("401");
|
||||
expect(fs.existsSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fetchGeoLite2Databases — gating", () => {
|
||||
it("skips when IP_GEO_LOOKUP_DISABLED=1", async () => {
|
||||
process.env.IP_GEO_LOOKUP_DISABLED = "1";
|
||||
const result = await fetchGeoLite2Databases();
|
||||
expect(result.status).toBe("skipped_disabled");
|
||||
expect(safeFetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips when no MAXMIND_LICENSE_KEY is set", async () => {
|
||||
delete process.env.MAXMIND_LICENSE_KEY;
|
||||
const result = await fetchGeoLite2Databases();
|
||||
expect(result.status).toBe("skipped_no_key");
|
||||
expect(safeFetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips a boot-mode fetch when the offline tier is already present", async () => {
|
||||
fs.writeFileSync(
|
||||
path.join(geoDir, "GeoLite2-City.mmdb"),
|
||||
Buffer.from("already-here"),
|
||||
);
|
||||
const result = await fetchGeoLite2Databases({ skipIfPresent: true });
|
||||
expect(result.status).toBe("skipped_present");
|
||||
expect(safeFetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fetches anyway (refresh) when present but skipIfPresent is not set", async () => {
|
||||
wireTarballDownloads();
|
||||
fs.writeFileSync(
|
||||
path.join(geoDir, "GeoLite2-City.mmdb"),
|
||||
Buffer.from("old-city"),
|
||||
);
|
||||
const result = await fetchGeoLite2Databases();
|
||||
expect(result.status).toBe("fetched");
|
||||
expect(fs.readFileSync(path.join(geoDir, "GeoLite2-City.mmdb"))).toEqual(
|
||||
CITY_BYTES,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* Runtime GeoLite2 database fetch (issue #659).
|
||||
*
|
||||
* A self-hoster running the published image cannot set a build secret, so the
|
||||
* two pre-existing ways to get the offline MMDBs — bake them at build time via
|
||||
* `scripts/fetch-geolite2.sh`, or mount your own and point `GEOLITE2_DIR` at
|
||||
* them — both need something the operator of a stock `:latest` pull does not
|
||||
* have. This is the third way: given `MAXMIND_LICENSE_KEY` at runtime, the
|
||||
* worker downloads GeoLite2-City + GeoLite2-ASN itself, extracts each, and
|
||||
* places them in `geoLiteDir()`. No rebuild, no manual mount.
|
||||
*
|
||||
* Fail-soft is the whole contract. Every download extracts to a temp file in
|
||||
* the SAME directory as the target, and a completed edition is promoted with an
|
||||
* atomic `rename` only after ALL editions are in hand — so a mid-fetch error
|
||||
* (bad key, MaxMind 5xx, a truncated body) leaves the previous databases exactly
|
||||
* as they were and never writes a partial `.mmdb`. The offline tier is optional;
|
||||
* a failure just means the online provider keeps answering, which is the
|
||||
* keyless default anyway.
|
||||
*
|
||||
* The MaxMind permalink (`download.maxmind.com/app/geoip_download`) 302-redirects
|
||||
* to a signed CDN URL, so the fetch opts into `followRedirects: true` on top of
|
||||
* the `requirePublicHost: true` DNS-rebinding pin.
|
||||
*
|
||||
* Extraction is pure Node: `zlib.gunzipSync` + a minimal in-process tar reader
|
||||
* (`extractMmdbFromTarGz`), NOT a shell-out to `tar`. The runtime image does
|
||||
* carry busybox `tar`+`gzip`, but this module is reachable from the worker-boot
|
||||
* import chain (`instrumentation.ts` → `reminder-worker` → this file), and a
|
||||
* `child_process` import there makes Next's file tracer give up and trace the
|
||||
* whole project into the standalone output. `node:zlib` keeps the trace bounded,
|
||||
* removes the busybox runtime dependency entirely, and is trivially unit-testable
|
||||
* — the MaxMind tarball nests a single `.mmdb` under one date-stamped directory,
|
||||
* a shape a ~40-line reader handles without a dependency.
|
||||
*/
|
||||
import zlib from "node:zlib";
|
||||
import fsp from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { safeFetch, SafeFetchError } from "@/lib/safe-fetch";
|
||||
import {
|
||||
geoLiteDir,
|
||||
offlineGeoReady,
|
||||
resetGeoLite2ReaderCache,
|
||||
} from "@/lib/geo";
|
||||
|
||||
/**
|
||||
* The MaxMind permalink download endpoint. Same URL shape the build-time
|
||||
* `scripts/fetch-geolite2.sh` uses: `?edition_id=…&license_key=…&suffix=tar.gz`.
|
||||
*/
|
||||
export const MAXMIND_DOWNLOAD_BASE =
|
||||
"https://download.maxmind.com/app/geoip_download";
|
||||
|
||||
export interface Geolite2Edition {
|
||||
/** MaxMind edition id, e.g. `GeoLite2-City`. */
|
||||
editionId: string;
|
||||
/** The `.mmdb` basename the resolver in `geo.ts` reads. */
|
||||
mmdb: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The two editions the resolver reads: City for location, ASN for the
|
||||
* authoritative carrier + AS number. Both must land for the fetch to count as
|
||||
* a success, matching the build-time script's all-or-nothing contract.
|
||||
*/
|
||||
export const GEOLITE2_EDITIONS: readonly Geolite2Edition[] = [
|
||||
{ editionId: "GeoLite2-City", mmdb: "GeoLite2-City.mmdb" },
|
||||
{ editionId: "GeoLite2-ASN", mmdb: "GeoLite2-ASN.mmdb" },
|
||||
];
|
||||
|
||||
// A GeoLite2 tarball is ~40-60 MB; give the download well clear of the 15 s
|
||||
// safeFetch default so a slow self-host link does not abort a good fetch. The
|
||||
// signal covers body streaming too (undici aborts the read on the same signal).
|
||||
const DOWNLOAD_TIMEOUT_MS = 120_000;
|
||||
|
||||
export type Geolite2FetchStatus =
|
||||
| "skipped_disabled"
|
||||
| "skipped_no_key"
|
||||
| "skipped_present"
|
||||
| "fetched"
|
||||
| "failed";
|
||||
|
||||
export interface Geolite2FetchResult {
|
||||
status: Geolite2FetchStatus;
|
||||
/** Edition ids that were placed on a `"fetched"` result; empty otherwise. */
|
||||
editions: string[];
|
||||
/** Human-readable cause on a `"failed"` result. */
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the MaxMind permalink download URL for one edition. Exported so the
|
||||
* unit test can assert the exact wire URL per edition without a live request.
|
||||
*/
|
||||
export function buildMaxmindDownloadUrl(
|
||||
editionId: string,
|
||||
licenseKey: string,
|
||||
): string {
|
||||
const params = new URLSearchParams({
|
||||
edition_id: editionId,
|
||||
license_key: licenseKey,
|
||||
suffix: "tar.gz",
|
||||
});
|
||||
return `${MAXMIND_DOWNLOAD_BASE}?${params.toString()}`;
|
||||
}
|
||||
|
||||
function fail(message: string, cause: unknown): Geolite2FetchResult {
|
||||
const detail = cause instanceof Error ? cause.message : String(cause);
|
||||
getEvent()?.addWarning(`geolite2-fetch: ${message}: ${detail}`);
|
||||
return { status: "failed", editions: [], error: `${message}: ${detail}` };
|
||||
}
|
||||
|
||||
const TAR_BLOCK = 512;
|
||||
|
||||
/** Read a NUL-terminated ASCII field from a tar header block. */
|
||||
function readTarField(header: Buffer, offset: number, length: number): string {
|
||||
const raw = header.subarray(offset, offset + length);
|
||||
const end = raw.indexOf(0);
|
||||
return raw.toString("ascii", 0, end === -1 ? raw.length : end).trim();
|
||||
}
|
||||
|
||||
/** Parse a tar octal size field. GeoLite2 mmdbs fit octal, so no base-256 path. */
|
||||
function readTarOctal(header: Buffer, offset: number, length: number): number {
|
||||
const text = readTarField(header, offset, length);
|
||||
if (!text) return 0;
|
||||
const n = parseInt(text, 8);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract one `.mmdb` from a gzipped tar. The MaxMind tarball nests a single
|
||||
* database under a date-stamped directory (`GeoLite2-City_YYYYMMDD/…mmdb`), so
|
||||
* we walk the ustar entries and return the first regular file whose basename
|
||||
* matches. Non-file records (directories, PAX/GNU extension headers) never
|
||||
* match and are stepped over by their declared size, so the reader stays
|
||||
* correct against both GNU tar and bsdtar output. Returns null if absent.
|
||||
*/
|
||||
export function extractMmdbFromTarGz(
|
||||
gz: Buffer,
|
||||
mmdbBasename: string,
|
||||
): Buffer | null {
|
||||
const tar = zlib.gunzipSync(gz);
|
||||
let offset = 0;
|
||||
while (offset + TAR_BLOCK <= tar.length) {
|
||||
const header = tar.subarray(offset, offset + TAR_BLOCK);
|
||||
// Two all-zero blocks mark end-of-archive; one is enough to stop on.
|
||||
if (header.every((b) => b === 0)) break;
|
||||
|
||||
const name = readTarField(header, 0, 100);
|
||||
const size = readTarOctal(header, 124, 12);
|
||||
const typeflag = header[156];
|
||||
const dataStart = offset + TAR_BLOCK;
|
||||
|
||||
// Regular file: typeflag '0' (0x30) or the legacy NUL (0x00).
|
||||
const isFile = typeflag === 0x30 || typeflag === 0x00;
|
||||
if (isFile && name.split("/").pop() === mmdbBasename) {
|
||||
return Buffer.from(tar.subarray(dataStart, dataStart + size));
|
||||
}
|
||||
|
||||
// Advance past the data, padded up to the next 512-byte boundary.
|
||||
offset = dataStart + Math.ceil(size / TAR_BLOCK) * TAR_BLOCK;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download one edition and extract its `.mmdb` to a temp file in `destDir`.
|
||||
* Returns the temp path — the caller promotes it with an atomic rename once
|
||||
* every edition has been staged. Throws on any failure so the caller can
|
||||
* discard all partial temp files and leave the existing databases untouched.
|
||||
*/
|
||||
async function downloadAndExtractEdition(
|
||||
edition: Geolite2Edition,
|
||||
licenseKey: string,
|
||||
destDir: string,
|
||||
): Promise<string> {
|
||||
const url = buildMaxmindDownloadUrl(edition.editionId, licenseKey);
|
||||
// The URL carries `license_key=<secret>` in its query string. safeFetch
|
||||
// embeds the full URL in its thrown error message, and the wide-event
|
||||
// warning / job-failure paths do not redact it, so a bare re-throw would
|
||||
// print the licence key to stdout, Loki and GlitchTip on any timeout or
|
||||
// network error. Re-throw with the edition and the error KIND only, never
|
||||
// the URL.
|
||||
let res: Response;
|
||||
try {
|
||||
res = await safeFetch(
|
||||
url,
|
||||
{ headers: { Accept: "application/gzip" } },
|
||||
{
|
||||
// The permalink 302s to a signed CDN, so follow redirects; keep the
|
||||
// connect-time public-host pin so a rebinding cannot swing the hop onto
|
||||
// a private/metadata address.
|
||||
followRedirects: true,
|
||||
requirePublicHost: true,
|
||||
timeoutMs: DOWNLOAD_TIMEOUT_MS,
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
const kind = err instanceof SafeFetchError ? err.kind : "network";
|
||||
throw new Error(`${edition.editionId} download failed (${kind})`);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// 401 = wrong key / unsigned EULA, 403 = throttle, 5xx = MaxMind-side.
|
||||
throw new Error(
|
||||
`${edition.editionId} download returned HTTP ${res.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
const gz = Buffer.from(await res.arrayBuffer());
|
||||
const mmdb = extractMmdbFromTarGz(gz, edition.mmdb);
|
||||
if (!mmdb) {
|
||||
throw new Error(
|
||||
`${edition.mmdb} not found inside the ${edition.editionId} tarball`,
|
||||
);
|
||||
}
|
||||
if (mmdb.length === 0) {
|
||||
throw new Error(`${edition.mmdb} extracted empty`);
|
||||
}
|
||||
|
||||
// Stage under destDir (same filesystem) so the promote is an atomic rename.
|
||||
// A `.tmp-` suffix keeps it out of the resolver's eye — `offlineGeoReady()`
|
||||
// only accepts the exact `GeoLite2-City.mmdb`.
|
||||
const tmpDest = path.join(
|
||||
destDir,
|
||||
`${edition.mmdb}.tmp-${process.pid}-${Date.now()}`,
|
||||
);
|
||||
await fsp.writeFile(tmpDest, mmdb);
|
||||
return tmpDest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch both GeoLite2 databases and place them in `geoLiteDir()`.
|
||||
*
|
||||
* - `IP_GEO_LOOKUP_DISABLED=1` → `skipped_disabled` (the whole tier is off).
|
||||
* - No `MAXMIND_LICENSE_KEY` → `skipped_no_key`.
|
||||
* - `skipIfPresent` and the offline tier already ready → `skipped_present`
|
||||
* (the boot path: only fetch when there is no database yet).
|
||||
*
|
||||
* On success the reader cache is reset so a running worker reads the new
|
||||
* databases without a restart, and any stale keyless-build `.empty` marker is
|
||||
* removed. Never throws.
|
||||
*/
|
||||
export async function fetchGeoLite2Databases(
|
||||
opts: { skipIfPresent?: boolean } = {},
|
||||
): Promise<Geolite2FetchResult> {
|
||||
if (process.env.IP_GEO_LOOKUP_DISABLED === "1") {
|
||||
return { status: "skipped_disabled", editions: [] };
|
||||
}
|
||||
const licenseKey = process.env.MAXMIND_LICENSE_KEY?.trim();
|
||||
if (!licenseKey) {
|
||||
return { status: "skipped_no_key", editions: [] };
|
||||
}
|
||||
if (opts.skipIfPresent && offlineGeoReady()) {
|
||||
return { status: "skipped_present", editions: [] };
|
||||
}
|
||||
|
||||
const dir = geoLiteDir();
|
||||
try {
|
||||
await fsp.mkdir(dir, { recursive: true });
|
||||
} catch (err) {
|
||||
return fail(`geolite2 directory not writable (${dir})`, err);
|
||||
}
|
||||
|
||||
const staged: { mmdb: string; tmp: string }[] = [];
|
||||
try {
|
||||
for (const edition of GEOLITE2_EDITIONS) {
|
||||
const tmp = await downloadAndExtractEdition(edition, licenseKey, dir);
|
||||
staged.push({ mmdb: edition.mmdb, tmp });
|
||||
}
|
||||
} catch (err) {
|
||||
// Fail-soft: drop every partial temp file, leave the existing databases.
|
||||
await Promise.all(
|
||||
staged.map((s) => fsp.rm(s.tmp, { force: true }).catch(() => {})),
|
||||
);
|
||||
return fail("geolite2 download/extract failed", err);
|
||||
}
|
||||
|
||||
// Every edition is staged; promote them all with atomic renames.
|
||||
try {
|
||||
for (const s of staged) {
|
||||
await fsp.rename(s.tmp, path.join(dir, s.mmdb));
|
||||
}
|
||||
// A freshly populated directory must not keep a stale keyless-build marker
|
||||
// (the resolver reads `.empty` as "offline tier deliberately absent").
|
||||
await fsp.rm(path.join(dir, ".empty"), { force: true }).catch(() => {});
|
||||
} catch (err) {
|
||||
await Promise.all(
|
||||
staged.map((s) => fsp.rm(s.tmp, { force: true }).catch(() => {})),
|
||||
);
|
||||
return fail("geolite2 promote failed", err);
|
||||
}
|
||||
|
||||
// The running worker latched its readers before the files existed; reset so
|
||||
// the next lookup reads them.
|
||||
resetGeoLite2ReaderCache();
|
||||
|
||||
return {
|
||||
status: "fetched",
|
||||
editions: GEOLITE2_EDITIONS.map((e) => e.editionId),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* Runtime GeoLite2 fetch (issue #659) — queue-wiring guard + boot gating.
|
||||
*
|
||||
* The source-text-grep half mirrors the other dead-queue guards: assert the
|
||||
* queue is imported, in `allQueues`, scheduled, and bound to a `createAndWork`
|
||||
* handler that runs the fetch — without booting pg-boss. An unregistered queue
|
||||
* silently never drains (the recurring v1.4.37 dead-queue bug), which here
|
||||
* would mean a keyed self-host never gets its databases fetched.
|
||||
*
|
||||
* The second half exercises the boot-discovery gate directly: it must send a
|
||||
* job only when a key is set, egress is not disabled, and the offline tier is
|
||||
* absent.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const REGISTRAR_PATH = join(
|
||||
__dirname,
|
||||
"..",
|
||||
"reminder",
|
||||
"register-maintenance.ts",
|
||||
);
|
||||
const workerSource =
|
||||
readFileSync(REGISTRAR_PATH, "utf8") +
|
||||
readFileSync(join(__dirname, "..", "reminder", "ops-handlers.ts"), "utf8");
|
||||
|
||||
describe("reminder-worker — geolite2-fetch wiring", () => {
|
||||
it("imports the queue symbols from the geolite2-fetch module", () => {
|
||||
expect(workerSource).toMatch(/from\s*["']@\/lib\/jobs\/geolite2-fetch["']/);
|
||||
expect(workerSource).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/);
|
||||
expect(workerSource).toMatch(/\bGEOLITE2_FETCH_CRON\b/);
|
||||
expect(workerSource).toMatch(/\benqueueGeolite2FetchBootDiscovery\b/);
|
||||
});
|
||||
|
||||
it("registers the queue in the allQueues loop", () => {
|
||||
const allQueuesMatch = workerSource.match(
|
||||
/const allQueues\s*=\s*\[([\s\S]*?)\];/,
|
||||
);
|
||||
expect(allQueuesMatch).not.toBeNull();
|
||||
expect(allQueuesMatch![1]).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/);
|
||||
});
|
||||
|
||||
it("schedules the monthly cron", () => {
|
||||
expect(workerSource).toMatch(
|
||||
/\[GEOLITE2_FETCH_QUEUE,\s*GEOLITE2_FETCH_CRON\]/,
|
||||
);
|
||||
});
|
||||
|
||||
it("registers a createAndWork handler for the queue", () => {
|
||||
expect(workerSource).toMatch(
|
||||
/createAndWork[\s\S]{0,200}GEOLITE2_FETCH_QUEUE[\s\S]{0,200}handleGeolite2Fetch/,
|
||||
);
|
||||
});
|
||||
|
||||
it("runs the fetch inside the handler", () => {
|
||||
expect(workerSource).toMatch(
|
||||
/handleGeolite2Fetch[\s\S]{0,500}fetchGeoLite2Databases/,
|
||||
);
|
||||
});
|
||||
|
||||
it("fires the boot discovery from enqueueMaintenanceBootDiscovery", () => {
|
||||
expect(workerSource).toMatch(
|
||||
/enqueueMaintenanceBootDiscovery[\s\S]*enqueueGeolite2FetchBootDiscovery/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Boot-discovery gate ──────────────────────────────────────────────
|
||||
const { sendMock, bossMock, offlineGeoReadyMock } = vi.hoisted(() => {
|
||||
const sendMock = vi.fn().mockResolvedValue("job-1");
|
||||
return {
|
||||
sendMock,
|
||||
bossMock: { send: sendMock },
|
||||
offlineGeoReadyMock: vi.fn(),
|
||||
};
|
||||
});
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({
|
||||
getGlobalBoss: () => bossMock,
|
||||
}));
|
||||
vi.mock("@/lib/geo", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/geo")>();
|
||||
return { ...actual, offlineGeoReady: offlineGeoReadyMock };
|
||||
});
|
||||
|
||||
import { enqueueGeolite2FetchBootDiscovery } from "@/lib/jobs/geolite2-fetch";
|
||||
|
||||
const ORIGINAL_KEY = process.env.MAXMIND_LICENSE_KEY;
|
||||
const ORIGINAL_DISABLED = process.env.IP_GEO_LOOKUP_DISABLED;
|
||||
|
||||
describe("enqueueGeolite2FetchBootDiscovery — gating", () => {
|
||||
beforeEach(() => {
|
||||
sendMock.mockClear();
|
||||
sendMock.mockResolvedValue("job-1");
|
||||
offlineGeoReadyMock.mockReturnValue(false);
|
||||
process.env.MAXMIND_LICENSE_KEY = "test-key";
|
||||
delete process.env.IP_GEO_LOOKUP_DISABLED;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (ORIGINAL_KEY === undefined) delete process.env.MAXMIND_LICENSE_KEY;
|
||||
else process.env.MAXMIND_LICENSE_KEY = ORIGINAL_KEY;
|
||||
if (ORIGINAL_DISABLED === undefined)
|
||||
delete process.env.IP_GEO_LOOKUP_DISABLED;
|
||||
else process.env.IP_GEO_LOOKUP_DISABLED = ORIGINAL_DISABLED;
|
||||
});
|
||||
|
||||
it("enqueues a boot-mode fetch when keyed and no database is present", async () => {
|
||||
const { enqueued } = await enqueueGeolite2FetchBootDiscovery();
|
||||
expect(enqueued).toBe(1);
|
||||
expect(sendMock).toHaveBeenCalledTimes(1);
|
||||
const [queue, payload] = sendMock.mock.calls[0];
|
||||
expect(queue).toBe("geolite2-fetch");
|
||||
expect(payload).toMatchObject({ mode: "boot" });
|
||||
});
|
||||
|
||||
it("skips when no MAXMIND_LICENSE_KEY is set", async () => {
|
||||
delete process.env.MAXMIND_LICENSE_KEY;
|
||||
const { enqueued } = await enqueueGeolite2FetchBootDiscovery();
|
||||
expect(enqueued).toBe(0);
|
||||
expect(sendMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips when IP_GEO_LOOKUP_DISABLED=1", async () => {
|
||||
process.env.IP_GEO_LOOKUP_DISABLED = "1";
|
||||
const { enqueued } = await enqueueGeolite2FetchBootDiscovery();
|
||||
expect(enqueued).toBe(0);
|
||||
expect(sendMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips when the offline tier is already present", async () => {
|
||||
offlineGeoReadyMock.mockReturnValue(true);
|
||||
const { enqueued } = await enqueueGeolite2FetchBootDiscovery();
|
||||
expect(enqueued).toBe(0);
|
||||
expect(sendMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Runtime GeoLite2 fetch — pg-boss queue wiring (issue #659).
|
||||
*
|
||||
* The actual download + extract + place logic lives in
|
||||
* `src/lib/geo/geolite2-fetch.ts`; this module owns the queue name, the cron,
|
||||
* and the boot-discovery enqueue, mirroring the geo-backfill split
|
||||
* (`geo-backfill.ts` holds `runGeoBackfill`, `ops-handlers.ts` holds the
|
||||
* `boss.work` handler). Worker-only by construction — nothing here runs in the
|
||||
* web request path.
|
||||
*
|
||||
* Two triggers:
|
||||
*
|
||||
* - BOOT: `enqueueGeolite2FetchBootDiscovery()` fires once at worker boot,
|
||||
* only when a key is set and the offline tier is not already present. The
|
||||
* first-run acquisition — pull `:latest`, set the key, restart, the
|
||||
* databases appear without a manual mount.
|
||||
* - CRON: monthly. MaxMind reissues GeoLite2 on the first Tuesday of each
|
||||
* month; the 8th is safely past that, so a monthly refresh keeps a keyed
|
||||
* host current. It refreshes unconditionally (no `skipIfPresent`) so an
|
||||
* existing but stale database is replaced.
|
||||
*/
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { offlineGeoReady } from "@/lib/geo";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
export const GEOLITE2_FETCH_QUEUE = "geolite2-fetch";
|
||||
|
||||
// Monthly on the 8th at 04:00 Europe/Berlin — past the first-Tuesday reissue
|
||||
// window, inside the existing 02:xx-04:xx maintenance band. NOT `cronIsTheRetry`:
|
||||
// a monthly cadence is far too long to be the retry, so the queue keeps
|
||||
// pg-boss's default retries for a transient MaxMind hiccup.
|
||||
export const GEOLITE2_FETCH_CRON = "0 4 8 * *";
|
||||
|
||||
export interface Geolite2FetchPayload {
|
||||
triggeredAt?: string;
|
||||
/** `"boot"` fetches only when no database is present yet; `"refresh"` always fetches. */
|
||||
mode?: "boot" | "refresh";
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire-and-forget boot discovery. Enqueues a single boot-mode fetch only when a
|
||||
* key is configured, egress is not disabled, and the offline tier is absent —
|
||||
* so a host that already has the databases (baked, mounted, or fetched on a
|
||||
* prior boot) enqueues nothing. The handler re-checks all three; gating here
|
||||
* just avoids a no-op job on the common keyless boot.
|
||||
*/
|
||||
export async function enqueueGeolite2FetchBootDiscovery(): Promise<{
|
||||
enqueued: number;
|
||||
}> {
|
||||
const boss = getGlobalBoss();
|
||||
if (!boss) return { enqueued: 0 };
|
||||
if (!process.env.MAXMIND_LICENSE_KEY?.trim()) return { enqueued: 0 };
|
||||
if (process.env.IP_GEO_LOOKUP_DISABLED === "1") return { enqueued: 0 };
|
||||
if (offlineGeoReady()) return { enqueued: 0 };
|
||||
|
||||
try {
|
||||
const payload: Geolite2FetchPayload = {
|
||||
mode: "boot",
|
||||
triggeredAt: new Date().toISOString(),
|
||||
};
|
||||
const jobId = await boss.send(GEOLITE2_FETCH_QUEUE, payload, {
|
||||
retryLimit: 3,
|
||||
retryDelay: 60,
|
||||
retryBackoff: true,
|
||||
singletonKey: "geolite2-fetch|boot",
|
||||
});
|
||||
const enqueued = jobId ? 1 : 0;
|
||||
annotate({
|
||||
action: { name: "geo.database.fetch.boot" },
|
||||
meta: { enqueued },
|
||||
});
|
||||
return { enqueued };
|
||||
} catch {
|
||||
// A transient send failure is a no-op — the monthly cron and the next boot
|
||||
// both re-attempt.
|
||||
return { enqueued: 0 };
|
||||
}
|
||||
}
|
||||
@@ -96,6 +96,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"geo_backfill_scanned",
|
||||
"geo_backfill_skipped",
|
||||
"geo_backfill_still_unresolved",
|
||||
"geolite2_fetch_status",
|
||||
"host_metric_pruned",
|
||||
"hourly_rows_upserted",
|
||||
"imported",
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
*/
|
||||
import { type Job } from "pg-boss";
|
||||
import { runGeoBackfill } from "@/lib/jobs/geo-backfill";
|
||||
import { fetchGeoLite2Databases } from "@/lib/geo/geolite2-fetch";
|
||||
import { type Geolite2FetchPayload } from "@/lib/jobs/geolite2-fetch";
|
||||
import { runTlsPinMonitor } from "@/lib/jobs/tls-pin-monitor";
|
||||
import { type PrDetectionPayload } from "@/lib/jobs/pr-detection";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
@@ -138,6 +140,52 @@ export async function handleGeoBackfill(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runtime GeoLite2 fetch worker (issue #659). Downloads + places the offline
|
||||
* MMDBs when a `MAXMIND_LICENSE_KEY` is set, so a self-hoster on the published
|
||||
* image gets the offline geo tier without a rebuild or a manual mount. The
|
||||
* fetch is fail-soft (a failure leaves the previous databases untouched); the
|
||||
* boot mode only fetches when no database is present yet, the cron mode always
|
||||
* refreshes. Skips (no key, egress disabled, already present) complete the job;
|
||||
* a genuine download/extract failure fails it so pg-boss's default retries get
|
||||
* a few attempts before the monthly cron catches the next window.
|
||||
*/
|
||||
export async function handleGeolite2Fetch(
|
||||
jobs: Job<Geolite2FetchPayload>[],
|
||||
): Promise<JobOutcome> {
|
||||
return withBackgroundEvent("job.geolite2_fetch", async (evt) => {
|
||||
// Batched worker mode always hands us an array; the queue is single-flight
|
||||
// in practice, so this loop is ~always one job.
|
||||
let lastStatus = "none";
|
||||
let failure: JobOutcome | null = null;
|
||||
for (const job of jobs) {
|
||||
const mode = job.data?.mode === "boot" ? "boot" : "refresh";
|
||||
const result = await fetchGeoLite2Databases({
|
||||
skipIfPresent: mode === "boot",
|
||||
});
|
||||
lastStatus = result.status;
|
||||
evt.addMeta("geolite2_fetch_mode", mode);
|
||||
evt.addMeta("geolite2_fetch_status", result.status);
|
||||
if (result.status === "fetched") {
|
||||
evt.addMeta("geolite2_fetch_editions", result.editions.join(","));
|
||||
}
|
||||
if (result.status === "failed" && failure === null) {
|
||||
evt.addWarning(
|
||||
`geolite2-fetch failed: ${result.error ?? "unknown error"}`,
|
||||
);
|
||||
failure = jobFailed(
|
||||
"geolite2 fetch failed",
|
||||
new Error(result.error ?? "unknown error"),
|
||||
);
|
||||
}
|
||||
}
|
||||
return (
|
||||
failure ??
|
||||
jobDone({ geolite2_fetch_status: lastStatus, jobs: jobs.length })
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function handleTlsPinMonitor(
|
||||
jobs: Job<TlsPinMonitorPayload>[],
|
||||
): Promise<JobOutcome> {
|
||||
|
||||
@@ -17,6 +17,12 @@
|
||||
*/
|
||||
import { PgBoss } from "pg-boss";
|
||||
import { GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON } from "@/lib/jobs/geo-backfill";
|
||||
import {
|
||||
GEOLITE2_FETCH_QUEUE,
|
||||
GEOLITE2_FETCH_CRON,
|
||||
enqueueGeolite2FetchBootDiscovery,
|
||||
type Geolite2FetchPayload,
|
||||
} from "@/lib/jobs/geolite2-fetch";
|
||||
import {
|
||||
TLS_PIN_MONITOR_QUEUE,
|
||||
TLS_PIN_MONITOR_CRON,
|
||||
@@ -176,6 +182,7 @@ import {
|
||||
handleHostMetricSample,
|
||||
handleFeedbackAggregator,
|
||||
handleGeoBackfill,
|
||||
handleGeolite2Fetch,
|
||||
handleTlsPinMonitor,
|
||||
handlePrDetection,
|
||||
} from "./ops-handlers";
|
||||
@@ -335,6 +342,12 @@ const allQueues = [
|
||||
HOST_METRIC_QUEUE,
|
||||
FEEDBACK_AGGREGATOR_QUEUE,
|
||||
GEO_BACKFILL_QUEUE,
|
||||
// v1.37.7 (issue #659) — runtime GeoLite2 fetch. A self-hoster who sets
|
||||
// MAXMIND_LICENSE_KEY on the published image gets the offline geo tier
|
||||
// without a rebuild or a manual mount: boot discovery fetches when no
|
||||
// database is present, a monthly cron keeps a keyed host current. Without
|
||||
// this entry pg-boss never provisions the queue and both silently no-op.
|
||||
GEOLITE2_FETCH_QUEUE,
|
||||
// v1.12.2 ios-coord — TLS leaf SPKI-change monitor. The iOS client
|
||||
// pins the served leaf certificate; this queue probes it every 6 h and
|
||||
// alarms when the served SPKI leaves the operator's known-good set.
|
||||
@@ -472,6 +485,11 @@ const schedules: ScheduleEntry[] = [
|
||||
// long tail of audit rows that landed with the offline MMDB
|
||||
// missing or the online provider unreachable.
|
||||
[GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry],
|
||||
// v1.37.7 (issue #659) — monthly runtime GeoLite2 refresh (8th, 04:00
|
||||
// Europe/Berlin, past MaxMind's first-Tuesday reissue). Deliberately NOT
|
||||
// cronIsTheRetry: a month between ticks is far too long to be the retry, so
|
||||
// the queue keeps pg-boss's default retries for a transient MaxMind hiccup.
|
||||
[GEOLITE2_FETCH_QUEUE, GEOLITE2_FETCH_CRON],
|
||||
// v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the
|
||||
// hourly sync crons). Surfaces a pinned-leaf rotation well inside the
|
||||
// ≥11-day re-pin window the iOS release owner needs.
|
||||
@@ -715,6 +733,14 @@ export async function registerMaintenanceQueues(
|
||||
{ localConcurrency: 1 },
|
||||
handleGeoBackfill,
|
||||
);
|
||||
// v1.37.7 (issue #659) — runtime GeoLite2 fetch. Single-flight: one download
|
||||
// at a time, and two ticks racing the same atomic place is wasted bandwidth.
|
||||
await createAndWork<Geolite2FetchPayload>(
|
||||
boss,
|
||||
GEOLITE2_FETCH_QUEUE,
|
||||
{ localConcurrency: 1 },
|
||||
handleGeolite2Fetch,
|
||||
);
|
||||
// v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one
|
||||
// short outbound TLS handshake per tick, no benefit to overlapping ticks.
|
||||
await createAndWork<TlsPinMonitorPayload>(
|
||||
@@ -1475,4 +1501,20 @@ export async function enqueueMaintenanceBootDiscovery(): Promise<void> {
|
||||
err,
|
||||
);
|
||||
}
|
||||
|
||||
// v1.37.7 (issue #659) — runtime GeoLite2 fetch. Only enqueues when a
|
||||
// MAXMIND_LICENSE_KEY is set, egress is not disabled, and the offline tier
|
||||
// is not already present — so a self-hoster who sets the key on the
|
||||
// published image gets the databases fetched on the next worker boot without
|
||||
// a rebuild or a manual mount.
|
||||
try {
|
||||
const { enqueued } = await enqueueGeolite2FetchBootDiscovery();
|
||||
workerLog("info", `[geolite2-fetch] boot discovery: enqueued=${enqueued}`);
|
||||
} catch (err) {
|
||||
workerLog(
|
||||
"error",
|
||||
"[geolite2-fetch] boot discovery threw an unexpected error",
|
||||
err,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,7 @@ const subsystemSurface = {
|
||||
"host-metric-sample": { audience: "system" },
|
||||
"feedback-aggregator": { audience: "system" },
|
||||
"geo-backfill": { audience: "system" },
|
||||
"geolite2-fetch": { audience: "system" },
|
||||
"tls-pin-monitor": { audience: "system" },
|
||||
"pr-detection": { audience: "system" },
|
||||
"medication-inventory-expire": { audience: "system" },
|
||||
|
||||
@@ -110,7 +110,7 @@ export function redactSecrets(input: string): string {
|
||||
// `passphrase` that derives the encrypted-export key must never land in
|
||||
// `http.path` / error strings reaching Loki / Glitchtip.
|
||||
.replace(
|
||||
/([?&])(secret|code|token|ticket|api[_-]?key|insurance(?:number)?|kvnr|totp|mfa|backup[_-]?code|recovery[_-]?code|passphrase)=[^&\s]+/gi,
|
||||
/([?&])(secret|code|token|ticket|api[_-]?key|license[_-]?key|insurance(?:number)?|kvnr|totp|mfa|backup[_-]?code|recovery[_-]?code|passphrase)=[^&\s]+/gi,
|
||||
"$1$2=[REDACTED]",
|
||||
)
|
||||
);
|
||||
|
||||
Reference in new issue
Block a user