fix(jobs): defer the dense intra-day retention boot drain past startup to avoid pool exhaustion

This commit is contained in:
Marc-André Bombeck committed 2026-06-03 02:22:35 +02:00
1 parent 1fe70b27f5
commit 3080828fb7
5 files changed
+43 -2

No files matched your search

+6
View File
@@ -1,5 +1,11 @@
# Changelog
## [1.10.0.1] — 2026-06-03 — Defer the intra-day retention drain past startup
### Fixed
- **The app no longer turns unresponsive shortly after a deploy or restart.** The new intra-day retention drain ran its catch-up pass the moment the background worker started, at the same time as the database migration, the other start-up backfills, and the first health checks — all sharing one connection pool. On an instance with a lot of history this exhausted the pool, so the app stopped answering until it restarted, then repeated. The catch-up now waits until start-up has settled before it runs; it is unchanged otherwise and the nightly pass is unaffected.
## [1.10.0] — 2026-06-03 — Derived wellness metrics, device-event awareness, deeper ingestion
### Added
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: 3.1.0
info:
title: HealthLog API
version: 1.10.0
version: 1.10.0.1
description: >-
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "healthlog",
"version": "1.10.0",
"version": "1.10.0.1",
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
"license": "AGPL-3.0-only",
"homepage": "https://healthlog.dev",
@@ -17,6 +17,9 @@ import { describe, expect, it } from "vitest";
const REMINDER_WORKER_PATH = join(__dirname, "..", "reminder-worker.ts");
const source = readFileSync(REMINDER_WORKER_PATH, "utf8");
const DENSE_RETENTION_PATH = join(__dirname, "..", "dense-intraday-retention.ts");
const denseRetentionSource = readFileSync(DENSE_RETENTION_PATH, "utf8");
function allQueuesBlock(): string {
const m = source.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/);
expect(m).not.toBeNull();
@@ -93,6 +96,20 @@ describe("reminder-worker — dense intra-day retention wiring", () => {
expect(source).toMatch(/await enqueueBootTimeDenseIntradayRetention\(\)/);
});
it("defers the boot-discovery drain past the startup storm (P2028 guard)", () => {
// The boot-time per-user retention jobs MUST carry a startAfter delay so
// the transaction-per-day drain never contends with the deploy storm +
// migration + other boot backfills + foreground health-checks for the
// shared connection pool. Dropping this re-opens the v1.10.0 pool-
// exhaustion (P2028) restart loop on a data-heavy tenant.
expect(denseRetentionSource).toMatch(
/DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS\s*=\s*\d+/,
);
expect(denseRetentionSource).toMatch(
/boss\.send\([\s\S]{0,400}startAfter:\s*DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS/,
);
});
it("folds the nightly dense-retention walk onto the drain-cumulative tick", () => {
// The steady-state nightly walk runs the global (no-userId) pass on the
// same tick as the cumulative + mean drains.
+18
View File
@@ -46,6 +46,21 @@ export const DENSE_INTRADAY_RETENTION_CONCURRENCY = 1;
*/
export const DENSE_INTRADAY_RETENTION_CRON = "50 3 * * *";
/**
* Boot-discovery jobs start only after this delay (seconds) so the
* catch-up drain never competes with the startup storm — the deploy
* container recreate, the Prisma migration, the other boot-time backfills
* (rollup / mean- + step-consolidation / mood / medication-compliance),
* and the foreground health-checks all share one connection pool. Running
* the transaction-per-day drain into that contention exhausted the pool on
* a data-heavy tenant (`P2028: Unable to start a transaction in the given
* time`), starving `/api/health` into a restart loop. Deferring past the
* boot window restores the pre-v1.10 boot profile; the scheduled 03:50
* cron is unaffected, and the per-user jobs still drain serially
* (concurrency 1) once the delay elapses.
*/
export const DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS = 600;
export interface DenseIntradayRetentionPayload {
userId: string;
enqueuedAt: string;
@@ -144,6 +159,9 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{
retryLimit: 3,
retryDelay: 60,
retryBackoff: true,
// Defer past the boot storm so the drain never contends with
// startup + foreground for the shared connection pool.
startAfter: DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS,
singletonKey: `dense-intraday-retention|${userId}`,
});
if (jobId) {