mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
fix(jobs): defer the dense intra-day retention boot drain past startup to avoid pool exhaustion
This commit is contained in:
1 parent
1fe70b27f5
commit
3080828fb7
5 files changed
+43
-2
No files matched your search
@@ -1,5 +1,11 @@
|
||||
# Changelog
|
||||
|
||||
## [1.10.0.1] — 2026-06-03 — Defer the intra-day retention drain past startup
|
||||
|
||||
### Fixed
|
||||
|
||||
- **The app no longer turns unresponsive shortly after a deploy or restart.** The new intra-day retention drain ran its catch-up pass the moment the background worker started, at the same time as the database migration, the other start-up backfills, and the first health checks — all sharing one connection pool. On an instance with a lot of history this exhausted the pool, so the app stopped answering until it restarted, then repeated. The catch-up now waits until start-up has settled before it runs; it is unchanged otherwise and the nightly pass is unaffected.
|
||||
|
||||
## [1.10.0] — 2026-06-03 — Derived wellness metrics, device-event awareness, deeper ingestion
|
||||
|
||||
### Added
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: HealthLog API
|
||||
version: 1.10.0
|
||||
version: 1.10.0.1
|
||||
description: >-
|
||||
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "healthlog",
|
||||
"version": "1.10.0",
|
||||
"version": "1.10.0.1",
|
||||
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
|
||||
"license": "AGPL-3.0-only",
|
||||
"homepage": "https://healthlog.dev",
|
||||
|
||||
@@ -17,6 +17,9 @@ import { describe, expect, it } from "vitest";
|
||||
const REMINDER_WORKER_PATH = join(__dirname, "..", "reminder-worker.ts");
|
||||
const source = readFileSync(REMINDER_WORKER_PATH, "utf8");
|
||||
|
||||
const DENSE_RETENTION_PATH = join(__dirname, "..", "dense-intraday-retention.ts");
|
||||
const denseRetentionSource = readFileSync(DENSE_RETENTION_PATH, "utf8");
|
||||
|
||||
function allQueuesBlock(): string {
|
||||
const m = source.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/);
|
||||
expect(m).not.toBeNull();
|
||||
@@ -93,6 +96,20 @@ describe("reminder-worker — dense intra-day retention wiring", () => {
|
||||
expect(source).toMatch(/await enqueueBootTimeDenseIntradayRetention\(\)/);
|
||||
});
|
||||
|
||||
it("defers the boot-discovery drain past the startup storm (P2028 guard)", () => {
|
||||
// The boot-time per-user retention jobs MUST carry a startAfter delay so
|
||||
// the transaction-per-day drain never contends with the deploy storm +
|
||||
// migration + other boot backfills + foreground health-checks for the
|
||||
// shared connection pool. Dropping this re-opens the v1.10.0 pool-
|
||||
// exhaustion (P2028) restart loop on a data-heavy tenant.
|
||||
expect(denseRetentionSource).toMatch(
|
||||
/DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS\s*=\s*\d+/,
|
||||
);
|
||||
expect(denseRetentionSource).toMatch(
|
||||
/boss\.send\([\s\S]{0,400}startAfter:\s*DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS/,
|
||||
);
|
||||
});
|
||||
|
||||
it("folds the nightly dense-retention walk onto the drain-cumulative tick", () => {
|
||||
// The steady-state nightly walk runs the global (no-userId) pass on the
|
||||
// same tick as the cumulative + mean drains.
|
||||
|
||||
@@ -46,6 +46,21 @@ export const DENSE_INTRADAY_RETENTION_CONCURRENCY = 1;
|
||||
*/
|
||||
export const DENSE_INTRADAY_RETENTION_CRON = "50 3 * * *";
|
||||
|
||||
/**
|
||||
* Boot-discovery jobs start only after this delay (seconds) so the
|
||||
* catch-up drain never competes with the startup storm — the deploy
|
||||
* container recreate, the Prisma migration, the other boot-time backfills
|
||||
* (rollup / mean- + step-consolidation / mood / medication-compliance),
|
||||
* and the foreground health-checks all share one connection pool. Running
|
||||
* the transaction-per-day drain into that contention exhausted the pool on
|
||||
* a data-heavy tenant (`P2028: Unable to start a transaction in the given
|
||||
* time`), starving `/api/health` into a restart loop. Deferring past the
|
||||
* boot window restores the pre-v1.10 boot profile; the scheduled 03:50
|
||||
* cron is unaffected, and the per-user jobs still drain serially
|
||||
* (concurrency 1) once the delay elapses.
|
||||
*/
|
||||
export const DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS = 600;
|
||||
|
||||
export interface DenseIntradayRetentionPayload {
|
||||
userId: string;
|
||||
enqueuedAt: string;
|
||||
@@ -144,6 +159,9 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{
|
||||
retryLimit: 3,
|
||||
retryDelay: 60,
|
||||
retryBackoff: true,
|
||||
// Defer past the boot storm so the drain never contends with
|
||||
// startup + foreground for the shared connection pool.
|
||||
startAfter: DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS,
|
||||
singletonKey: `dense-intraday-retention|${userId}`,
|
||||
});
|
||||
if (jobId) {
|
||||
|
||||
Reference in new issue
Block a user