mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 16:12:11 +00:00
fix(ai): write the provider health ledger on the app's clock
The ledger stamped last_failure_at, last_ok_at, next_retry_at and updated_at with the database's NOW(), while every reader compares them with the app's clock: the runner's skip hints and the background bench, the admin provider-health card, the server-health tri-state and the support report. A backoff or pause therefore sat wherever the skew between the two clocks put it. With the app clock pinned, a five-minute backoff lasted until the database's clock plus five minutes, months away, or was over before it began. Both writes now bind the instant they took as a parameter (`$N::timestamp`, the naive UTC columns), the same one-clock rule the summary reads follow. The escalating pause arithmetic is unchanged. The pause integration test pins the app clock years behind and ahead of the database's and checks the stored instants, a backoff that lifts after five minutes, a pause that benches background work for its full length, and a success the server health reads as recent. Refs #1279
This commit is contained in:
1 parent
dbe05c958c
commit
595a883440
2 files changed
+110
-9
No files matched your search
@@ -149,6 +149,16 @@ export function classifyFailure(httpStatus: number | null): {
|
||||
* `provider_health` table. Writes use an atomic SQL upsert (the
|
||||
* rate-limiter pattern) so concurrent workers never clobber each other's
|
||||
* counters.
|
||||
*
|
||||
* One clock: the app's. Every instant a write stores is the `Date` the
|
||||
* write took, bound as a parameter, never the database's `NOW()`, because
|
||||
* every reader (`getSkipHints`, the background bench, the admin card, the
|
||||
* server-health tri-state, the support report) compares those instants
|
||||
* with the app's clock. Two clocks put a window wherever their skew put
|
||||
* it: a process whose clock is pinned saw a five-minute backoff last until
|
||||
* the database's clock plus five minutes, months away or already over.
|
||||
* The columns are naive UTC timestamps, so the bound instant is cast with
|
||||
* `::timestamp` (see `request-clock-sql.ts`).
|
||||
*/
|
||||
export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
async getSkipHints(userId) {
|
||||
@@ -196,6 +206,7 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
async recordSuccess(userId, providerType) {
|
||||
try {
|
||||
// Atomic upsert; a success always clears the negative cache.
|
||||
const now = new Date();
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO provider_health
|
||||
(id, user_id, provider_type, last_result, last_status,
|
||||
@@ -203,14 +214,14 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
next_retry_at, updated_at)
|
||||
VALUES
|
||||
(gen_random_uuid()::text, ${userId}, ${providerType}, 'ok', NULL,
|
||||
0, (NOW() AT TIME ZONE 'UTC'), NULL, NULL, (NOW() AT TIME ZONE 'UTC'))
|
||||
0, ${now}::timestamp, NULL, NULL, ${now}::timestamp)
|
||||
ON CONFLICT (user_id, provider_type) DO UPDATE SET
|
||||
last_result = 'ok',
|
||||
last_status = NULL,
|
||||
consecutive_failures = 0,
|
||||
last_ok_at = (NOW() AT TIME ZONE 'UTC'),
|
||||
last_ok_at = ${now}::timestamp,
|
||||
next_retry_at = NULL,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
updated_at = ${now}::timestamp
|
||||
`;
|
||||
} catch {
|
||||
// Fire-and-forget — a failed write never blocks the hot path.
|
||||
@@ -228,10 +239,12 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
const pauseFrom = PROVIDER_PAUSE_THRESHOLD;
|
||||
const pauseBase = `${PROVIDER_PAUSE_BASE_MS} milliseconds`;
|
||||
const pauseMax = `${PROVIDER_PAUSE_MAX_MS} milliseconds`;
|
||||
const now = new Date();
|
||||
try {
|
||||
// Atomic upsert. `consecutive_failures` accumulates across workers;
|
||||
// an auth failure forces the cooldown regardless of count, a hard
|
||||
// failure extends it. The cooldown anchors on NOW() so a fresh failure
|
||||
// failure extends it. The cooldown anchors on the failure's own
|
||||
// instant (the app's clock, like every reader) so a fresh failure
|
||||
// always re-arms the skip window.
|
||||
await prisma.$executeRaw`
|
||||
INSERT INTO provider_health
|
||||
@@ -240,13 +253,13 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
next_retry_at, updated_at)
|
||||
VALUES
|
||||
(gen_random_uuid()::text, ${userId}, ${providerType}, ${result},
|
||||
${status}, 1, NULL, (NOW() AT TIME ZONE 'UTC'), (NOW() AT TIME ZONE 'UTC') + ${interval}::interval, (NOW() AT TIME ZONE 'UTC'))
|
||||
${status}, 1, NULL, ${now}::timestamp, ${now}::timestamp + ${interval}::interval, ${now}::timestamp)
|
||||
ON CONFLICT (user_id, provider_type) DO UPDATE SET
|
||||
last_result = ${result},
|
||||
last_status = ${status},
|
||||
consecutive_failures = provider_health.consecutive_failures + 1,
|
||||
last_failure_at = (NOW() AT TIME ZONE 'UTC'),
|
||||
next_retry_at = (NOW() AT TIME ZONE 'UTC') + CASE
|
||||
last_failure_at = ${now}::timestamp,
|
||||
next_retry_at = ${now}::timestamp + CASE
|
||||
WHEN ${result} = 'hard_failed'
|
||||
AND provider_health.consecutive_failures + 1 >= ${pauseFrom}::int
|
||||
THEN LEAST(
|
||||
@@ -258,7 +271,7 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
)
|
||||
ELSE ${interval}::interval
|
||||
END,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
updated_at = ${now}::timestamp
|
||||
`;
|
||||
} catch {
|
||||
// Fire-and-forget.
|
||||
|
||||
Reference in new issue
Block a user