Merge AI operations and error-log discipline into release/v1.42.0

This commit is contained in:
Marc-André Bombeck committed 2026-10-08 09:06:58 +02:00
commit 5c23f9ed63
71 files changed
+3650 -340

No files matched your search

+41
View File
@@ -17,6 +17,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
readActiveJobsStartedBefore,
readConsecutiveRunFailures,
readFailingQueues,
readQueueFailureForUser,
} from "@/lib/jobs/job-failures";
@@ -189,6 +190,46 @@ describe("readActiveJobsStartedBefore (#1031)", () => {
});
});
describe("readConsecutiveRunFailures (v1.42)", () => {
const STREAK_QUEUE = "insight-pregenerate";
/** Complete one job with the facts a handler returned. */
async function completeOneJob(did: Record<string, unknown>) {
const jobId = await boss.send(STREAK_QUEUE, {}, { retryLimit: 0 });
const [job] = await boss.fetch(STREAK_QUEUE);
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
await boss.complete(STREAK_QUEUE, job.id, { ok: true, did });
return jobId!;
}
/** Pin the finishing order so the test does not race the clock. */
async function finishedMinutesAgo(jobId: string, minutes: number) {
await getPrismaClient().$executeRawUnsafe(
`UPDATE pgboss.job SET completed_on = now() - make_interval(mins => ${minutes}) WHERE id = '${jobId}'`,
);
}
it("counts failed runs and all-failed nights, skips runs that do not say, and stops at a good night", async () => {
await boss.createQueue(STREAK_QUEUE);
// Oldest first: a good night, then a pg-boss failure, an on-demand job
// that reports nothing about the run, and an all-failed night.
await finishedMinutesAgo(await completeOneJob({ all_failed: false }), 40);
await finishedMinutesAgo(
await failOneJob(STREAK_QUEUE, {}, "worker died"),
30,
);
await finishedMinutesAgo(await completeOneJob({ forced: 1 }), 20);
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 10);
// Two earlier failures plus the run failing now.
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
});
it("answers 1 for a queue with no history: the run failing now", async () => {
await expect(readConsecutiveRunFailures("never-ran")).resolves.toBe(1);
});
});
describe("no queue schema at all", () => {
it("answers null rather than throwing, so the status page still renders", async () => {
const prisma = getPrismaClient();
+80
View File
@@ -0,0 +1,80 @@
/**
* v1.42 — the escalating provider pause, against the real upsert.
*
* The pause is computed inside the one `INSERT … ON CONFLICT DO UPDATE` that
* records a failure, from the count that same statement writes, so two
* workers failing at once still arm the pause their combined count deserves.
* Only Postgres can show the CASE arithmetic is right; the unit test can only
* show the statement carries it.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
import {
HARD_FAILURE_COOLDOWN_MS,
PROVIDER_PAUSE_BASE_MS,
PROVIDER_PAUSE_THRESHOLD,
postgresProviderHealthLedger as ledger,
} from "@/lib/ai/provider-health-ledger";
const prisma = getPrismaClient();
async function windowMs(userId: string): Promise<number> {
const row = await prisma.providerHealth.findFirstOrThrow({
where: { userId, providerType: "admin-openai" },
select: { nextRetryAt: true, lastFailureAt: true },
});
return row.nextRetryAt!.getTime() - row.lastFailureAt!.getTime();
}
describe("provider pause in the Postgres ledger", () => {
let userId: string;
beforeEach(async () => {
await truncateAllTables(prisma);
const user = await prisma.user.create({
data: {
email: "pause@example.test",
username: "pause",
passwordHash: "x",
},
});
userId = user.id;
});
it("backs off briefly, then pauses from the threshold, doubling, and a success resumes", async () => {
for (let i = 1; i < PROVIDER_PAUSE_THRESHOLD; i += 1) {
await ledger.recordFailure(userId, "admin-openai", 500);
}
expect(await windowMs(userId)).toBe(HARD_FAILURE_COOLDOWN_MS);
expect(
(await ledger.getSkipHints(userId)).get("admin-openai")?.reason,
).toBe("backoff");
await ledger.recordFailure(userId, "admin-openai", 500);
expect(await windowMs(userId)).toBe(PROVIDER_PAUSE_BASE_MS);
expect(
(await ledger.getSkipHints(userId)).get("admin-openai")?.reason,
).toBe("paused");
await ledger.recordFailure(userId, "admin-openai", 500);
expect(await windowMs(userId)).toBe(2 * PROVIDER_PAUSE_BASE_MS);
await ledger.recordSuccess(userId, "admin-openai");
expect((await ledger.getSkipHints(userId)).has("admin-openai")).toBe(false);
});
it("keeps the fixed cooldown for a dead credential, however long the run", async () => {
for (let i = 0; i < PROVIDER_PAUSE_THRESHOLD + 2; i += 1) {
await ledger.recordFailure(userId, "admin-openai", 401);
}
expect(
(await ledger.getSkipHints(userId)).get("admin-openai")?.reason,
).toBe("credential_expired");
});
});
@@ -0,0 +1,69 @@
/**
* v1.42 — the tiered context's live day band buckets on the reader's own
* calendar day, including on a daylight-saving changeover.
*
* For a reader far from UTC the 14–30-day band is read live, bucketed by
* `date_trunc('day', <local wall clock>)`. `measured_at` is a `timestamp`
* without time zone holding UTC, and the band read it with a single
* `AT TIME ZONE tz`, which interprets the value as local time and converts
* the wrong way: in Los Angeles every reading moved sixteen hours, and an
* evening reading was summarised as the next day. The Coach and the briefing
* then quoted a day's mean built from two different days.
*
* 2026-03-08 is the US changeover (02:00 PST becomes 03:00 PDT). Two weight
* readings on that local day, 08:00 PDT (15:00Z) and 23:30 PDT (06:30Z on
* the 9th), must form one bucket for the 8th.
*/
import { beforeAll, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
import { buildTieredSeries } from "@/lib/rollups/tiered-context";
const prisma = getPrismaClient();
const USER = "tiered-context-local-day";
const TZ = "America/Los_Angeles";
/** "Now" for the band arithmetic: the 8th sits inside the 14–30-day band. */
const NOW = new Date("2026-03-25T12:00:00Z").getTime();
beforeAll(async () => {
await truncateAllTables(prisma);
await prisma.user.create({
data: { id: USER, username: USER, timezone: TZ },
});
await prisma.measurement.createMany({
data: [
{ at: "2026-03-08T15:00:00Z", value: 70 },
{ at: "2026-03-09T06:30:00Z", value: 80 },
// A reading on the local 7th, late evening PST, for contrast.
{ at: "2026-03-08T07:30:00Z", value: 90 },
].map(({ at, value }) => ({
userId: USER,
type: "WEIGHT" as const,
value,
unit: "kg",
measuredAt: new Date(at),
source: "MANUAL" as const,
})),
});
});
describe("tiered context — live day band on the reader's calendar", () => {
it("puts an evening reading on its own local day across the DST changeover", async () => {
const series = await buildTieredSeries(USER, "WEIGHT", {
tz: TZ,
now: NOW,
skipEnsureFresh: true,
});
const byDay = Object.fromEntries(
series.dayBand.map((b) => [b.bucketStart.slice(0, 10), b]),
);
expect(Object.keys(byDay).sort()).toEqual(["2026-03-07", "2026-03-08"]);
expect(byDay["2026-03-08"]).toMatchObject({ count: 2, mean: 75 });
expect(byDay["2026-03-07"]).toMatchObject({ count: 1, mean: 90 });
});
});