mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 15:23:29 +00:00
Merge AI operations and error-log discipline into release/v1.42.0
This commit is contained in:
commit
5c23f9ed63
71 files changed
+3650
-340
No files matched your search
@@ -972,7 +972,15 @@ describe("derived response consumer guard", () => {
|
||||
`derived apiSuccess branch ${index + 1} must preserve producer lineage`,
|
||||
).toEqual(expectedLineage);
|
||||
}
|
||||
expect(variableCall(ROUTE_FILE, "derived")).toBe("computeDerivedMetric");
|
||||
// v1.42 — the single read goes through the derived cache; the value is
|
||||
// still the compute's, built inside the cache's builder.
|
||||
const derivedProducer = variableCall(ROUTE_FILE, "derived");
|
||||
expect(["computeDerivedMetric", "cachedSwr"]).toContain(derivedProducer);
|
||||
if (derivedProducer === "cachedSwr") {
|
||||
expect(readFileSync(ROUTE_FILE, "utf8")).toMatch(
|
||||
/const derived = await cachedSwr\([\s\S]*?computeDerivedMetric\(/,
|
||||
);
|
||||
}
|
||||
expect(variableCall(ROUTE_FILE, "assessment")).toBe(
|
||||
"resolveDerivedAssessment",
|
||||
);
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* A caught error in a background job is either reported or explained.
|
||||
*
|
||||
* The failures that cost the most in this codebase were never thrown at
|
||||
* anyone: a nightly pass that caught its error, annotated an `info` line and
|
||||
* carried on, completed in pg-boss every night while doing nothing. The
|
||||
* briefing warm read "all-failed" for a week before anyone could tell which
|
||||
* account and why; the environment fetch threw on every run for the life of
|
||||
* the module. A `catch` that drops what it caught is where that starts.
|
||||
*
|
||||
* Scope: every `catch` clause and every `.catch(handler)` under
|
||||
* `src/lib/jobs/` (tests excluded). A catch passes when its body does one of:
|
||||
*
|
||||
* - rethrows, rejects, or hands the error to a stream callback;
|
||||
* - fails the job (`jobFailed`);
|
||||
* - logs at warn or error: `logCaught` / `caughtAs`, `emitSignal`,
|
||||
* `reportWorkerError`,
|
||||
* `addWarning`, `setError`, `elevateLevel`, `workerLog("error", …)`,
|
||||
* `console.error` / `console.warn`;
|
||||
* - writes the failure to a ledger an operator surface reads
|
||||
* (`recordSyncFailure`, `failAiRun`, `markFailed`,
|
||||
* `recordThumbnailFailure`, `recordBriefingFailure`);
|
||||
* - counts it into a failure tally (`failed += 1`, `errored++`,
|
||||
* `failedSubjects.set(…)`) that the job reports in its outcome facts, so a
|
||||
* systemic failure shows as a count and a partial run as
|
||||
* `job.run.partial`;
|
||||
* - hands it back to its caller in the result (`error: err…`), which is
|
||||
* how every boot-time discovery enqueue reports to the registrar that
|
||||
* logs it at error.
|
||||
*
|
||||
* Anything else is listed in `ALLOWED` below with the reason it may stay
|
||||
* silent, keyed by file and enclosing function, with how many such catches
|
||||
* that function holds. The reasons are the review: "best effort" is not one;
|
||||
* "the pgboss schema is absent on a web-only deployment and null is the
|
||||
* documented answer" is.
|
||||
*
|
||||
* The list cannot rot quietly in either direction. A new silent catch fails
|
||||
* the second case. An entry whose catches were fixed or removed fails the
|
||||
* third case (stale entry). And the fourth case removes each entry in turn
|
||||
* and asserts the sweep goes red without it, so every entry is shown to be
|
||||
* load-bearing, and the sweep is shown to find what it claims to find.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import ts from "typescript";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { walkSourceFiles } from "./helpers/source-files";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
const SCOPE = "lib/jobs/";
|
||||
|
||||
const REPORTS =
|
||||
/\bthrow\b|\breject\(|\bcallback\(\s*err|\bjobFailed\(|\blogCaught\(|\bcaughtAs\(|\bemitSignal\(|\breportWorkerError\(|\.addWarning\(|\.setError\(|\belevateLevel\(|\bworkerLog\(\s*["']error|\bconsole\.(error|warn)\(|\brecordSyncFailure\(|\bfailAiRun\(|\bmarkFailed\(|\brecordThumbnailFailure\(|\brecordBriefingFailure\(|\berror:\s*err\b/;
|
||||
|
||||
/** A failure tally: `summary.failed += 1`, `errored++`, `failures += 1`. */
|
||||
const COUNTS =
|
||||
/\b[\w.]*(?:failed|failures|errored|Failed|Failures)\w*\s*(?:\+=|\+\+|\.set\(|\.push\()/;
|
||||
|
||||
interface Finding {
|
||||
/** `<file under src>::<enclosing function>` */
|
||||
key: string;
|
||||
line: number;
|
||||
body: string;
|
||||
}
|
||||
|
||||
function scan(file: string, text: string): Finding[] {
|
||||
const sf = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true);
|
||||
const out: Finding[] = [];
|
||||
const visit = (node: ts.Node, fn: string): void => {
|
||||
let name = fn;
|
||||
if (
|
||||
(ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node)) &&
|
||||
node.name
|
||||
) {
|
||||
name = node.name.getText(sf);
|
||||
} else if (
|
||||
ts.isVariableDeclaration(node) &&
|
||||
node.initializer &&
|
||||
(ts.isArrowFunction(node.initializer) ||
|
||||
ts.isFunctionExpression(node.initializer))
|
||||
) {
|
||||
name = node.name.getText(sf);
|
||||
}
|
||||
let body: string | null = null;
|
||||
if (ts.isCatchClause(node)) body = node.block.getText(sf);
|
||||
if (
|
||||
ts.isCallExpression(node) &&
|
||||
ts.isPropertyAccessExpression(node.expression) &&
|
||||
node.expression.name.getText(sf) === "catch" &&
|
||||
node.arguments[0]
|
||||
) {
|
||||
body = node.arguments[0].getText(sf);
|
||||
}
|
||||
if (body !== null && !REPORTS.test(body) && !COUNTS.test(body)) {
|
||||
out.push({
|
||||
key: `${file}::${name}`,
|
||||
line: sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1,
|
||||
body,
|
||||
});
|
||||
}
|
||||
ts.forEachChild(node, (child) => visit(child, name));
|
||||
};
|
||||
visit(sf, "<module>");
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Catches allowed to stay silent, by file and function: how many, and why.
|
||||
* Keep the reason checkable against the code it names.
|
||||
*/
|
||||
const ALLOWED: Readonly<Record<string, { count: number; reason: string }>> = {
|
||||
"lib/jobs/apple-health-import-worker.ts::reconcileOrphanImportJobs": {
|
||||
count: 1,
|
||||
reason:
|
||||
"A failed live-state lookup reads as live on purpose: the heartbeat is fresh, flipping a running import would kill it, and the next boot re-evaluates once the heartbeat goes stale.",
|
||||
},
|
||||
"lib/jobs/apple-health-import-worker.ts::safeUnlink": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Deleting a staged file that a previous step or sweep already removed; ENOENT is the expected case and nothing is lost either way.",
|
||||
},
|
||||
"lib/jobs/backup-restore.ts::writeProgress": {
|
||||
count: 1,
|
||||
reason:
|
||||
"A progress tick for the status poll; the next tick rewrites it and the job's own outcome is recorded by `finish`.",
|
||||
},
|
||||
"lib/jobs/backup-restore.ts::runBackupRestoreJob": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Waits out the in-flight progress write before the final state is written; that write's failure is the progress tick's, see writeProgress.",
|
||||
},
|
||||
"lib/jobs/backup-restore.ts::sweepInterruptedRestores": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Completing the dead worker's superseded pg-boss delivery; if it fails the delivery expires on its own, and the restore row it belonged to is already failed and counted.",
|
||||
},
|
||||
"lib/jobs/boss-instance.ts::startProducerAttempts": {
|
||||
count: 2,
|
||||
reason:
|
||||
"The start error is kept and thrown after the last attempt; stopping a boss that never started can only fail for the same reason.",
|
||||
},
|
||||
"lib/jobs/boss-instance.ts::superviseProducerReconnect": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Stopping the producer whose connection already failed, before reconnecting; the reconnect loop reports its own failures.",
|
||||
},
|
||||
"lib/jobs/free-text-encryption-backfill.ts::scrubContactAuditDetails": {
|
||||
count: 1,
|
||||
reason:
|
||||
"A parser: details that are not this action's JSON are left alone by contract (null means 'nothing to scrub'), not an error.",
|
||||
},
|
||||
"lib/jobs/geo-backfill.ts::runGeoBackfill": {
|
||||
count: 1,
|
||||
reason:
|
||||
"The row was deleted by the retention sweep between read and update; counted as still unresolved, and the next pass no longer sees it.",
|
||||
},
|
||||
"lib/jobs/host-metric-sampler.ts::captureHostMetric": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Disk counters exist only on Linux; elsewhere the sample records null for them by design.",
|
||||
},
|
||||
"lib/jobs/host-metric-sampler.ts::readLinuxDiskStats": {
|
||||
count: 1,
|
||||
reason: "No /proc/diskstats outside Linux; see captureHostMetric.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readActiveJobsStartedBefore": {
|
||||
count: 1,
|
||||
reason:
|
||||
"No pgboss schema on a web-only deployment; null is the documented 'no queue to ask', kept apart from an empty list.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readConsecutiveRunFailures": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Called from a failure path that is already logging at error; when the queue cannot be read, 1 (this run) is the only known count.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readFailingQueues": {
|
||||
count: 1,
|
||||
reason: "See readActiveJobsStartedBefore: null means no queue schema.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readLastQueueRun": {
|
||||
count: 1,
|
||||
reason: "See readActiveJobsStartedBefore: null means no queue schema.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readQueueFailureForUser": {
|
||||
count: 1,
|
||||
reason: "See readActiveJobsStartedBefore: null means no queue schema.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readQueueRunningSince": {
|
||||
count: 1,
|
||||
reason: "See readActiveJobsStartedBefore: null means no queue schema.",
|
||||
},
|
||||
"lib/jobs/job-observer.ts::emitJobLine": {
|
||||
count: 1,
|
||||
reason:
|
||||
"The catch around writing a log line itself; there is nowhere further to log to, and a log line must not break the job.",
|
||||
},
|
||||
"lib/jobs/offhost-backup.ts::getS3Client": {
|
||||
count: 2,
|
||||
reason:
|
||||
"headObject answers 'absent' for a missing object (its contract), and the lifecycle read maps NoSuchLifecycleConfiguration to 'missing' and anything else to 'unknown', which the status card shows.",
|
||||
},
|
||||
"lib/jobs/offhost-backup.ts::loadOffhostConfigSafe": {
|
||||
count: 1,
|
||||
reason:
|
||||
"A malformed key reads as 'not configured' for the status card; the backup pass itself calls the throwing loader and fails loudly.",
|
||||
},
|
||||
"lib/jobs/offhost-backup.ts::probeOffhostLifecycle": {
|
||||
count: 1,
|
||||
reason:
|
||||
"The lifecycle probe answers 'unknown', which the admin card renders as such; it is a display read, not a pass.",
|
||||
},
|
||||
"lib/jobs/offhost-backup.ts::runOffhostRoundtripTest": {
|
||||
count: 2,
|
||||
reason:
|
||||
"The admin's connection test returns its error in the result it shows (ok: false, error), and deleting its own probe object is cleanup.",
|
||||
},
|
||||
"lib/jobs/offhost-backup.ts::withEachKey": {
|
||||
count: 1,
|
||||
reason:
|
||||
"Trying each key of the ring in turn; the last error is thrown when no key opens the object.",
|
||||
},
|
||||
"lib/jobs/reminder/medication-reminder-check.ts::cleanupScheduledTelegramDeletions":
|
||||
{
|
||||
count: 1,
|
||||
reason:
|
||||
"Deleting a reminder message the person usually already deleted; the scheduled row is removed either way, and logging every expected 400 would bury the real ones.",
|
||||
},
|
||||
"lib/jobs/reminder/polar-sync.ts::syncUserPolarLegs": {
|
||||
count: 2,
|
||||
reason:
|
||||
"Both legs run, the first error is kept and thrown after the second leg.",
|
||||
},
|
||||
"lib/jobs/restore-drill.ts::runRestoreDrill": {
|
||||
count: 1,
|
||||
reason:
|
||||
"The per-account drill failure is recorded on the account's result (ok: false, error) and counted into the run's restore_drill_accounts_failed fact.",
|
||||
},
|
||||
};
|
||||
|
||||
function sweep(): Finding[] {
|
||||
return walkSourceFiles(join(SRC, SCOPE), { floor: 100 })
|
||||
.filter((p) => !p.includes("__tests__") && !p.endsWith(".test.ts"))
|
||||
.flatMap((p) =>
|
||||
scan(`${SCOPE}${p}`, readFileSync(join(SRC, SCOPE, p), "utf8")),
|
||||
);
|
||||
}
|
||||
|
||||
function unallowed(
|
||||
findings: readonly Finding[],
|
||||
allowed: Readonly<Record<string, { count: number }>>,
|
||||
): string[] {
|
||||
const byKey = new Map<string, Finding[]>();
|
||||
for (const f of findings) {
|
||||
byKey.set(f.key, [...(byKey.get(f.key) ?? []), f]);
|
||||
}
|
||||
const out: string[] = [];
|
||||
for (const [key, list] of byKey) {
|
||||
const allowance = allowed[key]?.count ?? 0;
|
||||
if (list.length > allowance) {
|
||||
for (const f of list.slice(allowance)) {
|
||||
out.push(
|
||||
`${key} (line ${f.line}): ${f.body.replace(/\s+/g, " ").slice(0, 120)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
describe("error-level discipline in background jobs", () => {
|
||||
it("the scanner flags a silent catch and passes a reported one", () => {
|
||||
const planted = scan(
|
||||
"lib/jobs/planted.ts",
|
||||
[
|
||||
"async function quiet() { try { await a(); } catch { /* best effort */ } }",
|
||||
"async function info() { try { await a(); } catch (e) { annotate({ action: { name: 'x.failed' } }); } }",
|
||||
"async function chained() { await a().catch(() => {}); }",
|
||||
"async function loud() { try { await a(); } catch (e) { logCaught('x.y.failed', e); } }",
|
||||
"async function counted() { try { await a(); } catch { summary.failed += 1; } }",
|
||||
"async function rethrown() { try { await a(); } catch (e) { throw e; } }",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(planted.map((f) => f.key)).toEqual([
|
||||
"lib/jobs/planted.ts::quiet",
|
||||
"lib/jobs/planted.ts::info",
|
||||
"lib/jobs/planted.ts::chained",
|
||||
]);
|
||||
});
|
||||
|
||||
it("every silent catch in src/lib/jobs is fixed or explained", () => {
|
||||
expect(unallowed(sweep(), ALLOWED)).toEqual([]);
|
||||
});
|
||||
|
||||
it("no allow-list entry outlives the catches it explains", () => {
|
||||
const counts = new Map<string, number>();
|
||||
for (const f of sweep()) counts.set(f.key, (counts.get(f.key) ?? 0) + 1);
|
||||
const stale = Object.entries(ALLOWED)
|
||||
.filter(([key, { count }]) => (counts.get(key) ?? 0) < count)
|
||||
.map(
|
||||
([key, { count }]) =>
|
||||
`${key}: allows ${count}, finds ${counts.get(key) ?? 0}`,
|
||||
);
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
|
||||
it("every allow-list entry is load-bearing: removing it turns the sweep red", () => {
|
||||
const findings = sweep();
|
||||
const notLoadBearing = Object.keys(ALLOWED).filter((key) => {
|
||||
const without = { ...ALLOWED };
|
||||
delete (without as Record<string, unknown>)[key];
|
||||
return unallowed(findings, without).length === 0;
|
||||
});
|
||||
expect(notLoadBearing).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* No queue may forget its failed jobs before the failure readers look.
|
||||
*
|
||||
* `job-failures.ts` reads pg-boss's own `state = 'failed'` rows over a 72-hour
|
||||
* window: the admin status card, the per-account "last run failed" hint and
|
||||
* the repeated-failure count behind the `job.run.failed` alert all rest on
|
||||
* those rows still existing. pg-boss deletes a terminal row
|
||||
* `deleteAfterSeconds` after it finished. v1.42 shortens that option on the
|
||||
* pure-volume queues, and a value under the reader window would make a
|
||||
* failing queue read as healthy, with nothing anywhere saying so.
|
||||
*
|
||||
* The rule: a `deleteAfterSeconds` in the source is either produced by
|
||||
* `failureReaderRetention(hours)` (which throws under the floor) or written as
|
||||
* a plain literal at or above `FAILED_ROW_AVAILABILITY_FLOOR_HOURS`. Anything
|
||||
* else, a named constant included, fails here, because a constant's value is
|
||||
* not checkable from the call site and the helper exists to make it so.
|
||||
*
|
||||
* Mutation check, run when this guard was written: a planted
|
||||
* `deleteAfterSeconds: 48 * 60 * 60` in a job file fails the second case, and
|
||||
* `deleteAfterSeconds: VOLUME_SECONDS` fails it too; the matcher self-test
|
||||
* below proves the scan sees both shapes, so an empty sweep is not mistaken
|
||||
* for a clean one.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { walkSourceFiles } from "./helpers/source-files";
|
||||
import { FAILED_ROW_AVAILABILITY_FLOOR_HOURS } from "@/lib/jobs/job-failures";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
const FLOOR_SECONDS = FAILED_ROW_AVAILABILITY_FLOOR_HOURS * 3600;
|
||||
|
||||
/** The one file allowed to spell the option out: the helper itself. */
|
||||
const HELPER_FILE = "lib/jobs/job-failures.ts";
|
||||
|
||||
const OPTION = /\bdeleteAfterSeconds\s*:\s*([^,}\n]+)/g;
|
||||
|
||||
/** Evaluate a sum of products of integer literals; null for anything else. */
|
||||
function literalSeconds(expression: string): number | null {
|
||||
const cleaned = expression.trim().replace(/_/g, "");
|
||||
if (!/^\d+(\s*[*+]\s*\d+)*$/.test(cleaned)) return null;
|
||||
return cleaned
|
||||
.split("+")
|
||||
.map((term) =>
|
||||
term
|
||||
.split("*")
|
||||
.map((factor) => Number(factor.trim()))
|
||||
.reduce((a, b) => a * b, 1),
|
||||
)
|
||||
.reduce((a, b) => a + b, 0);
|
||||
}
|
||||
|
||||
interface Finding {
|
||||
file: string;
|
||||
expression: string;
|
||||
}
|
||||
|
||||
function scan(file: string, text: string): Finding[] {
|
||||
const out: Finding[] = [];
|
||||
for (const match of text.matchAll(OPTION)) {
|
||||
out.push({ file, expression: match[1].trim() });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function violations(findings: readonly Finding[]): string[] {
|
||||
return findings
|
||||
.filter((f) => {
|
||||
const seconds = literalSeconds(f.expression);
|
||||
return seconds === null || seconds < FLOOR_SECONDS;
|
||||
})
|
||||
.map((f) => `${f.file}: deleteAfterSeconds: ${f.expression}`);
|
||||
}
|
||||
|
||||
function sourceFindings(): Finding[] {
|
||||
return walkSourceFiles(SRC, { floor: 3000 })
|
||||
.filter((p) => !p.startsWith("generated/"))
|
||||
.filter((p) => !p.includes("__tests__"))
|
||||
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
|
||||
.filter((p) => p !== HELPER_FILE)
|
||||
.flatMap((p) => scan(p, readFileSync(join(SRC, p), "utf8")));
|
||||
}
|
||||
|
||||
describe("job retention floor", () => {
|
||||
it("the matcher sees both a short literal and a named constant", () => {
|
||||
const planted = scan(
|
||||
"planted.ts",
|
||||
[
|
||||
"await boss.createQueue(Q, { deleteAfterSeconds: 48 * 60 * 60 });",
|
||||
"await boss.createQueue(R, { deleteAfterSeconds: VOLUME_SECONDS });",
|
||||
"await boss.createQueue(S, { deleteAfterSeconds: 7 * 24 * 3600 });",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(planted).toHaveLength(3);
|
||||
expect(violations(planted)).toEqual([
|
||||
"planted.ts: deleteAfterSeconds: 48 * 60 * 60",
|
||||
"planted.ts: deleteAfterSeconds: VOLUME_SECONDS",
|
||||
]);
|
||||
});
|
||||
|
||||
it("every queue retention in the tree keeps failed rows past the reader window", () => {
|
||||
expect(violations(sourceFindings())).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user