mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 06:06:10 +00:00
POST /api/measurements/batch ran one interactive reconciler transaction per fresh sample: a savepoint, three advisory locks, two candidate reads and the insert, about ten round trips. A 500-entry upload of new samples was 500 of those, the 2 to 15 second outliers. Fresh samples now land through INSERT ... ON CONFLICT DO NOTHING under the same per-account advisory lock. Both measurement identities are full unique indexes, so the statement inserts exactly what the reconciler would have inserted, and every skipped row still goes through the reconciler for its own verdict. Only rows whose external and natural keys each occur once in the batch are buffered, and the buffer is flushed before any other row is reconciled, so the order of effects stays the sequential one. The contract and every per-entry status are unchanged; a golden test runs each batch through both paths against the same table. /api/health took seconds after a start because register() awaited the whole pg-boss worker start (queues, schedules, boot discovery) before the server took a request. The worker, or a web process's producer, now starts in the background after the blocking boot checks; the probe answers at once with reason "starting" until it is up, and gives the database two seconds before calling it not answering.
242 lines
8.2 KiB
TypeScript
242 lines
8.2 KiB
TypeScript
import type { Instrumentation } from "next";
|
|
|
|
export async function register() {
|
|
// Only start the worker on the Node.js server runtime (not Edge, not build)
|
|
if (process.env.NEXT_RUNTIME === "nodejs") {
|
|
// First-boot readiness summary: a human-readable block on the core secrets
|
|
// (with the exact fix on a bad one) before any fail-closed loader throws,
|
|
// plus a one-glance status of the optional channels. Runs for every process
|
|
// type (web + worker). Never throws — the real gate stays the loaders.
|
|
try {
|
|
const { logReadinessSummary } =
|
|
await import("@/lib/boot/readiness-summary");
|
|
logReadinessSummary();
|
|
} catch {
|
|
// A readiness summary must never break boot.
|
|
}
|
|
|
|
// Does the configured encryption key open this database? A process whose
|
|
// key does not refuses every API request with one clear 503 and starts
|
|
// neither the producer nor the worker (a wrong key must never write).
|
|
try {
|
|
const [{ runBootKeyCheck }, { prisma }] = await Promise.all([
|
|
import("@/lib/boot/key-check"),
|
|
import("@/lib/db"),
|
|
]);
|
|
if (await runBootKeyCheck(prisma)) return;
|
|
} catch (err) {
|
|
console.error("[instrumentation] Encryption key check crashed:", err);
|
|
}
|
|
|
|
// Every connection is pinned to UTC; say clearly if one is not.
|
|
try {
|
|
const [{ runBootTimeZoneCheck }, { prisma }] = await Promise.all([
|
|
import("@/lib/boot/time-zone-check"),
|
|
import("@/lib/db"),
|
|
]);
|
|
await runBootTimeZoneCheck(prisma);
|
|
} catch {
|
|
// A diagnostic must never break boot.
|
|
}
|
|
|
|
// Every pulse reader follows the operator's pulse day-mean method; mirror
|
|
// it in this process and keep the mirror current.
|
|
try {
|
|
const { startPulseDayMeanMethodSync } =
|
|
await import("@/lib/measurements/pulse-day-mean-method");
|
|
await startPulseDayMeanMethodSync();
|
|
} catch {
|
|
// Without the mirror the process keeps the default method.
|
|
}
|
|
|
|
// Every Node process watches its own event loop — web-only
|
|
// containers included; a stall report from the process serving
|
|
// requests is the whole point.
|
|
const { startEventLoopLagMonitor } =
|
|
await import("@/lib/observability/event-loop-lag");
|
|
startEventLoopLagMonitor();
|
|
|
|
// The same for its heap: a trend line, and one report naming the work in
|
|
// flight before the heap runs out (a heap-limit crash logs nothing else).
|
|
try {
|
|
const [{ startHttpInFlightTracking }, { startHeapMonitor }] =
|
|
await Promise.all([
|
|
import("@/lib/observability/in-flight"),
|
|
import("@/lib/observability/heap-monitor"),
|
|
]);
|
|
startHttpInFlightTracking();
|
|
startHeapMonitor();
|
|
} catch {
|
|
// A diagnostic must never break boot.
|
|
}
|
|
|
|
// One line when the container's memory limit is below the supported
|
|
// floor. The app keeps running.
|
|
try {
|
|
const { logMemoryFloorAtBoot } = await import("@/lib/support/heap-limit");
|
|
logMemoryFloorAtBoot();
|
|
} catch {
|
|
// A diagnostic must never break boot.
|
|
}
|
|
|
|
// The background role starts after the server is up. Awaiting it here
|
|
// held every request, `/api/health` included, for the seconds pg-boss
|
|
// needs to start, provision its queues and run the boot discovery; the
|
|
// probe now answers `starting` meanwhile. A request that enqueues while
|
|
// the boss is not bound yet takes the same path as during a producer
|
|
// reconnect: a garnish skips its enqueue (the owning cron catches up),
|
|
// and a route that needs the worker answers 503 until it is up.
|
|
const [{ shouldRunWorker }, { markStartup }] = await Promise.all([
|
|
import("@/lib/process-type"),
|
|
import("@/lib/boot/startup-state"),
|
|
]);
|
|
markStartup("starting");
|
|
if (!shouldRunWorker()) {
|
|
// Split deployment: web requests still produce queue jobs, but this
|
|
// process owns no schedules, supervision, or consumers.
|
|
void (async () => {
|
|
try {
|
|
const { startGlobalBossProducer } =
|
|
await import("@/lib/jobs/boss-instance");
|
|
const boss = await startGlobalBossProducer(process.env.DATABASE_URL);
|
|
if (boss) {
|
|
const { registerShutdownHandlers } =
|
|
await import("@/lib/jobs/reminder/worker-lifecycle");
|
|
registerShutdownHandlers(boss);
|
|
}
|
|
markStartup(boss ? "ready" : "failed");
|
|
} catch (err) {
|
|
markStartup("failed");
|
|
// Queue-backed garnishes degrade independently; never make the web
|
|
// server unavailable because its producer connection failed.
|
|
console.error(
|
|
"[instrumentation] Failed to start pg-boss producer:",
|
|
err,
|
|
);
|
|
}
|
|
})();
|
|
return;
|
|
}
|
|
|
|
void (async () => {
|
|
const { WideEventBuilder } = await import("@/lib/logging/event-builder");
|
|
const { emitIfSampled } = await import("@/lib/logging/transports");
|
|
|
|
// Emit startup event
|
|
const startupEvt = new WideEventBuilder("background");
|
|
startupEvt.setBackground({ task_name: "startup" });
|
|
|
|
try {
|
|
const { startReminderWorker } =
|
|
await import("@/lib/jobs/reminder-worker");
|
|
await startReminderWorker();
|
|
markStartup("ready");
|
|
|
|
startupEvt.addMeta("reminder_worker", "started");
|
|
startupEvt.finish();
|
|
emitIfSampled(startupEvt.toJSON());
|
|
} catch (err) {
|
|
markStartup("failed");
|
|
startupEvt.setError(err);
|
|
startupEvt.addMeta("reminder_worker", "failed");
|
|
startupEvt.finish();
|
|
emitIfSampled(startupEvt.toJSON());
|
|
|
|
// Still log to stderr for container health checks
|
|
console.error(
|
|
"[instrumentation] CRITICAL: Failed to start reminder worker:",
|
|
err,
|
|
);
|
|
}
|
|
})();
|
|
}
|
|
}
|
|
|
|
export const onRequestError: Instrumentation.onRequestError = async (
|
|
error,
|
|
request,
|
|
context,
|
|
) => {
|
|
// Prisma-based settings lookup is not available in edge instrumentation.
|
|
if (process.env.NEXT_RUNTIME === "edge") return;
|
|
|
|
try {
|
|
const [{ getGlitchtipSettings }, { sendGlitchtipEvent }] =
|
|
await Promise.all([
|
|
import("@/lib/monitoring-settings"),
|
|
import("@/lib/monitoring/glitchtip"),
|
|
]);
|
|
|
|
const settings = await getGlitchtipSettings();
|
|
if (!settings.glitchtipEnabled || !settings.glitchtipDsn) return;
|
|
|
|
const err =
|
|
error instanceof Error
|
|
? error
|
|
: new Error(
|
|
typeof error === "string" ? error : "Unhandled request error",
|
|
);
|
|
|
|
// Skip expected errors from bot scanners (malformed JSON bodies)
|
|
if (err instanceof SyntaxError) return;
|
|
|
|
const userAgentHeader = request.headers["user-agent"];
|
|
const userAgent = Array.isArray(userAgentHeader)
|
|
? userAgentHeader[0]
|
|
: userAgentHeader;
|
|
|
|
const details = [
|
|
`Route: ${context.routePath}`,
|
|
`Type: ${context.routeType}`,
|
|
`Router: ${context.routerKind}`,
|
|
`Method: ${request.method}`,
|
|
`Path: ${request.path}`,
|
|
context.renderSource ? `Render: ${context.renderSource}` : null,
|
|
context.revalidateReason
|
|
? `Revalidate: ${context.revalidateReason}`
|
|
: null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(" | ");
|
|
|
|
const message = `${err.message} [${details}]`.slice(0, 1900);
|
|
|
|
// Try to get request_id from Wide Event context
|
|
let requestId: string | undefined;
|
|
try {
|
|
const { getEvent } = await import("@/lib/logging/context");
|
|
requestId = getEvent()?.getRequestId();
|
|
} catch {
|
|
// Logging not available in this context
|
|
}
|
|
|
|
const delivery = await sendGlitchtipEvent({
|
|
dsn: settings.glitchtipDsn,
|
|
input: {
|
|
environment: settings.glitchtipEnvironment || "production",
|
|
message,
|
|
stack: err.stack,
|
|
level: "error",
|
|
type: err.name || "RequestError",
|
|
url: request.path,
|
|
userAgent,
|
|
sourceTag: "healthlog-server",
|
|
requestId,
|
|
},
|
|
});
|
|
|
|
if (!delivery.ok) {
|
|
console.error(
|
|
"Global Glitchtip request error reporting failed:",
|
|
delivery.status,
|
|
delivery.details,
|
|
);
|
|
}
|
|
} catch (reportError) {
|
|
console.error(
|
|
"Global Glitchtip request error reporting crashed:",
|
|
reportError,
|
|
);
|
|
}
|
|
};
|