fix(jobs): run every admitted import inside its own background event

The admission drain ran each full-history import with no wide event in
scope, so every external call, warning and annotation the provider walk
recorded was dropped and a failing backfill left only its error message.
Each admitted run now opens a job.integration_backfill event with its kind,
identity, job id and slice, emitted when the run settles. The Health Connect
import worker gets the same treatment.

The external call list on one event is bounded so a long walk cannot hold
thousands of entries: failed and annotated calls keep the larger share,
the rest is counted in external_calls_omitted.
This commit is contained in:
Marc-André Bombeck committed 2026-10-15 11:07:28 +02:00
1 parent cdd1e9c87f
commit 82d8960044
6 files changed
+251 -3

No files matched your search

@@ -1,5 +1,9 @@
import { describe, expect, it } from "vitest";
import { WideEventBuilder } from "../event-builder";
import {
EXTERNAL_CALLS_MAX,
EXTERNAL_CALLS_PLAIN_MAX,
WideEventBuilder,
} from "../event-builder";
describe("WideEventBuilder.setHttp", () => {
it("stores plain http metadata as-is", () => {
@@ -140,3 +144,46 @@ describe("WideEventBuilder.capStatusLevel", () => {
expect(evt.toJSON().level).toBe("warn");
});
});
describe("WideEventBuilder — bounded external calls", () => {
it("lists failures beyond the share of plain successes, and counts the rest", () => {
const evt = new WideEventBuilder("background");
for (let i = 0; i < 500; i++) {
evt.addExternalCall({
service: "google-health",
method: `page=${i}`,
duration_ms: 1,
status: 200,
});
}
evt.addExternalCall({
service: "google-health",
method: "page=500",
duration_ms: 1,
status: 503,
error: "transient HTTP 503",
});
const event = evt.toJSON();
expect(event.external_calls).toHaveLength(EXTERNAL_CALLS_PLAIN_MAX + 1);
expect(event.external_calls!.at(-1)).toMatchObject({ status: 503 });
expect(event.external_calls_omitted).toBe(500 - EXTERNAL_CALLS_PLAIN_MAX);
for (let i = 0; i < 1000; i++) {
evt.addExternalCall({
service: "google-health",
method: `retry=${i}`,
duration_ms: 1,
status: 429,
note: "rate_limited_retrying_in_ms=2000",
});
}
const after = evt.toJSON();
expect(after.external_calls).toHaveLength(EXTERNAL_CALLS_MAX);
expect(after.external_calls_omitted).toBe(
500 -
EXTERNAL_CALLS_PLAIN_MAX +
1000 -
(EXTERNAL_CALLS_MAX - EXTERNAL_CALLS_PLAIN_MAX - 1),
);
});
});
+24
View File
@@ -35,6 +35,12 @@ function redactDeep<T>(value: T, depth = 0): T {
return value;
}
/** The most external calls one event lists. */
export const EXTERNAL_CALLS_MAX = 200;
/** The most calls without an error or a note one event lists. */
export const EXTERNAL_CALLS_PLAIN_MAX = 60;
/**
* Baut ein Wide Event Schritt fuer Schritt auf.
* Wird am Anfang eines Requests/Tasks erstellt und am Ende emittiert.
@@ -44,6 +50,7 @@ export class WideEventBuilder {
private startTime: number;
private highestLevel: LogLevel = "info";
private statusLevelCeiling: LogLevel = "error";
private plainExternalCalls = 0;
constructor(kind: EventKind = "http") {
this.startTime = performance.now();
@@ -208,6 +215,23 @@ export class WideEventBuilder {
addExternalCall(call: NonNullable<WideEvent["external_calls"]>[0]): this {
if (!this.event.external_calls) this.event.external_calls = [];
// Bounded. A long background run (a full-history backfill walks hundreds
// of pages in one event) would otherwise hold every call in memory and
// write them all as one log line. Calls that failed or carry a note are
// what a reader looks for, so they keep the larger share; plain successes
// stop being listed sooner. Whatever is left out is counted, never lost
// silently.
const calls = this.event.external_calls;
const notable = call.error !== undefined || call.note !== undefined;
if (
calls.length >= EXTERNAL_CALLS_MAX ||
(!notable && this.plainExternalCalls >= EXTERNAL_CALLS_PLAIN_MAX)
) {
this.event.external_calls_omitted =
(this.event.external_calls_omitted ?? 0) + 1;
return this;
}
if (!notable) this.plainExternalCalls += 1;
// An outbound failure carries whatever the remote said, and for several
// integrations that is the request URL — which for Telegram embeds the bot
// token. `setError` and `setHttp` scrub on the way in; this entry point did
+5
View File
@@ -123,6 +123,11 @@ export interface WideEvent {
*/
note?: string;
}>;
/**
* How many external calls the event did not list because it had already
* listed its share (`EXTERNAL_CALLS_MAX`, `EXTERNAL_CALLS_PLAIN_MAX`).
*/
external_calls_omitted?: number;
// Hintergrundaufgabe (nur bei kind === "background")
background?: {