fix(jobs): run every admitted import inside its own background event

The admission drain ran each full-history import with no wide event in
scope, so every external call, warning and annotation the provider walk
recorded was dropped and a failing backfill left only its error message.
Each admitted run now opens a job.integration_backfill event with its kind,
identity, job id and slice, emitted when the run settles. The Health Connect
import worker gets the same treatment.

The external call list on one event is bounded so a long walk cannot hold
thousands of entries: failed and annotated calls keep the larger share,
the rest is counted in external_calls_omitted.
This commit is contained in:
Marc-André Bombeck committed 2026-10-15 11:07:28 +02:00
1 parent cdd1e9c87f
commit 82d8960044
6 files changed
+251 -3

No files matched your search

@@ -1,5 +1,9 @@
import { describe, expect, it } from "vitest";
import { WideEventBuilder } from "../event-builder";
import {
EXTERNAL_CALLS_MAX,
EXTERNAL_CALLS_PLAIN_MAX,
WideEventBuilder,
} from "../event-builder";
describe("WideEventBuilder.setHttp", () => {
it("stores plain http metadata as-is", () => {
@@ -140,3 +144,46 @@ describe("WideEventBuilder.capStatusLevel", () => {
expect(evt.toJSON().level).toBe("warn");
});
});
describe("WideEventBuilder — bounded external calls", () => {
it("lists failures beyond the share of plain successes, and counts the rest", () => {
const evt = new WideEventBuilder("background");
for (let i = 0; i < 500; i++) {
evt.addExternalCall({
service: "google-health",
method: `page=${i}`,
duration_ms: 1,
status: 200,
});
}
evt.addExternalCall({
service: "google-health",
method: "page=500",
duration_ms: 1,
status: 503,
error: "transient HTTP 503",
});
const event = evt.toJSON();
expect(event.external_calls).toHaveLength(EXTERNAL_CALLS_PLAIN_MAX + 1);
expect(event.external_calls!.at(-1)).toMatchObject({ status: 503 });
expect(event.external_calls_omitted).toBe(500 - EXTERNAL_CALLS_PLAIN_MAX);
for (let i = 0; i < 1000; i++) {
evt.addExternalCall({
service: "google-health",
method: `retry=${i}`,
duration_ms: 1,
status: 429,
note: "rate_limited_retrying_in_ms=2000",
});
}
const after = evt.toJSON();
expect(after.external_calls).toHaveLength(EXTERNAL_CALLS_MAX);
expect(after.external_calls_omitted).toBe(
500 -
EXTERNAL_CALLS_PLAIN_MAX +
1000 -
(EXTERNAL_CALLS_MAX - EXTERNAL_CALLS_PLAIN_MAX - 1),
);
});
});