From b55850967dab9f7c8dff53175f25170e48a73483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:14:22 +0200 Subject: [PATCH 1/5] fix(idempotency): replay a key only for the same request body The replay cell was keyed on (user, key, method, path). A client that sends two different writes under one key got the first write's cached response for the second, and the second was never saved. Native clients up to 1.1.0 post the two halves of a manual blood pressure exactly that way, so the diastolic value was silently dropped. Each cell now stores a SHA-256 fingerprint of the request body (JSON with recursively sorted keys, otherwise the raw bytes; scoped by method and path; at most the first MiB is read). Lookup replays, or refuses as in flight, only when the fingerprint matches. A different body, or a row written before the column existed, runs the handler without claiming or caching and leaves the existing cell for its own retries; a lost claim race re-checks the winner's fingerprint the same way. The body itself is never stored. Migration 0371 adds the nullable idempotency_keys.request_fingerprint column. The OpenAPI parameter description, the client notes and the Home Assistant guide say what a replay now requires. docs/ops gains a read-only query that lists systolic readings left without their diastolic half, held to the schema by an integration test. --- CHANGELOG.md | 15 ++ docs/CONTRIBUTING-CLIENTS.md | 2 +- docs/api/openapi.yaml | 14 +- docs/integrations/home-assistant.md | 5 +- docs/ops/blood-pressure-missing-diastolic.md | 60 +++++ .../migration.sql | 8 + prisma/schema.prisma | 22 +- .../__tests__/idempotency-conflict.test.ts | 11 +- src/lib/__tests__/idempotency.test.ts | 208 +++++++++++++++ src/lib/idempotency.ts | 184 ++++++++++++- src/lib/openapi/routes/shared.ts | 5 +- .../idempotency-body-fingerprint.test.ts | 249 ++++++++++++++++++ 12 files changed, 761 insertions(+), 22 deletions(-) create mode 100644 docs/ops/blood-pressure-missing-diastolic.md create mode 100644 prisma/migrations/0371_idempotency_request_fingerprint/migration.sql create mode 100644 tests/integration/idempotency-body-fingerprint.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 53cb335cb..5654e7f10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [Unreleased] + +### Fixed + +- **Blood pressure saved without its diastolic value.** The iPhone app (up to + 1.0.3, and TestFlight 1.1.0) sends the two halves of a manual reading as two + requests under one `Idempotency-Key`, and the server answered the second + with the first one's cached response instead of saving it. The replay cache + now stores a SHA-256 fingerprint of the request body (migration 0371, + `idempotency_keys.request_fingerprint`) and replays only the same body; a + different body under the same key is saved normally. Readings saved before + this fix stay incomplete: delete the incomplete reading and enter it again + with both values and the original time. Operators can list affected + readings with the query in `docs/ops/blood-pressure-missing-diastolic.md`. + ## [1.39.9] — 2026-10-03 Workouts can be logged by hand, the body fat tile opens its own page, and diff --git a/docs/CONTRIBUTING-CLIENTS.md b/docs/CONTRIBUTING-CLIENTS.md index 2f5660022..e2f627e8c 100644 --- a/docs/CONTRIBUTING-CLIENTS.md +++ b/docs/CONTRIBUTING-CLIENTS.md @@ -19,7 +19,7 @@ Every JSON route answers the same shape: `{ data, error }`, with `meta` on the e - Refresh tokens rotate per device and are one-time-use. Reusing an old one revokes that device's whole token family, on purpose. Persist the newest pair atomically and never retry a refresh with a token that already succeeded once. - Bearer token scopes are fail-closed. A token minted for one purpose does not open unrelated routes, and a route that does not declare a scope refuses narrow tokens outright. Expect `403` to be a normal, recoverable answer, not an anomaly. - Admin surfaces are cookie-session only, by construction. A native client cannot be an admin client, whatever scopes its token carries. Do not build admin features. -- Mutations accept an `Idempotency-Key` header. Replays answer with the original result plus `X-Idempotent-Replay`, so retrying on a dropped connection is safe. Use it for anything a user would notice twice. +- Mutations accept an `Idempotency-Key` header. A repeat of the same key with the same body answers with the original result plus `X-Idempotent-Replay`, so retrying on a dropped connection is safe. A different body under the same key is treated as a new request, not a replay; still, give each distinct write its own key. Use it for anything a user would notice twice. ## Data semantics worth knowing before you sync diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 25431a0bd..50a2926cb 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -162,12 +162,14 @@ paths: pattern: ^[A-Za-z0-9_\-:.]{8,128}$ minLength: 8 maxLength: 128 - description: "Makes this write safe to retry. Send the same key again and the first response is replayed rather than the - work repeated, marked with `X-Idempotent-Replay: true`. A key that does not match the pattern is IGNORED - rather than refused — the write then proceeds unprotected and looks exactly like a request that carried no - key, so validate the key on your side. Scoped per user and per (method, path): the same key on a different - route is a different key. While a first request under this key is still in flight, a second is refused with - 409 rather than queued." + description: "Makes this write safe to retry. Send the same key with the same body again and the first response is + replayed rather than the work repeated, marked with `X-Idempotent-Replay: true`. The body is compared as + JSON with key order ignored; a different body under the same key is not a retry and runs as a new write, + while the first response stays stored for retries of the first request. A key that does not match the + pattern is IGNORED rather than refused — the write then proceeds unprotected and looks exactly like a + request that carried no key, so validate the key on your side. Scoped per user and per (method, path): the + same key on a different route is a different key. While a first request under this key is still in flight, a + second with the same body is refused with 409 rather than queued." requestBody: required: true content: diff --git a/docs/integrations/home-assistant.md b/docs/integrations/home-assistant.md index c521a8c74..71d679ccc 100644 --- a/docs/integrations/home-assistant.md +++ b/docs/integrations/home-assistant.md @@ -171,8 +171,9 @@ tells you which rows to retry. ## Retries and duplicates Send an `Idempotency-Key` header with a value your automation can reproduce for -the same reading. A retry with the same key returns the original response -instead of writing a second row: +the same reading. A retry with the same key and the same body returns the +original response instead of writing a second row. A different body under the +same key is a different reading and is written: ```yaml headers: diff --git a/docs/ops/blood-pressure-missing-diastolic.md b/docs/ops/blood-pressure-missing-diastolic.md new file mode 100644 index 000000000..1509a6ade --- /dev/null +++ b/docs/ops/blood-pressure-missing-diastolic.md @@ -0,0 +1,60 @@ +# Blood pressure readings missing their diastolic half + +## What happened + +The iPhone app (every App Store build up to 1.0.3 and TestFlight 1.1.0) +saves a manually entered blood pressure as two requests to +`POST /api/measurements`: one `BLOOD_PRESSURE_SYS`, one +`BLOOD_PRESSURE_DIA`, both carrying the same `Idempotency-Key`. Until the +replay cache learned to compare request bodies, the server treated the second +request as a retry of the first. It answered with the first request's cached +`201` (header `X-Idempotent-Replay: true`) and never wrote the diastolic +value. The app showed success, so nobody noticed. + +The fix makes the cache replay only a request with the same body. A different +body under the same key now runs normally. Nothing recovers the values that +were never sent to the database: they exist only on the person's phone, if at +all. + +## Finding the affected readings + +Read-only. Run it against the HealthLog database (with the bundled compose +file: `docker compose exec db psql -U healthlog healthlog`). It lists every live systolic reading that has no +live diastolic reading for the same account at the same instant. + + + +```sql +SELECT u.username, + m.id, + m.measured_at, + m.value AS systolic, + m.source +FROM measurements m +JOIN users u ON u.id = m.user_id +WHERE m.type = 'BLOOD_PRESSURE_SYS' + AND m.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM measurements d + WHERE d.user_id = m.user_id + AND d.measured_at = m.measured_at + AND d.type = 'BLOOD_PRESSURE_DIA' + AND d.deleted_at IS NULL + ) +ORDER BY u.username, m.measured_at; +``` + + + +Readings from the phone app carry `source = 'MANUAL'`. Rows from a synced +source (Withings, Apple Health) that show up here are a different gap, not +this one: those sources write both halves themselves. + +## What to tell users + +There is nothing the server can repair: the diastolic value never reached +it. A person whose reading appears in the list can re-enter it: delete the +incomplete reading, then add the blood pressure again with both values and +the original date and time. From this release on, both halves are saved on +the first try, from any app version. diff --git a/prisma/migrations/0371_idempotency_request_fingerprint/migration.sql b/prisma/migrations/0371_idempotency_request_fingerprint/migration.sql new file mode 100644 index 000000000..dcb360852 --- /dev/null +++ b/prisma/migrations/0371_idempotency_request_fingerprint/migration.sql @@ -0,0 +1,8 @@ +-- What request a replay-cache cell answered, as a SHA-256 of the canonical +-- request body scoped by method and path. A request that reuses a key with a +-- different body is executed instead of being handed the first body's cached +-- response. The body itself is never stored. Nullable: rows written before +-- this release carry none, and a null never matches, so they stop replaying +-- for anything but their own retries already in flight (they expire within +-- 24 hours either way). +ALTER TABLE "idempotency_keys" ADD COLUMN IF NOT EXISTS "request_fingerprint" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6a05a5659..9250aef3f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5545,15 +5545,19 @@ model ProviderHealth { /// (userId, key, method, path) so retries within the TTL replay the /// original outcome instead of producing a duplicate side-effect. model IdempotencyKey { - id String @id @default(cuid()) - userId String @map("user_id") - key String - method String - path String - responseStatus Int @map("response_status") - responseBody String @map("response_body") // serialized JSON envelope - expiresAt DateTime @map("expires_at") - createdAt DateTime @default(now()) @map("created_at") + id String @id @default(cuid()) + userId String @map("user_id") + key String + method String + path String + responseStatus Int @map("response_status") + responseBody String @map("response_body") // serialized JSON envelope + /// SHA-256 of the canonical request body, scoped by method and path. A + /// lookup replays only when it matches; a different body under the same key + /// runs normally. Null on rows written before it existed, which never match. + requestFingerprint String? @map("request_fingerprint") + expiresAt DateTime @map("expires_at") + createdAt DateTime @default(now()) @map("created_at") user User @relation(fields: [userId], references: [id], onDelete: Cascade) diff --git a/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts b/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts index adb31bbdc..dc93fd8e2 100644 --- a/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts +++ b/src/app/api/anamnesis/facts/__tests__/idempotency-conflict.test.ts @@ -58,10 +58,13 @@ vi.mock("@/lib/profile/health-facts", () => ({ import { NextRequest } from "next/server"; import { POST } from "../route"; +import { requestFingerprint } from "@/lib/idempotency"; /** The pending sentinel `findCached` recognises as "another call is running". */ const PENDING_STATUS = 0; +const BODY = JSON.stringify({ kind: "SMOKING_STATUS", value: "FORMER" }); + function request(): NextRequest { return new NextRequest("http://localhost/api/anamnesis/facts", { method: "POST", @@ -69,7 +72,7 @@ function request(): NextRequest { "content-type": "application/json", "idempotency-key": "key-in-flight", }, - body: JSON.stringify({ kind: "SMOKING_STATUS", value: "FORMER" }), + body: BODY, }); } @@ -99,6 +102,12 @@ describe("idempotency 409 envelope, over the real route", () => { it("serves `error` as a string when a request is already in flight", async () => { mocks.findUnique.mockResolvedValue({ id: "idem-1", + // The same request in flight: a different body would run instead. + requestFingerprint: requestFingerprint( + "POST", + "/api/anamnesis/facts", + new TextEncoder().encode(BODY), + ), responseStatus: PENDING_STATUS, responseBody: "", expiresAt: new Date(Date.now() + 60_000), diff --git a/src/lib/__tests__/idempotency.test.ts b/src/lib/__tests__/idempotency.test.ts index 61494372e..a93a23ee1 100644 --- a/src/lib/__tests__/idempotency.test.ts +++ b/src/lib/__tests__/idempotency.test.ts @@ -49,6 +49,8 @@ import { withIdempotency, defaultUserIdResolver, isCachableStatus, + canonicalJson, + requestFingerprint, } from "../idempotency"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; @@ -81,6 +83,13 @@ function makeRequest( }); } +/** The fingerprint `makeRequest("POST", …)` produces. */ +const OK_BODY_FINGERPRINT = requestFingerprint( + "POST", + "/api/example", + new TextEncoder().encode(JSON.stringify({ ok: true })), +); + beforeEach(() => { vi.resetAllMocks(); vi.mocked(prisma.idempotencyKey.findUnique).mockResolvedValue(null); @@ -144,6 +153,11 @@ describe("withIdempotency", () => { data: { responseBody: string; responseStatus: number }; } ).data; + const claimedFingerprint = ( + vi.mocked(prisma.idempotencyKey.create).mock.calls[0][0] as { + data: { requestFingerprint: string }; + } + ).data.requestFingerprint; // Second call: cache hit returns persisted envelope. vi.mocked(prisma.idempotencyKey.findUnique).mockResolvedValueOnce({ @@ -152,6 +166,7 @@ describe("withIdempotency", () => { key: "abc-12345678", method: "POST", path: "/api/example", + requestFingerprint: claimedFingerprint, responseStatus: persistedBody.responseStatus, responseBody: persistedBody.responseBody, expiresAt: new Date(Date.now() + 86_400_000), @@ -296,6 +311,7 @@ describe("withIdempotency concurrency claim (A3)", () => { key: "abc-12345678", method: "POST", path: "/api/example", + requestFingerprint: OK_BODY_FINGERPRINT, responseStatus: 0, responseBody: "", expiresAt: new Date(Date.now() + 60_000), @@ -928,3 +944,195 @@ describe("the record-session fence sits above the replay cache", () => { expect(handler).toHaveBeenCalledTimes(1); }); }); + +describe("request fingerprint", () => { + const enc = (text: string) => new TextEncoder().encode(text); + + it("sorts object keys recursively and keeps array order", () => { + expect(canonicalJson({ b: 1, a: { d: [3, 1], c: null } })).toBe( + '{"a":{"c":null,"d":[3,1]},"b":1}', + ); + expect(canonicalJson([{ y: 1, x: 2 }, "s"])).toBe('[{"x":2,"y":1},"s"]'); + }); + + it("treats the same JSON object in a different key order as one request", () => { + const a = requestFingerprint("POST", "/p", enc('{"type":"X","value":1}')); + const b = requestFingerprint( + "POST", + "/p", + enc('{ "value": 1, "type": "X" }'), + ); + expect(a).toBe(b); + expect(a).toMatch(/^[0-9a-f]{64}$/); + }); + + it("tells nested values and array order apart", () => { + const base = requestFingerprint("POST", "/p", enc('{"a":{"b":[1,2]}}')); + expect(requestFingerprint("POST", "/p", enc('{"a":{"b":[2,1]}}'))).not.toBe( + base, + ); + expect(requestFingerprint("POST", "/p", enc('{"a":{"b":[1,3]}}'))).not.toBe( + base, + ); + }); + + it("tells the two halves of a blood pressure apart", () => { + const sys = requestFingerprint( + "POST", + "/api/measurements", + enc('{"type":"BLOOD_PRESSURE_SYS","value":120}'), + ); + const dia = requestFingerprint( + "POST", + "/api/measurements", + enc('{"type":"BLOOD_PRESSURE_DIA","value":80}'), + ); + expect(sys).not.toBe(dia); + }); + + it("hashes a non-JSON or empty body as its raw bytes", () => { + const text = requestFingerprint("POST", "/p", enc("not json")); + expect(text).toBe(requestFingerprint("POST", "/p", enc("not json"))); + expect(text).not.toBe(requestFingerprint("POST", "/p", enc("not json!"))); + const empty = requestFingerprint("DELETE", "/p", new Uint8Array()); + expect(empty).toBe(requestFingerprint("DELETE", "/p", new Uint8Array())); + expect(empty).not.toBe(text); + // Invalid UTF-8 is raw bytes, not an error. + expect(() => + requestFingerprint("POST", "/p", new Uint8Array([0xff, 0xfe])), + ).not.toThrow(); + }); + + it("is scoped by method and path", () => { + const body = enc('{"a":1}'); + const base = requestFingerprint("POST", "/p", body); + expect(requestFingerprint("PUT", "/p", body)).not.toBe(base); + expect(requestFingerprint("POST", "/q", body)).not.toBe(base); + }); +}); + +describe("withIdempotency with a different body under one key", () => { + function postWith(body: unknown): NextRequest { + return new NextRequest("http://localhost/api/example", { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": "abc-12345678", + }, + body: JSON.stringify(body), + }); + } + + function liveRow(fingerprint: string | null, responseStatus: number) { + return { + id: "idem-1", + userId: "u-1", + key: "abc-12345678", + method: "POST", + path: "/api/example", + requestFingerprint: fingerprint, + responseStatus, + responseBody: "", + expiresAt: new Date(Date.now() + 60_000), + createdAt: new Date(), + } as never; + } + + it.each([ + ["a completed row for another body", "f".repeat(64), 201], + ["an in-flight claim for another body", "f".repeat(64), 0], + ["a legacy row without a fingerprint", null, 201], + ])( + "runs the handler without claiming or caching over %s", + async (_label, fingerprint, status) => { + vi.mocked(prisma.idempotencyKey.findUnique).mockResolvedValueOnce( + liveRow(fingerprint, status), + ); + const handler = vi.fn(async () => + NextResponse.json({ data: "dia", error: null }, { status: 201 }), + ); + const wrapped = withIdempotency<[NextRequest]>( + handler, + async () => "u-1", + ); + const res = await wrapped(postWith({ type: "BLOOD_PRESSURE_DIA" })); + + expect(res.status).toBe(201); + expect(res.headers.get("X-Idempotent-Replay")).toBeNull(); + expect(handler).toHaveBeenCalledTimes(1); + expect(prisma.idempotencyKey.create).not.toHaveBeenCalled(); + expect(prisma.idempotencyKey.updateMany).not.toHaveBeenCalled(); + expect(prisma.idempotencyKey.deleteMany).not.toHaveBeenCalled(); + expect(prisma.idempotencyKey.delete).not.toHaveBeenCalled(); + }, + ); + + it("runs the handler when a racing claim for another body wins the insert", async () => { + vi.mocked(prisma.idempotencyKey.create).mockRejectedValueOnce( + Object.assign(new Error("unique"), { code: "P2002" }), + ); + // First lookup: nothing yet. Second, after the collision: the other body. + vi.mocked(prisma.idempotencyKey.findUnique) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(liveRow("f".repeat(64), 0)); + const handler = vi.fn(async () => + NextResponse.json({ data: "dia", error: null }, { status: 201 }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + const res = await wrapped(postWith({ type: "BLOOD_PRESSURE_DIA" })); + + expect(res.status).toBe(201); + expect(handler).toHaveBeenCalledTimes(1); + expect(prisma.idempotencyKey.updateMany).not.toHaveBeenCalled(); + }); + + it("stores the fingerprint, never the body, on the claim", async () => { + const handler = vi.fn(async () => + NextResponse.json({ data: "ok", error: null }, { status: 201 }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + await wrapped(postWith({ type: "BLOOD_PRESSURE_SYS", value: 123 })); + + const claim = ( + vi.mocked(prisma.idempotencyKey.create).mock.calls[0][0] as { + data: Record; + } + ).data; + expect(claim.requestFingerprint).toBe( + requestFingerprint( + "POST", + "/api/example", + new TextEncoder().encode( + JSON.stringify({ type: "BLOOD_PRESSURE_SYS", value: 123 }), + ), + ), + ); + expect(JSON.stringify(claim)).not.toContain("BLOOD_PRESSURE_SYS"); + }); + + it("leaves the body readable for the handler", async () => { + const handler = vi.fn(async (req: NextRequest) => + NextResponse.json({ data: await req.json(), error: null }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + const res = await wrapped(postWith({ type: "WEIGHT", value: 70 })); + expect((await res.json()).data).toEqual({ type: "WEIGHT", value: 70 }); + }); + + it("reads at most a bounded prefix of a large body and still hands the whole body on", async () => { + const big = "x".repeat(3 * 1024 * 1024); + const handler = vi.fn(async (req: NextRequest) => + NextResponse.json({ data: (await req.text()).length, error: null }), + ); + const wrapped = withIdempotency<[NextRequest]>(handler, async () => "u-1"); + const res = await wrapped( + new NextRequest("http://localhost/api/example", { + method: "POST", + headers: { "idempotency-key": "abc-12345678" }, + body: big, + }), + ); + expect((await res.json()).data).toBe(big.length); + expect(prisma.idempotencyKey.create).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/idempotency.ts b/src/lib/idempotency.ts index 70e2ddcbc..82d2cc0b6 100644 --- a/src/lib/idempotency.ts +++ b/src/lib/idempotency.ts @@ -11,6 +11,19 @@ * the record the request claims is folded into `key` (see `cellKey`), so the * tuple identifies a request to one person's record rather than a request by * one person. A caller acting only as themselves is unaffected, byte for byte. + * + * A cell also remembers WHICH request it answered: a SHA-256 of the canonical + * request body (`requestFingerprint`). A key names one request, not a slot, and + * a client that reuses a key for a second, different request must not be + * handed the first request's response. Native clients up to 1.1.0 sent the two + * halves of a manual blood pressure as two POSTs under one key; the second half + * replayed the first's 201 and was never written. A lookup therefore replays, + * or reports in-flight, only for the same body. A different body — or a row + * written before fingerprints existed — runs the handler normally without + * claiming or caching anything, and the existing cell stays for its own + * retries. This is deliberately not the IETF draft's 422: refusing would turn + * every affected client's second write into a visible error with no way for + * the person to recover the value. */ import type { NextRequest } from "next/server"; import { headers } from "next/headers"; @@ -109,6 +122,122 @@ export interface IdempotencyContext { key: string; method: string; path: string; + /** See `requestFingerprint`. Never the body itself. */ + fingerprint: string; +} + +/** + * How much of a body the fingerprint reads before it stops. The wrapper runs + * before the handler's own bounded readers and upload slots, so it must not + * buffer an arbitrarily large body on their behalf: reading the clone holds + * what it has read in the original's queue until the handler consumes it. + * Every JSON write the cache fronts is far below this; the one multipart route + * (document upload) is above it, and there the prefix plus the declared length + * identifies a retry of the same upload, with the route's own content hash as + * the duplicate check behind it. + */ +const FINGERPRINT_MAX_BYTES = 1024 * 1024; + +/** + * JSON with every object's keys sorted, recursively. Arrays keep their order — + * it is part of the request. Exported for its unit tests. + */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) { + return `[${value.map((v) => canonicalJson(v)).join(",")}]`; + } + if (value !== null && typeof value === "object") { + const entries = Object.keys(value) + .sort() + .map( + (k) => + `${JSON.stringify(k)}:${canonicalJson((value as Record)[k])}`, + ); + return `{${entries.join(",")}}`; + } + return JSON.stringify(value); +} + +/** + * The fingerprint of a request: SHA-256 hex over method, path and the body in + * canonical form. A body that parses as JSON is hashed as `canonicalJson`, so + * the same object serialised with a different key order is the same request; + * anything else (multipart, plain text, an empty body) is hashed as its raw + * bytes. `truncatedAt` marks a body cut at `FINGERPRINT_MAX_BYTES`, hashed as + * that raw prefix together with the declared length. Exported for unit tests. + */ +export function requestFingerprint( + method: string, + path: string, + body: Uint8Array, + truncatedAt: string | null = null, +): string { + const hash = createHash("sha256") + .update(method) + .update("\0") + .update(path) + .update("\0"); + if (truncatedAt !== null) { + return hash.update(`partial:${truncatedAt}\0`).update(body).digest("hex"); + } + if (body.length > 0) { + try { + const text = new TextDecoder("utf-8", { fatal: true }).decode(body); + return hash + .update("json:") + .update(canonicalJson(JSON.parse(text))) + .digest("hex"); + } catch { + // Not UTF-8 or not JSON — fall through to the raw bytes. + } + } + return hash.update("raw:").update(body).digest("hex"); +} + +/** + * Read the request's body from a clone, at most `FINGERPRINT_MAX_BYTES`, and + * fingerprint it. The original request keeps its body for the handler. + */ +async function fingerprintRequest( + request: Request | NextRequest, + path: string, +): Promise { + const stream = request.clone().body; + if (!stream) + return requestFingerprint(request.method, path, new Uint8Array()); + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + let truncated = false; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (total + value.byteLength > FINGERPRINT_MAX_BYTES) { + chunks.push(value.subarray(0, FINGERPRINT_MAX_BYTES - total)); + total = FINGERPRINT_MAX_BYTES; + truncated = true; + // Cancelling this branch of the tee stops it from queueing the rest of + // the body, and leaves the handler's branch intact. Not awaited: a tee + // branch's cancel settles only once BOTH branches are done, which is + // after the handler has run. + void reader.cancel().catch(() => {}); + break; + } + chunks.push(value); + total += value.byteLength; + } + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return requestFingerprint( + request.method, + path, + body, + truncated ? (request.headers.get("content-length") ?? "unknown") : null, + ); } function getIdempotencyKey(request: Request | NextRequest): string | null { @@ -125,10 +254,16 @@ function getIdempotencyKey(request: Request | NextRequest): string | null { * - `{ kind: "pending" }` — another request holds an in-flight claim * for this key; the caller must NOT run the * handler again. + * - `{ kind: "mismatch" }` — the live row answered a different body (or + * predates fingerprints); the caller runs the + * handler without claiming or caching. * - `null` — no live row; the caller may claim + run. */ type CacheLookup = - { kind: "replay"; response: NextResponse } | { kind: "pending" } | null; + | { kind: "replay"; response: NextResponse } + | { kind: "pending" } + | { kind: "mismatch" } + | null; /** * Look up the state of a (userId, key, method, path) tuple. Distinguishes @@ -157,6 +292,13 @@ async function findCached(ctx: IdempotencyContext): Promise { return null; } + // Checked before both the pending and the replay arm: a different request + // under the same key is neither in flight nor answered. A legacy row's null + // never equals a fingerprint. + if (row.requestFingerprint !== ctx.fingerprint) { + return { kind: "mismatch" }; + } + if (row.responseStatus === PENDING_STATUS) { // A concurrent request claimed this key and is still running its // side-effect. Signal the caller to refuse rather than double-execute. @@ -208,6 +350,7 @@ async function claimKey(ctx: IdempotencyContext): Promise { key: ctx.key, method: ctx.method, path: ctx.path, + requestFingerprint: ctx.fingerprint, responseStatus: PENDING_STATUS, responseBody: "", expiresAt: new Date(Date.now() + PENDING_TTL_MS), @@ -233,6 +376,7 @@ async function releaseClaim(ctx: IdempotencyContext): Promise { key: ctx.key, method: ctx.method, path: ctx.path, + requestFingerprint: ctx.fingerprint, responseStatus: PENDING_STATUS, }, }) @@ -312,6 +456,7 @@ async function persistCached( key: ctx.key, method: ctx.method, path: ctx.path, + requestFingerprint: ctx.fingerprint, }, data: { responseStatus: response.status, @@ -502,6 +647,23 @@ async function narrowTokenFacet(): Promise { return apiToken.permissions.includes("*") ? null : `t:${apiToken.id}`; } +/** + * Run a request whose key is already taken by a different body: the handler + * runs exactly as it would without a key, and nothing is claimed or cached, so + * the existing cell keeps answering its own retries. + */ +function runUncached( + ctx: IdempotencyContext, + handler: (...args: Args) => Promise, + args: Args, +): Promise { + annotate({ + action: { name: "idempotency.key.body_mismatch" }, + meta: { method: ctx.method, path: ctx.path }, + }); + return handler(...args); +} + export function withIdempotency< Args extends [Request | NextRequest, ...unknown[]], >( @@ -589,14 +751,27 @@ export function withIdempotency< if (tokenFacet !== null) facets.push(tokenFacet); const url = new URL(request.url); + const fingerprint = await fingerprintRequest(request, url.pathname).catch( + () => null, + ); + if (fingerprint === null) { + // The body could not be read for the fingerprint (already consumed, or + // the stream failed). Without knowing which request this is, no cell + // may answer for it; the handler meets the body problem itself. + return handler(...args); + } const ctx: IdempotencyContext = { userId, key: cellKey(key, claimedRecord, facets), method: request.method, path: url.pathname, + fingerprint, }; const cached = await findCached(ctx); + if (cached?.kind === "mismatch") { + return runUncached(ctx, handler, args); + } if (cached?.kind === "replay") { return cached.response; } @@ -617,6 +792,13 @@ export function withIdempotency< // one caller may execute the side-effect for a given key. const won = await claimKey(ctx); if (!won) { + // The racing claim may be a different request under the same key — + // the two halves of one reading posted together. That one is not a + // duplicate of this one and must still run. + const raced = await findCached(ctx); + if (raced?.kind === "mismatch") { + return runUncached(ctx, handler, args); + } annotate({ action: { name: "idempotency.inflight_conflict" }, meta: { method: ctx.method, path: ctx.path }, diff --git a/src/lib/openapi/routes/shared.ts b/src/lib/openapi/routes/shared.ts index c96613bc0..831933630 100644 --- a/src/lib/openapi/routes/shared.ts +++ b/src/lib/openapi/routes/shared.ts @@ -391,7 +391,8 @@ export const profileEmailRateLimitResponse = { * * Scope, so nobody reads more into it than is there: only POST, PUT, PATCH and * DELETE are covered (`SUPPORTED_METHODS`), the key is scoped per user and per - * `(method, path)`, and `isCachableStatus` refuses to cache 5xx or the + * `(method, path)`, a replay needs the same body (a different body under the + * same key runs as a new write and is neither replayed nor refused), and `isCachableStatus` refuses to cache 5xx or the * transient 401 / 403 / 408 / 429 — a retry after one of those runs for real. * * `src/__tests__/idempotency-contract-publication.test.ts` holds this to the @@ -410,7 +411,7 @@ export const idempotencyKeyParameter = { maxLength: 128, }, description: - "Makes this write safe to retry. Send the same key again and the first response is replayed rather than the work repeated, marked with `X-Idempotent-Replay: true`. A key that does not match the pattern is IGNORED rather than refused — the write then proceeds unprotected and looks exactly like a request that carried no key, so validate the key on your side. Scoped per user and per (method, path): the same key on a different route is a different key. While a first request under this key is still in flight, a second is refused with 409 rather than queued.", + "Makes this write safe to retry. Send the same key with the same body again and the first response is replayed rather than the work repeated, marked with `X-Idempotent-Replay: true`. The body is compared as JSON with key order ignored; a different body under the same key is not a retry and runs as a new write, while the first response stays stored for retries of the first request. A key that does not match the pattern is IGNORED rather than refused — the write then proceeds unprotected and looks exactly like a request that carried no key, so validate the key on your side. Scoped per user and per (method, path): the same key on a different route is a different key. While a first request under this key is still in flight, a second with the same body is refused with 409 rather than queued.", }; /** diff --git a/tests/integration/idempotency-body-fingerprint.test.ts b/tests/integration/idempotency-body-fingerprint.test.ts new file mode 100644 index 000000000..c0d04d93c --- /dev/null +++ b/tests/integration/idempotency-body-fingerprint.test.ts @@ -0,0 +1,249 @@ +/** + * One `Idempotency-Key`, two different requests. + * + * Native clients up to 1.1.0 post a manual blood pressure as two + * `POST /api/measurements` calls — the systolic half, then the diastolic half — + * under the same key. While the replay cell was keyed on + * `(user, key, method, path)` alone, the second call was answered with the + * first call's cached 201 and the diastolic value was never written. The cell + * now carries a fingerprint of the request body: only the same body replays. + * + * Everything here is real — the shipped route, the shipped wrapper, the + * session resolver and the `idempotency_keys` unique index. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { POST as postMeasurement } from "@/app/api/measurements/route"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const USER_A = "idem-fingerprint-a"; +const USER_B = "idem-fingerprint-b"; +const KEY = "bp-entry-5c1e2a7f"; +const MEASURED_AT = "2026-09-30T07:15:00.000Z"; + +const SYS = { type: "BLOOD_PRESSURE_SYS", value: 128, measuredAt: MEASURED_AT }; +const DIA = { type: "BLOOD_PRESSURE_DIA", value: 84, measuredAt: MEASURED_AT }; + +const sessions = new Map(); + +function post(body: unknown, key = KEY): Promise { + return postMeasurement( + new NextRequest("http://localhost/api/measurements", { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": key, + }, + body: JSON.stringify(body), + }), + ); +} + +function signInAs(userId: string) { + cookieJar.set("healthlog_session", sessions.get(userId)!); +} + +async function rowsOf(userId: string) { + const rows = await getPrismaClient().measurement.findMany({ + where: { userId, deletedAt: null }, + select: { type: true, value: true }, + }); + // By name, not by the enum's declaration order. + return rows.sort((a, b) => a.type.localeCompare(b.type)); +} + +beforeEach(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + sessions.clear(); + for (const id of [USER_A, USER_B]) { + await prisma.user.create({ + data: { + id, + username: id, + email: `${id}@example.test`, + timezone: "UTC", + }, + }); + const session = await prisma.session.create({ + data: { userId: id, expiresAt: new Date(Date.now() + 60 * 60 * 1000) }, + }); + sessions.set(id, session.id); + } + signInAs(USER_A); +}); + +describe("Idempotency-Key reused for a different body (real Postgres)", () => { + it("writes both halves of a blood pressure posted under one key", async () => { + const first = await post(SYS); + expect(first.status).toBe(201); + + const second = await post(DIA); + expect(second.status).toBe(201); + expect(second.headers.get("X-Idempotent-Replay")).toBeNull(); + + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_DIA", value: 84 }, + { type: "BLOOD_PRESSURE_SYS", value: 128 }, + ]); + }); + + it("still replays a retry of the same body and writes one row", async () => { + const first = await post(SYS); + expect(first.status).toBe(201); + const firstBody = await first.json(); + + // Same object, different key order: the same request. + const retry = await post({ + measuredAt: MEASURED_AT, + value: 128, + type: "BLOOD_PRESSURE_SYS", + }); + expect(retry.status).toBe(201); + expect(retry.headers.get("X-Idempotent-Replay")).toBe("true"); + expect(await retry.json()).toEqual(firstBody); + + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_SYS", value: 128 }, + ]); + }); + + it("keeps the first cell answering its own retries after a different body ran", async () => { + await post(SYS); + await post(DIA); + const retry = await post(SYS); + expect(retry.headers.get("X-Idempotent-Replay")).toBe("true"); + expect(await rowsOf(USER_A)).toHaveLength(2); + const cells = await getPrismaClient().idempotencyKey.findMany({ + where: { userId: USER_A }, + }); + expect(cells).toHaveLength(1); + expect(cells[0]?.requestFingerprint).toMatch(/^[0-9a-f]{64}$/); + }); + + it("writes both halves when they are in flight at the same time", async () => { + const [a, b] = await Promise.all([post(SYS), post(DIA)]); + expect([a.status, b.status]).toEqual([201, 201]); + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_DIA", value: 84 }, + { type: "BLOOD_PRESSURE_SYS", value: 128 }, + ]); + }); + + it("still lets only one of two identical bodies in flight run", async () => { + const [a, b] = await Promise.all([post(SYS), post(SYS)]); + expect([a.status, b.status].sort()).toEqual([201, 409]); + expect(await rowsOf(USER_A)).toHaveLength(1); + }); + + it("does not replay a row written before fingerprints existed", async () => { + await getPrismaClient().idempotencyKey.create({ + data: { + userId: USER_A, + key: KEY, + method: "POST", + path: "/api/measurements", + responseStatus: 201, + responseBody: JSON.stringify({ data: { id: "legacy" }, error: null }), + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + }, + }); + + const res = await post(DIA); + expect(res.status).toBe(201); + expect(res.headers.get("X-Idempotent-Replay")).toBeNull(); + expect(await rowsOf(USER_A)).toEqual([ + { type: "BLOOD_PRESSURE_DIA", value: 84 }, + ]); + // The legacy row is left alone, not claimed over. + const cell = await getPrismaClient().idempotencyKey.findFirstOrThrow({ + where: { userId: USER_A, key: KEY }, + }); + expect(cell.requestFingerprint).toBeNull(); + }); + + it("never hands one user's cached response to another using the same key", async () => { + const first = await post(SYS); + const firstBody = await first.json(); + + signInAs(USER_B); + const other = await post(SYS); + expect(other.status).toBe(201); + expect(other.headers.get("X-Idempotent-Replay")).toBeNull(); + expect(await other.json()).not.toEqual(firstBody); + + expect(await rowsOf(USER_A)).toHaveLength(1); + expect(await rowsOf(USER_B)).toHaveLength(1); + }); + + it("the documented operator query finds a systolic reading left without its diastolic half", async () => { + const doc = readFileSync( + join(process.cwd(), "docs/ops/blood-pressure-missing-diastolic.md"), + "utf8", + ); + const block = + /([\s\S]*?)/.exec( + doc, + ); + const sql = block?.[1] + .replace(/```sql|```/g, "") + .trim() + .replace(/;$/, ""); + expect(sql).toBeTruthy(); + + // A complete reading through the fixed path, and the damage the old + // cache left behind: a systolic half on its own. + await post(SYS); + await post(DIA); + const lone = await getPrismaClient().measurement.create({ + data: { + userId: USER_A, + type: "BLOOD_PRESSURE_SYS", + value: 141, + unit: "mmHg", + measuredAt: new Date("2026-09-29T19:40:00.000Z"), + }, + }); + + const found = await getPrismaClient().$queryRawUnsafe< + Array<{ id: string; systolic: number }> + >(sql!); + expect(found.map((r) => [r.id, r.systolic])).toEqual([[lone.id, 141]]); + }); +}); From b63fa317f388038566b246f2a27be2d4496f2364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:37:23 +0200 Subject: [PATCH 2/5] fix(docker): hoist the canvas binary of the pinned version Refs #1124. The store holds two @napi-rs/canvas versions, the pinned one and an older one that pdf-parse's own pdfjs-dist pulls in. The runner stage took the musl binary with `find | head -1`, got the old one, and copied it into the new loader's directory, where the loader looks first. Rendering still worked, but `loadImage` calls `Image.decode`, which the old binary lacks, so every preview thumbnail failed with "image.decode is not a function" on both architectures. The builder now records the resolved canvas version and the runner hoists that version by name, as it already does for pdfjs-dist. The build stops when the binary it copies belongs to another version. --- Dockerfile | 51 +++++++++++---- .../canvas-binary-hoist-guard.test.ts | 65 +++++++++++++++++++ 2 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 src/__tests__/canvas-binary-hoist-guard.test.ts diff --git a/Dockerfile b/Dockerfile index 6b871db84..20fe839da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,9 @@ RUN NODE_OPTIONS=--max-old-space-size=8192 pnpm build # name; see the note there for why guessing was wrong. RUN node -p "require('/app/node_modules/pdfjs-dist/package.json').version" \ > /app/.pdfjs-version +# The same for @napi-rs/canvas, whose native binary must match its JS loader. +RUN node -p "require('/app/node_modules/@napi-rs/canvas/package.json').version" \ + > /app/.canvas-version # ── Stage 3: Production runner ───────────────────────────── FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2 AS runner @@ -120,21 +123,41 @@ RUN node -e "require.resolve('pg-boss'); require.resolve('pg')" # @napi-rs/canvas" → "Cannot polyfill DOMMatrix" → rasterize ReferenceError); # (2) @napi-rs/canvas's own loader then resolves its platform binary. Hoist BOTH # the canvas package and the musl binary package to /app/node_modules/@napi-rs -# via symlink into the .pnpm store, and also drop the prebuilt .node into the -# canvas dir so the loader's local-file fallback (`require('./skia..node')`) -# is belt-and-suspenders. Per-arch buildx installs only the matching musl triple. +# via symlink into the .pnpm store, and drop the prebuilt .node into the canvas +# dir: the loader tries that local file (`require('./skia..node')`) +# before the platform package, so it decides which binary runs. +# +# Hoist the version the app pins, by name, exactly like pdfjs-dist below. The +# store holds a second, older canvas that pdf-parse's own pdfjs-dist pulls in, +# and this step used to take whichever `find | head -1` returned first. It +# returned the old binary, which then sat next to the new loader: rendering +# still worked, but `loadImage` called an `Image.decode` the old binary does +# not have, and every preview thumbnail failed with "image.decode is not a +# function". Name the version, and fail the build when the binary that lands +# in the canvas dir is not that version's. +COPY --from=builder /app/.canvas-version /tmp/.canvas-version RUN set -e; \ - CANVAS_DIR="$(find /app/node_modules/.pnpm -maxdepth 4 -type d -path '*@napi-rs+canvas@*/node_modules/@napi-rs/canvas' 2>/dev/null | head -1)"; \ - MUSL_DIR="$(find /app/node_modules/.pnpm -maxdepth 4 -type d -path '*@napi-rs+canvas-linux-*-musl@*/node_modules/@napi-rs/canvas-linux-*-musl' 2>/dev/null | head -1)"; \ - NODE_BIN="$(find /app/node_modules/.pnpm -maxdepth 5 -name 'skia.linux-*-musl.node' 2>/dev/null | head -1)"; \ - if [ -n "$CANVAS_DIR" ]; then \ - mkdir -p /app/node_modules/@napi-rs; \ - ln -sfn "$CANVAS_DIR" /app/node_modules/@napi-rs/canvas; \ - [ -n "$MUSL_DIR" ] && ln -sfn "$MUSL_DIR" "/app/node_modules/@napi-rs/$(basename "$MUSL_DIR")"; \ - [ -n "$NODE_BIN" ] && cp "$NODE_BIN" "$CANVAS_DIR/"; \ - chown -R nextjs:nodejs /app/node_modules/@napi-rs; \ - echo "canvas hoisted for pdfjs: $CANVAS_DIR (musl=$MUSL_DIR)"; \ - else echo "WARN: @napi-rs/canvas not found; PDF rasterization will degrade to local text"; fi + CANVAS_VERSION="$(cat /tmp/.canvas-version)"; \ + rm -f /tmp/.canvas-version; \ + CANVAS_DIR="/app/node_modules/.pnpm/@napi-rs+canvas@${CANVAS_VERSION}/node_modules/@napi-rs/canvas"; \ + MUSL_DIR="$(find /app/node_modules/.pnpm -maxdepth 4 -type d -path "*@napi-rs+canvas-linux-*-musl@${CANVAS_VERSION}/node_modules/@napi-rs/canvas-linux-*-musl" 2>/dev/null | head -1)"; \ + if [ ! -d "$CANVAS_DIR" ] || [ -z "$MUSL_DIR" ]; then \ + echo "ERROR: pinned @napi-rs/canvas@${CANVAS_VERSION} or its musl binary did not reach the traced image."; \ + echo "Present instead:"; \ + find /app/node_modules/.pnpm -maxdepth 1 -name '@napi-rs+canvas*' 2>/dev/null; \ + exit 1; \ + fi; \ + NODE_BIN="$(find "$MUSL_DIR" -maxdepth 1 -name 'skia.linux-*-musl.node' | head -1)"; \ + [ -n "$NODE_BIN" ] || { echo "ERROR: no skia binary in $MUSL_DIR"; exit 1; }; \ + mkdir -p /app/node_modules/@napi-rs; \ + ln -sfn "$CANVAS_DIR" /app/node_modules/@napi-rs/canvas; \ + ln -sfn "$MUSL_DIR" "/app/node_modules/@napi-rs/$(basename "$MUSL_DIR")"; \ + cp "$NODE_BIN" "$CANVAS_DIR/"; \ + chown -R nextjs:nodejs /app/node_modules/@napi-rs; \ + BIN_VERSION="$(node -p "require('$MUSL_DIR/package.json').version")"; \ + [ "$BIN_VERSION" = "$CANVAS_VERSION" ] || { echo "ERROR: canvas binary is $BIN_VERSION, loader is $CANVAS_VERSION"; exit 1; }; \ + cmp -s "$NODE_BIN" "$CANVAS_DIR/$(basename "$NODE_BIN")" || { echo "ERROR: the canvas dir holds a different binary"; exit 1; }; \ + echo "canvas hoisted for pdfjs: $CANVAS_DIR (binary $BIN_VERSION from $MUSL_DIR)" # pdfjs-dist is `serverExternalPackages` (NOT bundled — the bundled copy's render # path breaks in the standalone image), so the server does a runtime bare diff --git a/src/__tests__/canvas-binary-hoist-guard.test.ts b/src/__tests__/canvas-binary-hoist-guard.test.ts new file mode 100644 index 000000000..d230fab26 --- /dev/null +++ b/src/__tests__/canvas-binary-hoist-guard.test.ts @@ -0,0 +1,65 @@ +/** + * Structural guard: the runtime image puts the @napi-rs/canvas native binary + * of the version the app pins next to that version's loader. + * + * ## The failure this freezes (GitHub #1124) + * + * The pnpm store holds two canvas versions: the one in package.json and an + * older one that pdf-parse's own pdfjs-dist pulls in. The runner stage picked + * the musl binary with `find | head -1`, got the old one, and copied it into + * the new loader's directory, where the loader looks first. PDF rendering + * still worked, so nothing looked wrong; `loadImage` called an + * `Image.decode` the old binary lacks, and every preview thumbnail failed + * with "image.decode is not a function" on both architectures. + * + * ## What it proves and what it does not + * + * The builder records the resolved canvas version, the runner hoists by that + * name, and the build stops when the copied binary's package is a different + * version. Whether the binary loads is proved when the image runs; this file + * reads the Dockerfile only. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +const DOCKERFILE = readFileSync(join(process.cwd(), "Dockerfile"), "utf8"); + +function stage(name: string): string { + const start = DOCKERFILE.search(new RegExp(`^FROM .* AS ${name}$`, "m")); + expect(start, `the Dockerfile has a ${name} stage`).toBeGreaterThan(0); + const next = DOCKERFILE.slice(start + 1).search(/^FROM /m); + return next < 0 + ? DOCKERFILE.slice(start) + : DOCKERFILE.slice(start, start + 1 + next); +} + +describe("canvas binary hoist", () => { + it("records the resolved canvas version in the builder", () => { + expect(stage("builder")).toMatch( + /require\('\/app\/node_modules\/@napi-rs\/canvas\/package\.json'\)\.version"[\s\\]*>\s*\/app\/\.canvas-version/, + ); + }); + + it("hoists the loader and the musl binary of that version only", () => { + const runner = stage("runner"); + expect(runner).toContain( + "COPY --from=builder /app/.canvas-version /tmp/.canvas-version", + ); + expect(runner).toContain( + 'CANVAS_DIR="/app/node_modules/.pnpm/@napi-rs+canvas@${CANVAS_VERSION}/', + ); + expect(runner).toContain("canvas-linux-*-musl@${CANVAS_VERSION}/"); + // A version-blind pick is the defect: any `canvas…@*` glob decides by + // readdir order which of the store's copies wins. + expect(runner).not.toMatch(/@napi-rs\+canvas[^"'\s]*@\*/); + }); + + it("fails the build when the copied binary is another version", () => { + const runner = stage("runner"); + expect(runner).toMatch( + /\[ "\$BIN_VERSION" = "\$CANVAS_VERSION" \] \|\| \{[^}]*exit 1; \}/, + ); + }); +}); From 4b73099e75bdc87a4e36cb331a748861bf50aae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:37:24 +0200 Subject: [PATCH 3/5] fix(documents): render PDFs with pdfjs font data and refuse blank pages Refs #1124. The rasterizer opened pdfjs without its standard font, CMap and wasm directories and told it to use system fonts. The runtime image has none, so any text pdfjs could not draw from an embedded program (a non-embedded standard font, or an embedded one it cannot parse) came out as nothing, and an image-only provider was sent the page background. A workstation hides this because its own fonts stand in. pdfjs now gets the directories from its own package, resolved at runtime from the working directory, and Liberation Sans from that package is registered as the canvas' sans-serif, serif and monospace faces for the text pdfjs draws with a substitute. A document whose every rendered page is blank now reports render-failed, so the read records raster-failed instead of indexing a transcript of an empty page as a success. --- .../prepare-vision-raster-failure.test.ts | 15 ++ .../__tests__/rasterize-pdf-cleanup.test.ts | 9 +- .../__tests__/rasterize-pdf-font-data.test.ts | 155 ++++++++++++++ src/lib/documents/rasterize-pdf.ts | 189 +++++++++++++++++- tests/fixtures/pdf/fallback-fonts.pdf | Bin 0 -> 1801 bytes tests/fixtures/pdf/invisible-text-only.pdf | Bin 0 -> 747 bytes .../pdf/ocrmypdf11-two-images-type1c.pdf | Bin 0 -> 43948 bytes 7 files changed, 358 insertions(+), 10 deletions(-) create mode 100644 src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts create mode 100644 tests/fixtures/pdf/fallback-fonts.pdf create mode 100644 tests/fixtures/pdf/invisible-text-only.pdf create mode 100644 tests/fixtures/pdf/ocrmypdf11-two-images-type1c.pdf diff --git a/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts b/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts index 05af83fdc..3e01f2d75 100644 --- a/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts +++ b/src/lib/documents/__tests__/prepare-vision-raster-failure.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { Buffer } from "node:buffer"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; /** * Refs #776 — the raster-failed chain, end to end through the REAL @@ -68,4 +70,17 @@ describe("raster-failed end to end (#776)", () => { const vision = await prepareVisionInput(DOC as never, true); expect(vision).toMatchObject({ ok: true }); }); + + it("a PDF whose pages all render blank is rasterFailed, not an image of nothing (#1124)", async () => { + // Text in render mode 3 only: pdfjs opens it and draws an empty page. + // Before the blank-page guard this went to the provider as one white + // JPEG and came back as a near-empty transcript indexed as a success. + vi.mocked(decryptDocumentContent).mockReturnValue( + readFileSync( + join(process.cwd(), "tests/fixtures/pdf/invisible-text-only.pdf"), + ) as never, + ); + const vision = await prepareVisionInput(DOC as never, false); + expect(vision).toEqual({ ok: false, reason: "rasterFailed" }); + }); }); diff --git a/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts b/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts index 4e0d2cdba..bdafb0bfa 100644 --- a/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts +++ b/src/lib/documents/__tests__/rasterize-pdf-cleanup.test.ts @@ -18,9 +18,16 @@ vi.mock("pdfjs-dist/legacy/build/pdf.mjs", () => ({ canvasFactory: { create: () => ({ canvas: { + width: 1, + height: 1, toBuffer: () => Buffer.from([0xff, 0xd8]), }, - context: {}, + // One black pixel: an inked page, so the blank-page guard passes. + context: { + getImageData: () => ({ + data: new Uint8ClampedArray([0, 0, 0, 255]), + }), + }, }), }, getPage: async () => ({ diff --git a/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts b/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts new file mode 100644 index 000000000..86fd7c5fe --- /dev/null +++ b/src/lib/documents/__tests__/rasterize-pdf-font-data.test.ts @@ -0,0 +1,155 @@ +import { Buffer } from "node:buffer"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { createCanvas, loadImage } from "@napi-rs/canvas"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * GitHub #1124 — PDFs from older OCRmyPDF archives got no thumbnail and an + * image-only vision read of them came back nearly empty. + * + * The runtime image has no system fonts. pdfjs was opened without its own + * font data and told to lean on system fonts, so any text it could not draw + * from an embedded program (a non-embedded standard font, or an embedded + * program it cannot parse) came out as nothing, and the page that reached + * the provider was the background alone. A workstation hides this because + * its own fonts stand in, so the rendering assertions below are regression + * guards here and the container is where they were proven red; the option + * and registration assertions are red on any host. + * + * Fixtures (`tests/fixtures/pdf/`, synthetic text only): + * - `ocrmypdf11-two-images-type1c.pdf` — the reporter's shape: PDF 1.7, + * pikepdf producer, one 630.5 x 861.6 pt page with a 150 ppi RGB JPEG and + * a 75 ppi RGB Flate image, Type 1C Helvetica and Palatino-Roman, and an + * invisible OCR text layer. + * - `fallback-fonts.pdf` — a letter in non-embedded Helvetica and in a + * Type 1C Palatino-Roman whose program pdfjs cannot parse. + * - `invisible-text-only.pdf` — text in render mode 3 and nothing else, so + * every page renders blank. + */ + +const { annotate, registerFromPath } = vi.hoisted(() => ({ + annotate: vi.fn(), + registerFromPath: vi.fn(), +})); +vi.mock("@/lib/logging/context", () => ({ annotate })); + +const getDocumentOptions: Record[] = []; +vi.mock("pdfjs-dist/legacy/build/pdf.mjs", async (importActual) => { + const actual = + await importActual(); + return { + ...actual, + getDocument: (opts: Record) => { + getDocumentOptions.push(opts); + return actual.getDocument(opts as never); + }, + }; +}); +vi.mock("@napi-rs/canvas", async (importActual) => { + const actual = await importActual(); + return { + ...actual, + GlobalFonts: { + ...actual.GlobalFonts, + registerFromPath: (path: string, alias?: string) => { + registerFromPath(path, alias); + return actual.GlobalFonts.registerFromPath(path, alias); + }, + }, + }; +}); + +import * as raster from "../rasterize-pdf"; +import { generateThumbnail } from "../thumbnail"; + +const FIXTURES = join(process.cwd(), "tests", "fixtures", "pdf"); +const fixture = (name: string) => readFileSync(join(FIXTURES, name)); + +/** Share of dark pixels in a JPEG, measured the way the rasterizer does. */ +async function inkOf(base64: string): Promise { + const image = await loadImage(Buffer.from(base64, "base64")); + const canvas = createCanvas(image.width, image.height); + const ctx = canvas.getContext("2d"); + ctx.drawImage(image, 0, 0); + const { data } = ctx.getImageData(0, 0, image.width, image.height); + let dark = 0; + for (let i = 0; i < data.length; i += 4) { + if (data[i]! + data[i + 1]! + data[i + 2]! < 384) dark++; + } + return dark / (data.length / 4); +} + +beforeEach(() => { + annotate.mockClear(); + registerFromPath.mockClear(); + getDocumentOptions.length = 0; + raster.__resetPdfjsAssetsForTests(); +}); + +describe("rasterizePdf font and decoder data (#1124)", () => { + it("gives pdfjs its font, CMap and wasm directories and no system fonts", async () => { + const result = await raster.rasterizePdf(fixture("fallback-fonts.pdf"), 1); + expect(result.ok).toBe(true); + const opts = getDocumentOptions.at(-1)!; + expect(opts.useSystemFonts).toBe(false); + for (const [key, probe] of [ + ["standardFontDataUrl", "FoxitSerif.pfb"], + ["standardFontDataUrl", "LiberationSans-Regular.ttf"], + ["cMapUrl", "UniJIS-UTF16-H.bcmap"], + ["wasmUrl", "openjpeg.wasm"], + ] as const) { + expect(typeof opts[key], key).toBe("string"); + expect(existsSync(`${opts[key] as string}${probe}`), probe).toBe(true); + } + expect(opts.cMapPacked).toBe(true); + }); + + it("registers a fallback face for every generic family pdfjs falls back to", async () => { + await raster.rasterizePdf(fixture("fallback-fonts.pdf"), 1); + const families = new Set(registerFromPath.mock.calls.map((c) => c[1])); + expect(families).toEqual(new Set(["sans-serif", "serif", "monospace"])); + for (const [path] of registerFromPath.mock.calls) { + expect(existsSync(path as string)).toBe(true); + } + }); + + it("renders the OCRmyPDF page with its scan and its text", async () => { + const result = await raster.rasterizePdf( + fixture("ocrmypdf11-two-images-type1c.pdf"), + ); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(1); + expect(await inkOf(result.images[0]!.dataBase64)).toBeGreaterThan(0.005); + }); + + it("renders a letter whose fonts pdfjs has to substitute", async () => { + const result = await raster.rasterizePdf(fixture("fallback-fonts.pdf")); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(await inkOf(result.images[0]!.dataBase64)).toBeGreaterThan(0.001); + }); + + it("reports a document whose every page renders blank as render-failed", async () => { + const result = await raster.rasterizePdf( + fixture("invisible-text-only.pdf"), + ); + expect(result).toEqual({ ok: false, reason: "render-failed" }); + expect(annotate).toHaveBeenCalledWith({ + action: { name: "documents.rasterize.failed" }, + meta: { reason: "blank_pages", pages: 1 }, + }); + }); +}); + +describe("generateThumbnail on the OCRmyPDF shape (#1124)", () => { + it("produces a preview of the first page", async () => { + const result = await generateThumbnail( + fixture("ocrmypdf11-two-images-type1c.pdf"), + "application/pdf", + ); + expect(result.ok).toBe(true); + }); +}); diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts index f61e74fc3..452ddfae5 100644 --- a/src/lib/documents/rasterize-pdf.ts +++ b/src/lib/documents/rasterize-pdf.ts @@ -31,6 +31,9 @@ * preserves the "a bad document never aborts an upload/batch" contract. */ import { Buffer } from "node:buffer"; +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; import { annotate } from "@/lib/logging/context"; @@ -60,6 +63,14 @@ export const RASTER_TARGET_LONG_EDGE = 1600; /** JPEG quality (0-100). ~80 keeps text legible at a fraction of PNG's bytes. */ export const RASTER_JPEG_QUALITY = 80; +/** + * A rendered page counts as blank when fewer than this share of its pixels + * are dark. One short line of 16 pt text on an A4 page is ~0.1 %; a page + * number alone sits near this floor. A page set that is blank throughout is + * not a reading of the document, whatever the provider would make of it. + */ +export const RASTER_MIN_INK_RATIO = 0.0001; + /** One rasterized page, shaped as a vision `input_image` part. */ export interface RasterImage { mediaType: "image/jpeg"; @@ -98,9 +109,17 @@ interface PdfCanvas { height: number; toBuffer(mime: "image/jpeg", quality?: number): Buffer; } +interface PdfContext2D { + getImageData( + x: number, + y: number, + width: number, + height: number, + ): { data: Uint8ClampedArray }; +} interface PdfCanvasAndContext { canvas: PdfCanvas; - context: unknown; + context: PdfContext2D; } interface PdfCanvasFactory { create(width: number, height: number): PdfCanvasAndContext; @@ -125,12 +144,127 @@ interface PdfLoadingTask { destroy(): Promise; } interface PdfjsModule { - getDocument(opts: { - data: Uint8Array; - verbosity?: number; - isEvalSupported?: boolean; - useSystemFonts?: boolean; - }): PdfLoadingTask; + getDocument(opts: PdfDocumentOptions): PdfLoadingTask; +} +interface PdfDocumentOptions { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + standardFontDataUrl?: string; + cMapUrl?: string; + cMapPacked?: boolean; + wasmUrl?: string; +} +interface GlobalFontsLike { + registerFromPath(path: string, nameAlias?: string): unknown; +} + +/** + * Where pdfjs finds the data it does not carry in its JS: the standard 14 + * fonts (a PDF that names Helvetica or Times without embedding them), the + * CMaps (CID fonts with a predefined encoding), and the wasm image decoders + * (JPEG 2000, JBIG2). In a browser pdfjs fetches these from a URL; under Node + * they are read from disk, and without a directory pdfjs simply draws the text + * without glyphs. + * + * The runtime image carries no system fonts at all, so nothing else catches + * that: a letter set in a non-embedded font rendered as an empty page in the + * container while it rendered correctly on a workstation, where the operating + * system's fonts stood in. The image-only provider then read the empty page + * and returned a line of nothing. + * + * Resolved from the working directory with Node's own resolution, which is + * what finds the hoisted `node_modules/pdfjs-dist` in the standalone image + * and the pnpm link in a checkout. The specifier is assembled at runtime so + * the bundler leaves the lookup alone. The ICC profile option is not set: its + * loader needs a synchronous XMLHttpRequest that Node lacks, and pdfjs falls + * back to each colour space's alternate. + */ +export interface PdfjsAssets { + standardFontDataUrl: string; + cMapUrl: string; + wasmUrl: string; + /** The Liberation Sans faces registered as the generic fallback families. */ + fallbackFonts: string[]; +} + +let cachedAssets: PdfjsAssets | null | undefined; + +export function resolvePdfjsAssets(): PdfjsAssets | null { + if (cachedAssets !== undefined) return cachedAssets; + cachedAssets = null; + try { + const anchor = createRequire(join(process.cwd(), "package.json")); + const root = dirname( + anchor.resolve(["pdfjs-dist", "package.json"].join("/")), + ); + const fonts = join(root, "standard_fonts"); + const assets: PdfjsAssets = { + standardFontDataUrl: `${fonts}/`, + cMapUrl: `${join(root, "cmaps")}/`, + wasmUrl: `${join(root, "wasm")}/`, + fallbackFonts: [ + "LiberationSans-Regular.ttf", + "LiberationSans-Bold.ttf", + "LiberationSans-Italic.ttf", + "LiberationSans-BoldItalic.ttf", + ].map((file) => join(fonts, file)), + }; + if ( + existsSync(join(fonts, "FoxitSerif.pfb")) && + existsSync(join(root, "cmaps", "UniJIS-UTF16-H.bcmap")) && + existsSync(join(root, "wasm", "openjpeg.wasm")) && + assets.fallbackFonts.every((file) => existsSync(file)) + ) { + cachedAssets = assets; + } + } catch { + // Unresolvable: render without the data, as before, and say so below. + } + return cachedAssets; +} + +let fallbackFamiliesRegistered = false; + +/** + * Give the canvas a face for the generic families pdfjs falls back to. + * + * When pdfjs cannot use a font program (a damaged or unsupported embedded + * font, or a non-embedded face outside the standard 14) it draws the text + * with the canvas' own `fillText` under the PDF's font name and a generic + * family. With no system fonts in the image that text vanished. Registering + * Liberation Sans, which pdfjs ships, under `sans-serif`, `serif` and + * `monospace` makes it legible instead. Once per process. + */ +async function registerFallbackFamilies(assets: PdfjsAssets): Promise { + if (fallbackFamiliesRegistered) return; + fallbackFamiliesRegistered = true; + const { GlobalFonts } = (await import("@napi-rs/canvas")) as unknown as { + GlobalFonts: GlobalFontsLike; + }; + for (const family of ["sans-serif", "serif", "monospace"]) { + for (const file of assets.fallbackFonts) { + GlobalFonts.registerFromPath(file, family); + } + } +} + +/** Share of dark pixels on a rendered page (luminance below half). */ +function inkRatio(context: PdfContext2D, width: number, height: number) { + const { data } = context.getImageData(0, 0, width, height); + let dark = 0; + for (let i = 0; i < data.length; i += 4) { + if (data[i]! + data[i + 1]! + data[i + 2]! < 384) dark++; + } + const pixels = data.length / 4; + return pixels > 0 ? dark / pixels : 0; +} + +/** Test hook: the asset lookup and the font registration are per process. */ +export function __resetPdfjsAssetsForTests(): void { + cachedAssets = undefined; + fallbackFamiliesRegistered = false; } /** @@ -163,11 +297,31 @@ export async function rasterizePdf( const pdfjs = (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; + const assets = resolvePdfjsAssets(); + if (assets) { + await registerFallbackFamilies(assets); + } else { + annotate({ + action: { name: "documents.rasterize.assetsMissing" }, + meta: { reason: "pdfjs_assets_unresolved" }, + }); + } + task = pdfjs.getDocument({ data: new Uint8Array(buffer), verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. isEvalSupported: false, // never eval font programs (defence in depth). - useSystemFonts: true, + // There are no system fonts to use: substitutes come from pdfjs' own + // standard font data and the fallback families registered above. + useSystemFonts: false, + ...(assets + ? { + standardFontDataUrl: assets.standardFontDataUrl, + cMapUrl: assets.cMapUrl, + cMapPacked: true, + wasmUrl: assets.wasmUrl, + } + : {}), }); doc = await task.promise; @@ -175,6 +329,7 @@ export async function rasterizePdf( const cap = Math.min(Math.max(1, maxPages), RASTER_MAX_PAGES); const pageCount = Math.min(doc.numPages, cap); const images: RasterImage[] = []; + let blankPages = 0; for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { const page = await doc.getPage(pageNumber); @@ -196,6 +351,12 @@ export async function rasterizePdf( viewport, canvas: cc.canvas, }).promise; + if ( + inkRatio(cc.context, cc.canvas.width, cc.canvas.height) < + RASTER_MIN_INK_RATIO + ) { + blankPages++; + } const jpeg = cc.canvas.toBuffer("image/jpeg", RASTER_JPEG_QUALITY); images.push({ mediaType: "image/jpeg", @@ -217,9 +378,19 @@ export async function rasterizePdf( }); return { ok: false, reason: "render-failed" }; } + if (blankPages === images.length) { + // Every rendered page came out empty. Sending those to a vision model + // buys a transcript of nothing that would be indexed as a success; say + // the render failed instead, so the read reports `raster-failed`. + annotate({ + action: { name: "documents.rasterize.failed" }, + meta: { reason: "blank_pages", pages: images.length }, + }); + return { ok: false, reason: "render-failed" }; + } annotate({ action: { name: "documents.rasterize.ok" }, - meta: { pages: images.length, cappedFrom: doc.numPages }, + meta: { pages: images.length, cappedFrom: doc.numPages, blankPages }, }); return { ok: true, images, pageCount: doc.numPages }; } catch (err) { diff --git a/tests/fixtures/pdf/fallback-fonts.pdf b/tests/fixtures/pdf/fallback-fonts.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ee01b4fa016de054a70d677131593bbaf01ce44e GIT binary patch literal 1801 zcmeHIU1$?o6kdxU%r0tW#iAfbq*B?9Gj}F4$wWp!bpEc@iH>tCTLsL)qYS^Qc5ia#j&q$r9%_)tL~MDgB9V`3D2^u-BbhI7w3 z_niBE=ev#RL^6ODgoMTuFZaC>5J=#xSt~>$Af|Dicpx)N7Q~JXy#&NKrnunbAsQ8k zZE{DxTn>mkeKrLI;!?}>pbHc~w7VQ!%2x^H%au@@5}SGuEN%BQF)iHU46rN``>DDL z;h+pGR~oy_v0PS`cut>dusV6gNr$cYA1P2viIQp1{I6(~Rb5?5RTF&EawToV&eI-1 zVVUK#3Y56)UBHwih~b!oM|spGxF`(l-hAoO-Q*^Z;v!QPaYYLq^BDPgL94_8r!&c#oT?P z_a5P0htthxM#hf4`lIE_zs{_Mo%Q;}*>$s%+mG}=80z}AaQ*witJiOz79Z}Yzxw*c zq<_L=E1NlR;|+{M$WN-=eGB%8e~My?6#5CZ3=HBDn{{s4c@R4-J7#-U#n9fjhWgCl z{9wqJXyRw60L=mTaDy~vr)t?wAnO#d{mEUDmg~`Y4|YM}s4$ch#4}v0mpU$le0aq+ zJnn*I4a85>#hNf_59g;UF=l=Je~~ljt{Oj%V3{8=&*0-7;vmLxZ;Dv?9txVyaOarw zwSXLEOk*>d_dp4j_|Q_)iU86yzODe0WCXe#;*!A$!$z4X5H4|r8Pl&6~5Rn)6coF^`~@ILqAAdX8)P+_HfMo|oNHO`*9AZAh>smr5* zRNI0~>cZ#`nWm=m%f+8A{<8YOvB)=Pe{EXy;TR9NBqvCq3(~s>G*wkp$iX`sV#CfK zuq!r=$zP2P$xNJUY_i5KhZ>vuk$f6rcf$LA2&qg1YHW&Hm5;iZ0p$APopl_3i}=@0 zyS0&k4q|Hpx+12mg!B+HOjR?nDycaGVO=L`SdwKuhjP)+d-$~TD0ZoTTqqcpg~rAu H%ag)e)?od) literal 0 HcmV?d00001 diff --git a/tests/fixtures/pdf/invisible-text-only.pdf b/tests/fixtures/pdf/invisible-text-only.pdf new file mode 100644 index 0000000000000000000000000000000000000000..929c6870ccfc33702efebc7fdec1a66ebe2d402b GIT binary patch literal 747 zcmah{!EVz)5JieCsy$VR4;U%R0SWNh&N{J_S|Uk9t5V@66&$J_w)Q$Ka(9uvp@9Rp z9xLSodg^yn;=&z-khn+u0S9h~U8k|;X$G{#?MKd{tFhoowCeef|!a&Z10tf6po)mBdUP(Q!$Sb{? zYQi;DC?-?^?NzybmT=mabFG5)M*;%fwhLM-qPb42Y>inVXL+oB8pk84BjsfWNGTMC z+Us2dcdVOIRs9!COw}9cU@gKk#d7`6sA5AF%Y+$Hp>jq~(fqPN2l&;-qw%-1-w|#f zZ0t_37vcNM#Y5|2@niYG`g*cDfSM_-ZRBx3ih_pPPmwY{hEtsqTn?!nVqO|JZ4C`{i|7(3U#;V zIZF`+AK}{rJ|P}~6mBs`+b{=2jrqDyYnbEe*{xwN(Yv#Td4XP!3R5}dnQp9^&sa*c zDCK5N_(ujl*tLsq%2|7-k5I>;=lQec{5HQlkt{bFI8vZ}ycr`7WtMbY1Zh%^? I?Uy6;4-YiYi2wiq literal 0 HcmV?d00001 diff --git a/tests/fixtures/pdf/ocrmypdf11-two-images-type1c.pdf b/tests/fixtures/pdf/ocrmypdf11-two-images-type1c.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b9b9bd6a669397953861a28c48111a936fff1abd GIT binary patch literal 43948 zcmeFZb#NWck}oK>EM{irBW9K?X117_;fTRv28(5}EM{hAW@ct)UVr!9x$nC(vk`B1 zWB;3q=;-RItY2kiXJ%#hIoae2qGI$+3>+}zosYR6Fib>@M0SRjFuc4(02Oz86C!}T zp{2634G}=b#MaE&oQR2$iG>ItW?}7Y;`rxmZQyJoYGPz({D)GGh>7{HE=MP4A`T8B zetsAyXGaqQ8yL6rdbL&CU3MhzOZ71=8?*+A6J0tmPGAPsFV^CW3&?)MZi&e9fh349 z-LlUPmISI%IX}o6DhPixvjckuaZ(ulEn&1DUwUA>z96%7rLsU1+k=M^p@8-w4Ki&} zJO&}A2k7%pfCbt^+G9pFKaRG#ZmkV@!Gk9vb~meP7AQ$?5oFdt9NJcnw(IU`?zaT{ z4n;q7`Z)m>OM=zTw-spY3(TA87wjDj>{s z_EdwQ9r)0IZ9J=zb+nRP+%sVbS*xMaZ9D`r>y zD^GVW>FaE5v&RSRt7fOXfWT}0%bNx8=?Wfp)Cl&Q`QVndb#jf`0P>-wClv=r%FTgG z!HMX8@fcgX@>x#Qa||!_3|x1{xA19`7seJSvwGh-s7wR{PITWxY#N&1x>*!ZAL51b zn6>jc3zm&TSum(?I*BFW2>44PAqi71YNsb40g{@$CB*KENm})JzgbY3N%+vzBbLkQ zgsukSmc_5It+aDEOUuY$8kWz%gp_B98vCs_Em_S*s+#96J+H(P}rI@Lf7vg>geTZ<9*`T-)Yv9JynxF=Mk{m!1n8!7YyQ=nW zW_VhPrt-nb!4wJfB#De18?qpDDd!(2Yh)L|!3beI%YI+#XDi@@7G_y>@nZ@KmG6z( zIq=esr+a7(9_cyX!{^H^(^o)Fo$0C>(cA-a>5M(FJJHC&Uv1`F zPQHuXc{QG1B{oIH9H|RX9!6d@3Yc8+$jJdk^(SezgtYudkI{1>Xcx5=X2GaK9fz-QS!zngev* zt|eGnMA$mp3L`IA$_HS4VG&C!-DuOCH&s7d#J+rdp?=GUf-$i*{-^!_sr}_sm{|Yj zT$Ei5o&VuqG-PG|!_~-|I2#xnI2-)S*>Ezk{mqT=3b@%D7+IM(6B(MAS=jQCT()+S z5Lp=Wk*KrDGRoSEn3!8gcsQCUd&sF6c~}{78IuU`!|=LsyV=;={DCHNv$3{y;&$UB zG5Eue3HP7+U(Em#ei$MmUPoh7ZY5E%e-iz<;v+G4cDCmR09;*N8C+Qy>>SMiOk7-C z07hm2Gc)}k3VJ7ZTW13|dRr&be_;F*MbyN}$kD>y*}~5DpJ?`fL$h^a_{a1NMs_xU zKOF!j1{MIxfAD5w|F3EPjnK`&{tqGJzX<;fH27=fe`C?W!oubsO#ameuyq1h82=&v zYgq<{b}r6*B>&{`50`%^xD_1jj9rXO9Qo}ntW5qi5HT~bF);J~#nS)o`y-gY3hy5Y za*O=&<$s+0pM#&7k(r&IiIJX>RfUn0o0*ZDnS+**k(-f`7x14b{|kVfql%rKHNTya zqmBDtvobL;urvJGHi%SAoSaM?4UC+LJ|GzOnw!azRf4hbMbCc)b{F~XirPJ=sJh~#xjN4Blj70(-E)s#?Tg`P zN2S&6hYxpP_GtE)S=gsoIG7m~K!jjfMEF?1k;ErK!lWklXCBAcK#spcfV?E2OZ_)o zvHu@(#l*_^Z*0+z3&Hd-B0)}-zUBM>j@QW5>Q`%3PJ<2P#vqeqi`u%D)k^Jrib&j>)~Q@3rWH7Z1qos0yGnAh-o1ewbqIU#n*3rC>F96UF5o*lrWL zum9M`vI!si>Ii=;YJzcQ$wEO%2flAuH%#$>hZ6iI0;41xm;1SnNFqXq`rj;y<8O-+ zws3Y*FmV*Iv$40cHL-Oj;v@oy*jd{-D*y3ke*z~_6QG5WiITYRzXB_*{}ft@im3cE z%F-bMNSIidnL86Pad5E_0siGjC2b7M{$TyDpiSdX)MoO>lmO}$#(xEEEPs6OzaltR zX2yRPcq4^az|Bl4D`_HVd+WFD}& zu3(UC59&pIdfT z_6r>^VM8THDWIIKu8)v`qJydnd~QQUM^1HldtPm6Yhk0Ev9q<8p`)e8!RyS;$jQn} zOw{;$KyTh|{H&hYVdriSn0?j9`?cP{w;$24xR{t6PrHnH%oF{lwEBP3j}c4O?*po?ph4Ysp- zXp>)61)_NMF)E^V>cUog{4(iH*hT1R`7-m<3U=QX!|6v$xON1-F7UV@SrAlIFrTCz ziU}rmMvCy{GP+;W`X##>3zT;PVIS;c@MYenZ=jNr0&4PDFilFLkQ>yKzQ6Dn+&Jq> zU_IOX*d{ARl;8H5)Mtxx&S?Plj6b$C%8*REY0jGW#z?E$!F2f2SUAym>*p=s`(x~8 z1fA7a_eug+lQU2JDj$Qeghv5vG0+lZ&x=0H;iSC!5h9b;lne1gqhn_(qLs4}C|)I< zTT)uI8s!V^i9rNn(t}IZO>Qwztr)5sI&VI^Guh_d)UOmRe3%L&myb8j02Nnz$D9hc zsNRTsm513B-O73Q;+3fD=W1a?FCUK;+0TZm>NrF~PLfVy%ZFKWdUHOlTaa0ty-*)8 zc__U_?ZzpNMZyuPuA-A7B%uZ)y!s{lOP`#vvri$B2J#Oe?=P=wRXT@y9vPMe9;~*B zH(kEYtYJ?^7ROrWMoC>SA7(`R^w_W#xJA7+@>Rxy=o(Q_>$yTqp~ICWC4<{==qyI+ zduF#xoCR8L5Bt=;o=mmzly5ers$=Ju6eAy`*ubO3P0n`cRyb4hb~vCFU&c4kh36$p z-!PlDz>-++&==gm^xt&?M7{nzZnJW}%nh%p(_;}rJ5oCZ{c&@=i3YD~rCsOlyRK)) z-=}IXT)0-!e&*h+62gyX25F*s=jLI?VOw!TrGs7aC&0WPbbMS-57|=WxW!#txtKNY?2`E}g{FQLJIZ)?5cSsrDw0Oh}D zGN5}d275Pm7chDBpm{i^cCiL~8qKSRh3h;L9_&v2Yyb6$@UzR*%Tt?z^mRT+=LvLM zn}S+H7g4chdoSfXp~7;}(yfJRP&{`BS){!>&Wh?z!9Z;0S_CyB;9tb0xr#Vq>0vYT-)Zx9?1hnzrM70==)R@WnG1(Hz>dDWhnEj z@#p*ats`E#&6hWuR``^agWsna`#!zE61^ld)ZI(GqS+>}72WB$jE+n{Ws**8><4M0 z>!e2080mZQ@?12cczvFHSa)nRMz_T*XvKzxR<7cl@^seLnetSp)E%#33QHhobtEsx zYpT1>2qphJY^Tq}xsBcB)Q>FMNB9NJ+-qgHVJNRq8Ht~qIO64G2tPOAn*-F})~=P; zaqdmlV|YH5ry)*gr@>CD>r>Tk#q2lMC` zFXhbYU}4E8Cwz~=*<-h0_rju)|8zic!~2O%hF0!ZK9*z_*dY`wfvv&b=Tn$_*8EC- z5z#_oFOWlGM-)dzErcMBMw5B2f5o~To%n+EjZj%;&cIxXN?}Kr(MZYQQFgO!G-3%QGKTF(tofj(BUd@<=j{>m(6FnBbGO zxOS#NOF0~q`$dW4 zeP0SsI&t`Za*rSDRRiC~Fr2Tttxgq7VDqO|UDI-e-{ENJX?7Nx2)+kaYfP}^;03FC1XNs^=5w2DW{BLL7-G`tvEwSAx(_x#e^EB2A!tu=N_ zU&}1j|G0=h&GAxrl^7=P&`ymB1Pcx4$bT-kGgMfy@})bxQmWu0G-mR6?lf zSP_A9OU1$@ML8eEJ-I05TT7&`5!? zQrg$hx^~51RK6#G%4SKd-Kpl$)Ec6=Ui15ghpSn$WmaF?`sis?-NT+B_e07ipf-R- zZQndPtnp;s#nP&p!LtNrzolW8SgV)YtK%DPd|=r3^kC3HYiBdr-$L|j@@!h1ma=+t zzyM#zUz|kU;v$DU>fwHErDnruMIF=ZK?h&?hFq_&7N6P(l*Um+XM1(C?sPuB2&L}) zyu~KCJr)h?#icnNV!mB#cuD)Y{H$gqq_l)9vG*8o5Jb&zyAQTHxb;N%(Rw`<8=tmuz!^d+k z_NTBecKv*Dt6~NMYw>sdjMj7BnV-F_kb%mgtIMV+4m2?cze!$yPR_o`G>NZbuLhUq zKB1h~wb%_f`BlM5Kg(eL=Iwz$kD#@(3(mspv=n} zS?xs}TDVAmqrR6FKP+!#IpcgeNc#@uI4sbAXDz6GTX?87(Q*7N5FK1`@;*cc1R@_aRq|mTmG%>_s_u#_)%+9X{=$^jx(#$e-tMGIQ%_-bx6Yi$ooGNMzn%42f z?IB>(cuiAGR>^`S;6wC>io?gUO)iXr!a{Lad{(JJ6Enr7KOnX{}+D7 zz>LN5#T9@}aYmKT&td>(z`t^h{=>WILF&8uIRgBVP4{c;7a*=991&kkBuaF1JtX*S zkXcV6CMsy--tsoY8MPm>tEllt)Nb^9v>Gtr;l91FRzP|Hgnk{-69ZjH4~1*CiPUNE z>w0HpZ5mcFSCRm2))#Rz8W$yj89XC=9TKB$d^feH>7uH(Z&qV{3iucwkrwYLi1u$K z!b8Fz=`upI&K=|Sj?!T6YM>yiGUQcQ+tTjo-5eBJ zc{)iS7aCLyj!n_^+!pa}9Ztjd9a7I3ta|2*GN~$VGJ*s-#ew-^quFWK2`!<*l5TT| zA+I)vk)6K_OD_0NZxMdxQPasrxeZKjTx@ks?=8@CGIz`nt;^o3x5)McgeFf5gLg%1i{@6~E^_3g2r^%0W|@6aqlaE| zB{mH!4;p$p>=4rld@q)C0=40KO8?R?w)Q@EeVc5D_=N!KY2==yJh*xR?=HY;8!uni zCf^_gt<#^iBu#Tx2F|nYDcK$`iPZai)G`yB0r#x*{E+^3_TCaM7cj@v}-yjSFlN%HVwW(?f^BPzK39jY@PT^GUCp>g?G z!QgQ8*QuT85%+j^Rkxu2N32eNX#%TXjm-^T5d$GGSXYEdO5(fI-Axp47Ye7TQ46*~! zuYz0ZY_a7Q6s?&P`hClMB4BmmQS@OS+aPeqO@4B&qq&!}3}842@DLDXxK0$Cz5a9@6~&3uul$Oqqtz0MQ1 za3|g@I+rr(B-zk@xRU@SIk-F-lR{m4kSZsW;iWTanGbXldy8w5=>q4~8b&l4AZKZ& z_IMg=-(x2I^3csTjxQ3dV*%}2fKRzd?S52adca+9NK0+`y6ki}wk;L)zNHcU11ui99d?rX}^@?MV%*;m;at7JC2Pfy{C z>xMnmx1?x-?4X$Rea<8A-+x$T<_wFR-r=?aICX~A&eVl89!;=v>;j%j)ZX;Sh-~=X zoNsvEs0Ct`AF}FJ#or$mZ-!R7J2z|!xpztyfgYO91QV!JC@OZt4erU%x!2FX z${(u(*yns=+QfQokADG7b$~dgz2A}MUD#IYU)SVE$yr)9&ZjMeyb)Z$fBW=THMv;)jowUndvo_hK7dQ$L25Q#+kY=Wls<+y_}Wx;i=lMhreT=; z)NmcbN8Qr7%wl&NM(nNiD6at0tL~iEp`??{XgYUx57VW~wT#8=c;aFA$V>4^J4O_U zHxPWQ3|!+!`z-Ek);ZlqfC(*mW!t#C&83TID<8hO<|vJHIj?jiic0a{OFn-nonJ~KJ;DR{F$H3Jd}O(s|KfssqN*@9f^)` zY?{Fi2z?x|sBRLCv~>U)aNkd)jCIg#fuoJx<yiSXFU$7-x3$mlAXMM3jjoxg2t&)d`8|*(I6G5 zI~-?b(g`^8kN2Z0wlgkJXJ2v|7BNN=!`@xjc!%jl$_^s*4C+L z+K?jcivyBPo#<-BRyWdMs8pabBd!LOtH|uqH0!T;?MCS1GR=c;R;D>T6)=X!2`3RO zvx}i6UutzXGmXvtcDa(y7|iQvT<&UG z5Amcx89f%q7swym{dy3A6yEm@Xesi0WGf5KRON#YWvkrKIfGQ(_GW3@HiYF)$C(qo zICky$=9$VcFK?Rc5jS398-NB9Gb`@J*_$B(DyxoQxFSc*O09fLygeFUN}_Q_s>Sp@ z3jYxj@mSmN>x7)NIrhp(eDN2esMsNa21PRyN#7y7ARm zd#Bdy56yXsGDO@+KQB4_Q6Z2=!#Jnq1Ne&PowSexA<0V{6$0vP0{N4eu>gbq2 zwI3>rWBN&DH(IpnO#D(q2i<^o+V5PnNEZsGp$V{lzXhop84QMK#RomB-L|96oa2Q3 zRaYpy4jCP%9hYYv2RZny?Wj|HG--2NH2Ffp@q6D3=Wp8}Au9Bh7h3qX%D2hEs*md1 z{IQO5wJ4{x9{oly4*3betMBrsq+~AfFvX7-VQAQpqQBjEvhsCrS0+f&rV4DMzP>K%Ld0=gr|OwGp-Ces!D1kUqyx}rXa7yYgBV_KJ+)~@$FThM0)xF~y+ zbv5;gmF@7;H3oVYieWj`e(~`)RV4BbEw=`YT6fwyG%07>=H+f;3N(eIewtsSKw^Qi zz*D!9W>qiC5U8Vwa|qyTiJ7;HkF6AV>A^pRcT$M^Ir)hWslCEC@|53I0;QWb^0O>} zt!t5EXwTKX2`!+byY4}ktgaujr~-!f%tlhj5N7q_>if9y=xd&9v77JogpNto}#$Z6d+&ca0zGxYDKa{PwXj63l{O{LAo(nq$&8|`cM6NB3&m|t>hJn!~x z5e$fJ6h=INGj{U*5o;-v_NJ9Xv6W{c8+nzfU+*k?KSMzoAGqb#Yot;p!(vZ02a#Lc zRjsQUXIO8D3w0Ud+Db2f0a7LMtTS!$NCVrGOG-CZm`i+fpJsD2T8{ZKg2?I{COJ%v z*IV%D39WNmz1^zT17NP?aCbAg!kb5!a|eRrd~3r`1PD=QQ7m$PU^=c$L0aDtA)kF5 z0PMBQ`p+PZZj&Xu$kLvUhd01@nq&#?l_)wEqog~vcTda0*d8?7r+FCXX@806WZ%59 z>sLU}^u1h))s`g;+Wh%_h4FK!&RONhz!M)5M)daCV4Qm#CAg4=>POh9<)c*4tB~sR zshygnDt{ix!kaaxmAB1MnvAx3qTm9&tr%ra$JhI;+WvA3B%pw+1{KndzW!~a8VrI` z;zg~T=5+WoL&KwCkWykbO7hx+_7ckXIDpc{kwE8bYR9NgCMo0v=Yj3=9{posdZrFGIlQ`y7{y%u|i*p8~D{ zBt(5|J@eZ?pSPd{#mnyxrTSTMCuVKVAv8U!ozIgbYZ|m&JNELJDR0xNIzXmZE4Cey ze;p|ZhU)v2dSpb?_;rkw;zFWPe#TfO`=2R*=y1vYD*C8MPpBBL&&ol?FmyqS-S#?O zT~Q5?+YP#fS6)B8AmND2NW2zrqf1x=QYfZ1H0kx4^Q_iP@nx9bsulEy{sK2vrZD=AxSWy zU2EXRWMKq$M7UjWkci_g$Dvf@tq=2-)PZu+(X7`do0wRs)^k3w6e|TZ>dUK4HQ$kn z-mX8J*^ zwp<53mh&31KFyvwaf2S+rt2N%s^|m=o_B7PCGmTClA~gsqI<;!1=CvH3Uijq&rVl^|T=gAKIWEMAS(D5{4pOGFoe$?Ik>VF4lCj6hY7K6Ams zZC@HRx4ybT+1G`>Wh5X=xYJU-ajB3a9Jhxn#q%=Ot@r{d^c`6eVlix{hwkP;1KWX5 zx1$Ur@7T$BB0dt_OnXH(>PjjRYWHhCrLZH`WqQcOHY_Q&AxyX|`m~VCbMe?XU$6(K zXbiz3s^}5I%!~ZD@SegQvD5bDjASao?9a1t>v(94vrBnkiW`9X*@<6fif-QBY}?oH z&bntA-5d8o7}RuV)gW-zckJ6+F|mjpT(ET`mkHYH#f7}*$k(2lul4>Fe4#J9ctW2= zp-P))yw#(wG-HyWD8fA04FY(;1H_BhSF<-duE)_9AB&82BYIoNQH1Pg#W$x8WmEm9 zhBRSOj`;Z(a@RxONH%-a$%W*k@g>#8rmi5OZ3gU)o{Cbm|u#0sg*o@seGdKO)!f$$WEG>zfr<4^tG-@YUon$WUB#C+! zLZm#umBpi$maOVpAJ)QcE+M$4~ce4T&v#dWfK3#WjK>7!lb z_jwnKM&s|(EXQ<{_5~ii^~Gety%{bbT-*%Y1&tGK3aMLOc*JOpkM8gjc9jCdExlSL z_x7#STx>k7KR4C~IV}vu?amRXuLFX4JhKGbpDoAGNL(PID>6a{#UT_0D$fR6`SnZV zLnnI547Y6byFB?Iw^qj=gU%<+plO0m_)#yLe@u*toUP$YZOnMh|1!YroOmylOz<07DDD>Dh$;-)2fJY1ZEfSp ztzWusd)kvPi-l$>3Rni9?HcBN9*F}`F`6sCA(O6Ch~8T(JmB*-I8OTZm!LrM zikQ7B=Tk@__Hc{lSodqgd`+?+Hh6L_p5^ObiQz{WL%aSBagdk%kW78qdk%lFx)-bU z<<`Rf(7wQg@o43iGuk!2MIBnSc(4VLfi7Pxg>ZDbpoRG0fxWFpCE!Q0Il_2Sr+bluNzl zV#XnCXWC7UYN*LLLR{N!P-BImh{*mV6Ff0c8X|BRp9DcSOCmQi>+DvuYUkMFyObmO zvE5$FammAb9oO4<8aqbB{Hz&AMR?`2`WFterxNbIAUi@2v!cVjVLcJhoRW`2fi7{p z)V`INMyjYvOUS4GYS*v(Zm%5YFY6KKmmT!pS2AJxpIg@!970b|)G(o9>OTqJDp4GR z-_?iGW4aE}QUiU94Z|rB8v7_bp3@Sr!APL4%Z(Z8&M^?L1{Au4Nnm0D+&8N4C+z63 zVf&G83yb6F8c>3_V;+2|G=~~<8|E@Q=}|n}TJ!LeyzNwY3WqV;b+1b^bdQ3&K5=9G ze_qWu&h?xs39dY(K|)-;$=FwfX%(wAC@}UK{<=TNzCvw@ZQkP#d*tyREr}cP9W4%r zJd<;^ z)5>Hu7=;D&58tcF+gr5$7<-I5+7+NyRb%JJUMZPGo%Hs!Rgjsp3H&XOH>^&*uJ@9)7m}YM*3BJcJWYoO>Eh0Ay`i7>@%94~gqbxy68ju^k$Pmz3<{yuO zHe}76O2A{2g{D;U!xMY$3w~G03n`eJJ71a$Nd4$M!5z~3g|O*xw%=Xg*Y3j?uO7k| z;}Y@9;$%`8GMo8E?r(v8m#0lS+me}fk4cPDU8Z7Jd%TxpSlZ7|gvow{I!HoQW?v9r z7puOAn}hh;>Kh7YXSd`8d)=$r`{8wWMaU=X-6Mu5rOO=9_Vjhpl$v0o-PS7wqm_o37=%O}_tw1sb^6wOB3RF5%-2u$XO z*FCxjjzs)$own--7!|X3L)P?!;$82uxPbJmHW|XMQX={;_Zy@8-K}#@o@%*m*DxS{ zerN~Zc}5a^@)3rA^476D^&~vwJU4bBDAv&3Dr9$$W`X~p%0F&yHVz{H1lMznya9|7 z4v;C^Np~fQp-?V$^uoJlbQ&cMri!6KEZe zVdut&DrVZB>$geXi3t5Rxj5`@f@hU1#Z~J zrqN`%U^EdRjWyoJPMvpZtQ&3L?sv3oXl^d{`D$g6lIAhwXA=3m>3$3}ZMTVMOV55$ zeT*EnpX}H90gj|^*rNZ0U0I=iV4g-msc%PdTAAVCK;cMzIm%r!Cx};aSGKdJ>)EN` z_xoP2WtTb3O!%e~XoPDzz}?q|`!l{85ViXA4god-{2h*)4k6SmTFlqa3hK5zrh>m5 z@%vept?5ta&F~j{jlRvq%&Hls1L_2ZC<#=E5Wh|EEdf`?y^!aso9n+5fm!>d;YaWgH$8Fi z+ZyD7m`fz%SlPEYwh#Xh1&k4wvi97L9N*7N^8?>vd`HecI1;}pE3#Yqd%6q>Fw>L1 z>Pnf(ph}-s^WK9HN(!`~sVi}cla!-q~L3@OlT><+VATqoJj~wIEIGqTtJV zMPm)$DE&C_O4K^}c0bt)*o}Pd)eyQMYnBHKs*n!eu}ozYzQ=sJGf_93$W+pzlVv3#A;xNMfNs@AB|*N^f~<&#*Bw^;4T0)$k?l6JTiLY~Jetst9+| z%ps2A8h+|LaZrPKP}J<=#c_9QN;^N|9>Ls5T{bz^?K_Er>+L}KA}UTLrxxPw^@mE* zI`OH4A#OBhMs(VMwY&@$UA2CxIufS9db&2mog9b{!n`J(q4boN#80s z=;XTlsFK~kn5Sp)bKq^{Te{gUyfu<^ccx#8hpovJ7v=*GX}`KKgyn+MPc?^R+)*4h zR?-1G&Glvd1t;CDNSM3lcJv;f$H?BzS_BGS>0~WTDw9(2LOTEv;c0F;`Jy^1%z;jgI%pEFKQ6z z)g(9R&K}JFerYm9u^{0HfKa~hV_CC_jZ zkT=ac1HVS;l`GKX|Y zCrZnJTIn(Y{I;==oj(KEh0(NR2#$lnm-iHR?0JO(z$&tRaNZ+d?BTb;+KQg^lB>-XG+YTZ76mr+bOu)qny1gO*WIb6sMmi z-R^zTD@x4SZPc{xaZhOW_N7Zo{piK|%qvKW@Vw3EkbV4Zl*TKXoXQf3vv(!f!cTs0 zf_N#QJxe)8rW0`cQM)+tRywo)OhV1}?!sp(2+11_9)giLXTC$*2?eKb&uxw9M^_u! zT=KpL-gA9o+)+HTnk-)8?NEycY3*${4t^RHr0lr&vm?>RZbn4Ll};GTbYz#^`btp7 zDAx8FQ%Dip=A8COveZvY4+Ydrh~L#?VBWKIt#rkz@m=PA<9j0xIo)XyHsRvD$a6nI z^1(;o0qBArubeVTIOZh}1MPS|l)dOaPl^RcgqYFvxO68l(3(3mm3^KKQVLsuZS2wb zPG55a;0Rn*RN}EMl08iMI`wGwxPj;)5eP;Hu*E>XTH`dY3KIey-{r^WZKKY-CY!KU zx_Wi}y&m^NwRLmwzTOgUl>Q7#?_d{#5K+JHia6U|J{J>$k_vp{-k;fNd)s3|4! z!#n%tsA-m`@fjjLYYVv#A+o$0p@u9+M!wvkNZo?Yl%d$LNLT`eU=)7}tc z6zyE9fY6tQ@65DLdlvK4>zFUH)@Tk`)3#e}X=(Elmnw#(kQBr0a|q&^Y0Y29M_&}= zQTXvkjCw6^^*RTo&i5}#4QTITnt4y4Oz*@zu{C_I=gC3n&hKis$ADA!x(ocQs9ro* zUYga7@pqeaQ6>_LTxUb!=k?Rt=ii!CV?q{a#lIU|r98zTAe!_RQ#4XDje?{?$b&x6 zHgKb?qmuoIf4liq)wgMUy-z_V_x83&?dEXC);;xITlFyXwL4l|Ha}h9@D*yd3W}gO z?{^Ok?`I2odVMlf`!Z#KUu0ITgT87wqnRPj6S{m0U00u2cdHaf#(u6An{nIQ1Ze!_ zb=843iB+mndfio>nxyO}y6)1hc$Iq#bw}6&Z%jn+AhvS4gR$kd&IGeCT*Y$Tp);njadge}d14--P7(q$#$D9wOmEM?hy5yCcBDrFMRO)T|<*iKCT0!F} zz?fH7d3#DZj;E?vK7uZ*>&-D+v|YmDn&T$H=Eu$INV>RQ!nu}Nu$M`{4eFA7pJ%1^ zp7}aH4jtGv5@7zf2y9jya^1%4L#l5T7q<=CW$560O5nco?<-vczRF%eIa>PlKBhtl4Ni?Rm`3 z*P&)?<*D&hl~ujVmC=jBJ0WT0Y&Dmb^#kfI#d_Mjo^6i;p4&mtYd*){Ti%Xu7SgBJ zV&#QSqp&BDlE4y{buossF-2WBPJm(r6Wsf;(8G4#NtP7N9WgSF>=b(uIqw;4O=ZxI z_1jDM1Z^ctNkMQa$D0j&?3OLuD?d*ln`mV;xq>#1fhLb;OYhsy^wtY-9uxz@@X{`I znlglvmiNIqM@l`jsFW?>OFNcc&nBdTUQeWpm&M5=w@zoYM&hXKt_t|lO}9tWUdvOv zClTG*@p4U%W;jpf8d@*{EQ`2Qh8h*R!S~8bkv;&2x{#pTLMzJ8uGa8y1Ozt=DJ5Iw z%Be(YG0c}L4a?F~Bp_U%&E4;T>I#X)PAJm==<6B@H^#=vgn8)@29j`?Pe=+(#t+1V z%7GQh`C2)MN?uAZuj0+ax_))#-_W&amw! zx1G0Ny`{0^U)jz_d3-8k1OwxjbI5~CGY$f)V~kqE9Hp?XqH1W9DYmaYS_1|aD!K`9 zle?qS=$3;l`=zKYJhY2v=|kjQ6$Jyaxyu)@SFOrFj658-;5O>Zr5lu)xl7U=BEgcZ zsgc6vIVJ*l4xx77=JdZ$9wUN#lZP3KD|AI`^E zw)0$c3}b%9;e>6AX3W@$~r&?BgifQ}87GY4)Z2h95v2N9oyiSd7Ze4VVJ1(gDAW^Eo zeMW{%$NDy$=VzX}V1RZd@>R}GR|YWCr58OZSllXr83ZQdEF!B+hmMtnIdacD zNVHp-ENl`0`@K5q3%y@^sJ~Nt>|_6IaiB{8lPaBu#|{eTQG$6dl{tT-Bjft-fcOLu zP2g+&X}i0d|6x!6FCma(tFp7mD$HMAB2NwC$?N!#ew2Ke- z!HBoeHPN<2feDiO$r^_=x8!yjqd{*xZ@Ao62hov*!!0xp%}*{08d#^&Jii(Mt7sX_Mj4s5Nj$gIW?(intHJn`@k=I+JUS;Y@fKO z3R_`-x3ZGE&LfBLf2s{IiWniWqXjgIUS2m>LfAv zK$(4r0%yN7!=TV@1j8;S@MRspj-gC;|7ul|Xmm6=^omq2Q2DjR)T7ePblAG2eyfCX zdWuq!=wjvQkf9QFZOI*>*Qh^;JIpgkRK^{A4(%O=y%k{``9mR;ofuboPf-^7=o0?^ z(_3+u*wI3G)!r-xXb`(0?Pp70&Pe$OTBcAD-g=wq+|0&GH(u9-qNL zD6P;VZ$g<1%*F2~6vkOAIdWd=?vmsHbq(h~JGivbc9L#Sq9s#KB+3w@ev2rQL`Fg( zH0T`gD`ks>Jgi7%`YLc&!pz|Dfj0uhZm9r)W}BeIi>F#J+eo3bJ+^kFw*Q(1ggNLF zvNBXRDA;3P36xh+Tl1-GEli2)WQvtYNgyKJNr^@vL**$&KxU>OpnF+4mac^06xKnW zsvvX-TwdDm1eSZ=UEezp!xR;L=j9OsU9U|>kjiOV8J`dF^g0XFZ(OdrAIrOd`dtxK zZqmZ2Z18+4ys&oW=gK248wqS;8VF+O?Hr86MHA8>`-=WPQxU~ZGmlRD8Z$+y+P%L0 zl3qM9Aim97TBNFTmGuCM#a;nw*PYelT-MAlt#$T#__K#LKZgGfB(e8`RZwmjte&A)~BdZ z=a2lrNk)++s!2cr&*jv~=4}p`Akj&4Xa-9cf`VRf6E6Z0N*5P`55J!ZD%J8*k&e|S5htQFb zFyG&K$hGDi64r!<4$b|S0nA$93YrIQCJj*8bz|2f*6@NBsk@2GIDRwCgS545Ao(WI z>ZVp8Qlog4V+$Y&NnEWNOy?Oh#5# zSl=|nyjQeqbO5ut8qANQ8};tDw)4zZ3OBB=gBpFzbPfT}E00;07j=u5Z+$%+1nI3O zPK^~yHShMGw|&JD8Qb=2Nk zZbnJ#65`lagoKC}_`16vdL8a${a@_8Q;==Zwx*l5S8}Co+qP}nwr$(CZQFL{O51j3 zt$pHD)jn1G)~%|j8*v{h=6o0ry~Ui-TkB)azmLCvty8!CIHmaN?;dyNjqhRb@&32) z=5@ENqi3V~!p^t-N7~QaYocgOcK3HnvY_B#Cs>Rw^Mh=5;=zsV@p@#>FMy6IBVCaT zcEFR9+Z%drB zEG%DNhFP7*Y=Ajqk`D$A-VTEk@Y<`Nk?vWm&yVl<7g5@?iYHH>r`wj}eMMGkZeO1= zZ|Ct}wBL`cni&_M(7C$pHhP~%yxng{Ca0ShFVdP!PJ5E~$FuW|oD4P%q-<}YD4iKE z>^%Y}-$psvyD$LSh7Gi}Pj|mg-cxTbJMG71eZFF6rspL+{^rKJ5hs(!Q=RczmbhwZV~2g`Tl4v;wYuv})AIS6cl}S+yAk~7iO>V+eY~ORXxH2P z3+Y4EI9MA}+9m5|hkE_ACcH}xubs8;N%2fV4-eC4dM%Z0?W*<0OLwmIdDq6}eNSL% z#yF0*&lE^2$*apFaWDNJHfIyNjcLhT&%N8UUYCDV#RcX8fGSYpmx(4W_xDC9?}r&< zCn4JR`pcrO#)^A+W0CK+!|o>Ux1^`{@ul8}pw?AouiNb#=k>bXj-aL}4fA@T@99No z?`JRW`+1ga%TodA-2|rx)xG3e$oaP{$lS^Roc>Pq#;DHSYIA+RG-f=m>DYT- z{@L)Za_ns^dinfZec)|~b6QxFvqG2P?8 z3HU#wkor*0dyAZ4y#9qwW&dZLDq!pOS6kBkeX%go(XcY$v9bJRI^q2lmH!ekB=Q%2 z^ly+FdOTVMV@F$O2SZ~=ynpZpMQs0~asF{(Lj4ylh4&9EkiuW!&p+P(RiFMN{J*2z z{5!UYiRC{^(qg4f8!UR5%Nxp9;8D3EA_+c(zF5b2nm(5hO3dpTy9uxE^3@s_*iB(O z1eJyGtUZ`w)-ZuG7^7Zf>=6#no`KxOD+eUC?|@28xPXn`3dHe2hhW@VQ;y9ZY8`yf zLuCCo@Yvf#GPeTsFeR(CtTQyO{(HI z1!58xUKI1UrjLR9#Y8H8IUxqK%zsWFUSC#$TtwhF4pE_(k7Cp)uu^&LwK=D+2F{nK zYer$3tlFM5*5Vt!a6^jxGHVg(thz8aDQY%obU44k%cbkh+`o!w|0(A=ROY74G7hcD zwtu3=`n@>mK~D`Sov_T4ad@-~HAYhJ2&sMB>9?U5LLP#zAIk{kL{jZlDQoTBnNcyF zy{&w21~o0kb=>6(ZxPDV#A>WHnmt&o#ak1fR}`C!94o@CS!$AWM+CA#F4qNRe7ozR zE_0J_j4$%vmmh2$KxqmC5e?_JI_u63iJMy22l-6AN=R1Toa=wC5TCV>?8$@n~ya=sO22t1?z`*syi&Se*)I#|SlS z1j1|KC4hk}(H|;%qR|iydCHav*Ov;!=e-BEkPx8>H>#2wm{|b}V-aA;Pc*{7PpT0j zc$(f`Lux*Pz6{3|hl^4@(r@c(}2|FPWg^8^L}`1$!81p9a30Nze-lQ8=KI0OLT z;_ra}y_l>2#n1TxK=>E+!|*>>KlFIafA4Jn8KK3<$oLyE(ioTy(Zr`DBeydUT<5FKOzK1ni1bU9x8-wngSWZtnEW`a5k{j`O&U;3TiV z87f#^8vyVxU=V;OJG##w_yqs4KxqLmDLq7zJY+&C@ZWl1p+J-&Y{+#2SP`IYA0M5J zYsQTwOt(}O_~Pk_A%*%&S|h0n$5r0`#SY!+(|(}2xEL={z?pq9E>Lv zqV6}mt1>1GwqLd@o)@}543^?2*WcOQFY6~z@!5k!obh5r>s4V=azo^SKtan=Iwqr& z!d;t0J{jCUL4*F|&z@qtvl5nIgRau-PgM^%rr+;MoGB3-_?_e~H7<)n+XadRly?+I z0l|@I(q2?0xR|I_+FiQ&D`bVrHqa;3T4>mbMJgd7!|l^>83!$9{Q<$W)Eaem`I>qO znp$ZC*0mp~t=f0~HWvkj20%&@!6r4Da!nG2OGD(!`8Mk1Q9;r zPMet6MM^`&(_gChs3<6rk+F&p0i6R0OcQw1zWVDpe5vw)_CF$%LAy8Om;}rbAjJaQ zsTvR&tgF;~D3`}F!r#MxLd1mSvOKip@?@zBl_&68VJqZ{CHV2v<6Ip#QL1AKZ_Uig zdcjTi?vEzP*q~9Ncdb;x$_rh~Q8Iq~mw@HthhKC3LA9NadM6pyOg`QSpJYr6pI^}q7fsB3m15CcGBtJU|*_r zp1JtDm_kY9Mn{i}B>WF^GLre%qn*nSK$6~3ErpMH>)wn)7OF3~}uI;&P z+{GNE#;Dz=9jcFOjXR8^jsuG49yTbEz$TVRevr_nbG@%Akj5*nCq)mP6Mb6F?WOqiihg+|5C9v+DV?uNo8TcG zZ3LSpN6TyY3Uub`7Vs^mxIM<5#3f>;-R5Y%rfU-LByP|y=o~>489aujo~!d4@@)WX?)`M-VOt`)BZe~iUf^Q;Tb>?c zlIr5gR;_a@g83B36Jm}(aB{5-65z$SwiWx~%r`!Fu zHUAQJxUyB)GZwyub`MVb#_St2kA)Fq2?-IV`nG^o{&x&lYmv-y^37U_%=nwz824JQT@ zmo)VxbRA-_Mwwj=`xTh$#V!#zG&^9LA?s!Pv2}-?;hcVa)TXQK#lo6sQ0+!GfXmy| zTGJ0@lM<97)IvUf7PQO#)1)U9D9{|Vb*NJWf4$hP zA;g^1XtR$BME~h<>zH>|ZrRMUO|fryd$OeksV;3_P|5W;^V2oy_?6U?XR9MhQ)u0p`jt0E zU*p%~xQ9)B&{Tu5BdzSIdlpZ0v6m!BPvHvbhL9K0pYd)2eAxrs_TmJqO)i^sK1O}E zke@=~rTl{wNju(~nQn@7GekGkXpb&Ya54GDj<7J%o~N%}$foW}@|mvZv_RTrs)Vt|KnG24~=(N%X0) z)tuLMwIP>_wm?EF@#C-dU8pzxv3tDxs&_!2$o2$`CDxhU>V{=kae{Yy34_9?)LP9r zF#J-Vb{6hrD4rg`?lEGAG>t7!SZ!pSkb%|Pg=0q|;WzTj;(WYbZ>n6dwFEY=;tjDZ z4a-PKV;5Cd2?+ujW08X$2ncoX$VxV z15{TrnD1}VB=OFH7aZc$EOr;fGO2rHKaQPYy2Fueut@h%D-M{%@^G1EOI6Qp%k_#z z+5RUEnZe9X54_Id6=yv*@m4zH)+j}sZd=3+4s2^Ire5ro4%dLzE4uMULf>|)9-^tXLzG;FGRGOR<@VEb;2jQeL zTYU+nIN~tKb3=^|z-AQnh%QE}s*@CjvD$%=8X{u%;9>$<;KG-8mL&xqKUcF_hsF~r zyh{Wv9bD`Wnvp4R!zVFx@rT0Qa8C-^d+k4EA@CnuMJ`9Z?{n=|II`~{E&#%R{fsIl zD%&8h$2b~v zsATjkFWB$8v>=`TauEESwQ6hiY!=GM*Mq5c_8f0~?v zh^vM`;~dt=Ek#6LtRTd6pNGQ3v@Vzha9j&uH^`3;mf|Mge;8AuG3q+O-lx_on@gAa zEGZ=5Isc7Df{$a$PAp7)MaFwT^+!Y-viL4Y@|jIYF1^^?_CeM)>=jgPNEk}%1X${` z=^<%$ojBx%&R!k~El^^AfiLGPCK3nD4gThGm{u`eeJwpQ9s#a)-vIQ1k65%$HMA@P zes^gJ7RfEi_e4r0)bT=)%WZY|SL)U-6BF}35!)=jj)FGY1{sst zyI%VlOAc*E)XC-n#GeUs;y@P4RHm2?U2t3MQUgwZtD-|Aah&wYVyRONn*ki9RX@w~ zAR2vm`*h0;JGt1*h`jpj`W(mQ`MO6YcbMKcI+?+CcKS`0&u zUATct^;fmU&xA9}#C8lG9NiuB>o(O$ign@8fCd7jBuH}n-Iy^F)Rj?79r0J9e0kuD z(N&$~z3ICvdcw0E908dYQc{MW>d0SlqB`r_lY+FvKc6^452uO7};hY>~-_v?H935BrQup>Hbp>)hHjq{)4gOsp$ zCT-zaIwP1oGH+|^!+oB6vhA;Nr*xo1f(rGqknLUH9;#O6Y}wxf21%WAvtT1@+e@5K z+jH^ssF7MQhUX?Rl4!KKm&ZfplG^)+(GzY%((p|V zrsvH3p#CswR6+Vg`t$6?%2M5Pt5pjcV)HkE#mgK zMfiJRv9}#0yZObNpTwmqAx#d|o7TPDhKib!l9HN|yb_c9y_`KizqvlYzWv_eD4eWI zg9%sv&rr5Qy(F>@$y9@;&53EM!Qb;R!b;TVNZDxHh~8B1`U^?ojwJ3f7EP^*92HobSfg0HPE8Ui5-s8r#J~!i<&jPyP5kfuhi{3Aq(6^od-b6*vYg}D zI*t~0znwgGjj|bhBSw*zlLE+1OSas--VZKtW!*o=4y0p$7@Q}TgcVn1r?cL>8zUbo zXTE2mXGpVpUD6Nlt0oE1f+e#g(m{fpsN9tk*Kzc)=OBx33JHE5*ywn!C@Eklc0O-;p|X9Wj6It(!*U8+OhDaAsEdac zZmWn(oW!ZBTt=WRS(!rRM+q$qD}Bq*h2WW)tS=~(6W z?Ma}Noxl+1jq9)YjhV1C%U_!rl5vlCy z`e^!lH?c{Hs3>J4qu{c*NBr{9r$}+`a$Tv>k@wKHoMeN?{#$no=SBU*KdfL?ru zPa+rZpd&UV5@N>$R~)vy?x>S7JFdu+6>Sdtfu`uG1o7$1077yR4(3~1()yyfFuWC| zi6)h_$O-^;iv6x57E)B56_QyVAKJXb)Gw^=rglId&{@OlKi58(*8nCfU6Pl2 zs8xTCrMKiN+YTn|F6`T&fjjkny_#@C3(i_)oappt!Lif`-F$LH?!60vkrH$*Q z(>;tjhP&ud%62NnxxRhafwo7G$5_Ex!x` z*zZ~YT=2HXsk_bb^=o(BNg*>*>`>$3{W+J++U%0BfpjJVvKm&6v#(8E~E*=ZPTEv(2ef(N%Y4#BqePh5TY zbX+zXVfraJ$)rPFX+L80kYQQ+`qSo(-mg+wog_}?EI zYudqgD048!@A(sOO3T3q{%0Q53^4onI;j{rp#=A*)mjom4pca`@oMrbvjfDs%>-JZ_aOZcSVDM#~a-!<|m4(mq&cot=TYE zT8pO}9bl>ahc|-bdn9TDGpPN%oJ;;Ts%c`guvAE5*x+=NJ(u9qOsohwVhbH6K!>9y zRalDZ4MVwrypK$CftY_@RyF3L9+ccua{J{T60&RPsY~rc+R6jU6)!y)AgVF(8w%Su z*F{BllbKo4+`JBRoq)W`+#CpYlj`zwpy+p-UtP)mRMN@?b1He07&Qz(4M=k z_Y?WLAo{%PkdvtLkaBIw@`q{j>mrsRgIk_(RjEe1P7BoM<^WAdu4FefFA!XZKSq=u zObjV3WeAG1{&wY$E#8NUn_OJ?xK%@tTf7}t?EVp9>9UDw`=8Ldpx{ar>sCob>2_yh zauhqTJEpqEk_=4YF@YYX#+&1%3%%Mp+4Z&D;D;yW8>pn&rOVBAXAY0nUw&)37He_* zfnQStHPq7OMH+ReroX76bI1?YhZVjmBb!j{u{=+lJvnT61& g}%n=-4nRykclJY zh@0%=y|ZG@4)s6M!EfkvayI-c;Vw@K7j&Yd{NZ zfLlhS2#;^>=HcK&;3mkI-5{x%tCP4CiH7YPH0-BJh$Wcqodc2pz!1uTDLj1aE08lp z`^NODY>~M?bMwWvkAKB(c(h90j8Y4FP6n+FrLT+R~2Eu81j*1#_sb)&dp=J=Jv@ zlkPnTIf~TDq@7V>#<6X~b=nFwXLHLy8B^)8*pX;UfWp2-LSsWi%`8B!bjWOyEp|Qq z8;}dL#}i-;z#lRV`JTjo*Kbn)AB%%mK7Nq7$ak3>0!=ud2zLzL;*R-x^#FQYCI!V zdbcI4z1QBZGF9qc=HG3W>UYIEAxt;c$d&oZFt5G=Xz34{*O*`}NuS)A{hN%CPT9v+ zFMu8VPZsFZis)6d1;kI}(CDrVbyo73tyY;RNr!Ww09}Ah^1aOC#2BEx$Fxu{q z7U0Jj4e>xq&S@1LG-bc1Y^SIc3$M|_gSx~?9$chcuJVz{0$U~;7K52G=$Sc_v~%P= z5%%adTdVzCZ|mKfsZ)MGANLwpfMxvVfDRfd;L1`s^M;PEZacwgF~!*(&A6~hm4gMf zXu;1to2;HF05kIX~*O(E$SlLv?2VGEk-abK~(>4nH6CNjo> zG$|Bp<^#8w;hoqM47RGIO(*B40g?z0nyCGDW&b1*#F_|iF2Zo@;W5B~ z58qQLtXWVQz&rn^1@UHhHSL^4Q5em-sNP_ivcs>N-@k%7q}?wCkau{(Z{8%p4QUB# zivDve4|0}L1Q%s3%U2&SxL2@rG_j=%xJmyDk=97zyy-|IEI3F=PS8MDLlbFW@$5D! zXXfze2zvrBDMeYHa&-B!)H3E5=|I(^I-*_-HkO(7bppS$OYuyae2y;A@Yt5hvSYb? zvvf;1G1(TUc>Z&AgsLBI6-IO$7@B_5_trSn>`2kCN0xQ{OH84KS25`jumZduNRi$B zx!rD&4E7J^{rRW{7zlxuT#EJ1lk|DNoTbD^1oz`f+&gHso^!Qv`FUIq~RUERNzwFReBzAZCFbxCA- zSp!)rPk1szODAsn-<};?A7AwmZrHfxOE24VnOk z6Sed{h$Z&L@PRlWyubC5SQ7$M5d*ezRnk#PlP!$>ejv(tXLv8WD>Zr~iX3S= zr?di=JiaV%tRa?17`k*b@ER#HI21Vo$Jn_S_D4+C*6Ydlw!j|qm3TNP6x{eKXf@7B zSioJ<)6={66PnC#=A4xpoA8l?CGRGxCC}3of;8nFqcAeiPfY*J#iLfk+vH|?2*ti% z->C|wJhB({MMR|SH_-eHCgK`K?d$vf>WwWHu?xeQqL%bI)aYg!LVGaQ`~VfS#VvUceUH`iFyUZ+9rvezrF<<&r~!(Jr^Wn%jMG)F;xhD6^Eh}m47>!hbk64n z+(rRB)SMolIyNSQ@T8df$alF3-M=}+mMic<*6-DBC1vD~rRzX&>6H{=ZuOrVzf4mm zpZy4C3^kXQ=^ilaV1W!!IOTHGJTlu9{Te7?ccaGugqQOkCbR4aq}+dx?sc~*Sb0PxN`heSF_z86ux{KGI*R< z!$cb>7W&}VodV>uQxRH*r>@Dw*ePJ7XD;n+Bd^h~L=$&DICEre%Nkj{_(Y4f^ThIU zB=g&fItnI?T9lXpIgHnZOAyqTcwF7t$}|){f`V?zC&>Z zCxxVHR~fHqzvX?fz$iLm@$ZqLH5n9JOd#UP29T0J0%#X~Sh|1C-~3Y`T+gp?qTOCs zrGF#2^wIT*J1=btbd0Ye{OQxN;8~%j3~tDZDB3vPcr@Ybo~{OVI4f&d)N}zruK$yH zs>DQ?Gc7YcGtbXrge6WMQ5HrS6$n~q%t;kY9muou(fez;AOr##3*~}x6M$zF2`A62 zVT+)~jEsgnkbrEOcGh|^m!W9@&F%@x3yu|KurY+CS%g&(?cA#Qaa<#BeJ*V-PA$Ls zPIN&?Si%A*&n(cKFGiO8cY361#rn3S;Bzf!n88fk=H>BTo~x1VAm8Y&cRu3>_B@6@ zh<=WnfWdOZ|eFahD7z8PmLzP7H3G_HP%u#wEO_D=U{j;<2$tCwaBckB}QB zp_Ws@tHJY!!cZDM#dIL24+{*_=Cr zaGaq}M!rMErJpic`Cjh{K551nVUP19M6<*%_oxjFZ7sR-p)94OHbMQAidE}pIz}u~ zZzAScC#Lz6TrT5|EtB*;ykSTws|pJ{#>WYf3#$jKqlCfz*|@1^RFoa{k0GG=$p~RS zVw8R>W{-m^4{1uHLp~ooL_I-1K@Enl3@t04INzTcOltPaC;(AJ9*~U#maQ zyD@(5$w@;>ko{#|e9I{Ldl)1t#s9m0KcOI@V80^hJSYRiQB9p8+vTz2GRzB~dt3`} zIl^A)0T!d4sZ0Cf7H(uNV;&}0PEp>1Js>KSyh79`)?!Ct5iqFOh|Qcg(Tfi|Ki|-u zWLgD)j7`g~?bHqtxu9i@q*$O(!3|zWE6$l;Q&`YYP*KqLWfhswhr*wEdmrhn1A{LHG6f4URG_Ikv(7=z7{zT_uW z*SzQlTu+bO^k4XE{<$sdzjTNFKly9eS^pj={AW{LvIn@D(rO0_;r03?#*hF@xha?= zRlN|?2$E-iL;wL4h1iW?X!+D!a4ei426~7NNIZewO*dxev6@WBxVVf%jj8oD!OPF; z$l=!E=Z|-eBd)y3jdG7|_s&N%b3|1Ld?Ql0P@$IBfcL%tay(;;kjN`pkZPh^Itne? z9o*on{mQ52Ah%VO>aStn8C(d%*FW;2JD#S0)V{CLDN|}%1yl;FE~`hvIfP1uSNBoy z8?jENe7-EFe?1p}_w-HYf=TmFV&L`|aTrKT9~Ka+oLWM=sT4V&Hlx zLfdu$LaS{qu(bBQu~kGwgx<&mfDB4GUxg_Q1P#hP_~GLH1rt+5L56}}-^j33z`%v( z;P>D(3BLq@_LUZY*t%r+Q-_B~K_Q^E1C;CWed>Q0hSx)RMCJy6W;?rdUO)Dh_v!&5 z@d4=<%Rze67qPvNzu|10#&Hj;;oxs0BQZ3{qI!Lzl!9K_t5iUO$$-d+YF9bg+Jr`i zIbf{tjMfvRtHu;t(=5v!oHw?XSpYm z8-%$9v}HG&Vyo-21T8nFTQN?EG!vFYrLh_ehHVqdM7J^T4~OB3e&UXcje91fCqyIZ zhAS|TVN++5IwyEYbQYTywHD15^KNt*Gfs{yiWoMomMk4ljlql7ptoBZ4UBOT`qpDx zb1tq7yTn!-EDgy&h_uIWuz9?mvWm{4SL14Py)UDiMA=Inr2VPiS~E9n9M!@6sNPoU z%(T7qI-ps#uJv*RayO>@Tq^F`pVbgr`%Y%o30Jj;OUMPEY_mtwZ1C12@b*p69Z&z( zcmmLn&z``-EL0dC+LLaaLsUbc) z{ktAbD#V?;N38`t9?k`hEmz_d(LB8SRZ6e-n%0kJ;|C43F*&vqfbdkxa&cJ*JIWl7 zur@zV!sbu=kjH80uPg3cmAtP*&+AHS=e;^emEaIsB|=Ru`SqY!CQ}xD=L5vaHOX9q z)%Ql%+Z2vY(-40YwxCY5tyc4mDM+|TS?eBh&pQasgZTS4Et{^P&{fA@L@p07zlhgz zd`G%z>xU^5W!ilwc&#xvKc1#X*Y1+CUGPvf9rb-6#lMwGUV5}sXSOWaTwZMiWu2ew z`Iw2g{JC@V7X(2rV>a4-u2(4gO}%(W31eKbLpV?CFY<+PYqu}SfNKl#j+x2ua@==@ zt?8+RT$B@QJw64WF|@4Oe6iQfYpZxsy>V>%(sNQe!?X@u?RSx`8CQ896*M@r_(nH3 zVzF!eNX1^fF^{?2{_tP$@4N3a?TlmZGLwwYHO*;n6oVU`_dyH@esL1_z<0dsW& zofkt>D7;Quzaj@U^r1%<)aNMTH#5r9%*%s!=_MG@Ud!$jXkcw{q6BV1-5yz4anr_S z4Zi*oGsTFLU*`sXXm93I>Sfyokq^bDKaOR@_T}2ClHS++p=zf; zg~GJs1b90{1o#e8_ev%-_glPm4S3G}Y#$|<+G#mng{L}Jh(MPXkPyKN9)=GKBVvr} zW~Jn&trYsk-h$&qvLh>70cPpjdU{Y5Bp6B-S_x*RfIj^cb5@Y*8(;gl&$JtHGWwZ5 zs7~GEWJ_K^@z$F(jMZSshkW81z3x*Iu-Df?Tvl71g6E~$_<^1IS;*L7sEZ+^UcjyMzCDswc)yY3w)rt9dr?;7hSg7cD>}&(s zfZO+h38ShO^7YNo=p}ftfz8)WfDwmNUnImDN9RY~=4CFv^{9`1!!&v)E73o6kD+60 z)m_l#Vb_MVeH-mT4Aj-pwGx-D@JUDKBK4Z>tr@0f8LP`Ki6AiX<8^PkV5PR2cF1w< zk5~2IK4he>o@V=fR9unW2d-N`r!B{@OJWP_iDR8Z&ocs6#JvEb(%#)ZMm!iiIa09# zuRu7vTdG5zw43g@lE!u$PW=l?%A0B{P^Yw^#|gp?=haKz_HfR7vmj_L2c(XQ#6d8* z6l@I$M7N4o+|Q6LEKZxw`_)dbz~S1HUHySuOySypTRm%;nMF4!+O)cK47unu`qxbJUNw$xVN>2T#{61NK! zWf5wBmDgZOZVbNLS@M1__SB+o<%Kd(=pDS8TCIkWAIewXv^LbRPaR^aL3Lju!9NE#{Cq4_e9y`%O-(J#_S( z0UnMUOM2MX9!nPaPYLe&G@_w+ zjf=F&zo`b3fFkF1IZ#t7m?InFkbKAYUwXU!tB>YtuVF3anMBf8lNEND9ogcmDJjYm zu#pYV-mo8zJ>O*#d{0O9#Z6Sdg?pc0Bly-JJre{B_Pf6V8BYK=s>!!$Te~YRPJFKs z?|#B}4tWwFCg94I*>3?^Rqw|Mt_{Pj>7Dj_0^5othkPT^93O~UI#0uCdVCf~RUI(g z2J{I165P{lqykXP>@c8l&z>Y1%LYCw*NNXc9NhT0;bTp8`I=kY04oMN*z70vg_xHE zqec<;y+Mbm89r0cV?i26m@-wc6|~`hVsYmEz`wy+b?<)<7k<402-{bF0ZmkKWMGBN z-?}dP;Cm}vbiDo)$o}L-_}+VT&~YFb^@_3av;GiLFX`-~Y#DH@eO)NX_dfOcuF)cv zU{a9}bk;v&^Oo+&u-9eVG$|Rlnx)f87=HU=BfwS-%(fjN5zri}h*`NR23vUC-mU6Y zCsnESS0f$6b@7{aBrIp86Ym$WgR6dinYh=M_ugDz#gOtT@;(#Z?7fVsG^De|wTjij zU}d7=ZqGFra$d_LbF^i*v!lgA^)&)e?@zL&VR0_oy6)n@2QymvKUL?80?QEhR>aKWBJJX*jDWo@?mf+3*FTd2S0atH$}UaCrtA$~)i2uYW4BG~SY_ajWM z$?C32?+Lo`+C&dsS2r*9u8Lo-)RgwJLWrrq-gpXsrgr&nvN_%Rt=ttI7KnRJuD)+2a^z4fHY>ggxf$cC$C5*Z@3NQWI_l zQDYZ8y_<1}jfsehQSt=dv^}{%f`;5V$Vgof61wu~#ufR1aF~0zV~)z)-4{Mv|clTxM2l zR=bpUDQlKN6yf+P0StFVV}{DSGm;T1J-y;Z_JevtjPC(IqZ#l5FSGkB7PKGM%1hnx z0b^TGVR`8t^eD2E{bPPvn9Jj1@`6>1UAN~&?9zJuZ7}!-StwMGYm1d}*GzX{yOuq3 zo4V$mD6QfESlgOqj`)>+oob2u>cKm&o2zbE=p8q=GPl<%6Y+QZ6f+q2g7p|)X(N!F zBQ<p&%Bdr24GZ(g=Ds8RM0MV<{4x-xdz&>hUKzDKex#0Ah5li+XgS`taHD|Jo6ok0FJU)avUy2H_sh7+> z)r&4I_zEmgl0)GDhOv{1v>dr+&T1M5qFVC0*j@7-g?jJLXrZd_oxc~Q+o(rpzV_fC z2x98%Jaxf4nwsF#d;&yGgfp!aO|Z^TFH_y-K-#u6;;;{saa4{-)wmGD^ZTXk(aD~a z;g7tKmfBjyLd`@TgpYsk$$+o%fmh@yAoHTA(jLS+1yg&9t`;SSJqB%?b+qZ;VwM0M z$x(-N<|#|I+E-j;>`KuPFw_!@PV&KZZ?kj(6|>2a@z_G5qT+%g(3SvjVRJ~+qmx)@ zH@mvK`YAFyblHI`AW}kO0%PJlvcfYDjsAkP=VU*Gva#^TLtOafT!Xgy--eOl5Swv0B)r6=)U<_S z%5fSH!E#DA=R&WrUa{sK@>*LNtVv9vLNHaO1O?yX8aye_GJ6QBDC?nS+YJZgE&3Ii z8kCz3Wnt?YEzFZ?a2#`%%6I+a8%EL&3MTR*`XccKvfFS+dK9!)C?@abl4~kuQX8$% z-je-jiJ($NwLH{Lw8UE^;g5D3NMz6;@g*-a880 z5VD~;H>)jw7z!0!v|XG!1d8S6#ae=0%&DqWhO4E!;`L&RkJ#0v$>q;Bt1MKQOLoRe zvOf-2`+r89Ek9dgFzqy6B7c2kE}2n0k?oB4t55ffITo>Z*jd9d)U_%#Bi6wnQaI^2 zPkA3T2pKa?;d*aDuVwtGmT_9&~2d#yqLhoJhy3OX_3TIqq9$X;D+uO%S zY`h+?Y;2&WZ%b~mAuo>6c>aEO@I|Yyhj#XxUSj$Ao$}YOngaa8Oht=8(Dj;&sAFr8 z4ZR?#BkGo_YxpB4HSs&*OGN*W#AFlV<7W7~DSz!oxEX6T<_lAZmuE1uTvDB^*cG`< zTEJIrM1XqfBu4<_Z7Qc}Y>JynV2XTum! z+%t;Rvq&my{p#k*?lwc<)|^>!}mX06e$$6JII7_0&$E z_xlfEs#U+(=M5^-&%he*5)sWI#Ybeh@vFG7tzw`}S*SdGDC z4<@$c;Iggdj5Vb|9%)fPiS|=qGM)19O-K2in4QaSPSnXe--fvy!5=(LZO|K=Asv)` z0nBmdac6UFejObjet949oM>rRAATPn9eH{Ep6N0FBimbE2hppl7D(M+W7Yquu(Jw^ z1Bli%Hn_V78-fKLGz52dcXx;24nYD0cMF3%4DRkWK!RIv5AIp+-P&7qcW>1`bXV7T z>8gJEPuG9GpS(J!TCD23);GzvpL(hZhn&h>;2{Yz`t9FG;!6HD<9(0S-idNv}u^QjMw%?Hv7{gX&|b zvPpD`YRr$73wi=lPt6|j3shayuhUjPryR0bvrl-~q@74e*c)st{)}EYiaRn(M@w~m zwtFmdEYc{dNigDA;oI-xcr<7I-X^>si^?~cVCp94uAmYb&$ucE2PyQkFd3aYpr^65 zcL<}QVdbIu$ z{i0*;BCk>1%?>;J+(?fn#cJr^zZY3WK`Nh3nm}pSfz?vT+2P@=ncdgff}_;e9WyNi zVtBTM;+TNL*Fm_nvy8Pv!O2U$-xdH zaSfcOBK}>4I`Gjyz##}x9Axpp3uSwmlbGwcIq-#&s8Kl+yo9(|M00$z-Ee=!aoXLQ zLD8I!n#QbZteVbJGD#(tyt)<)<{eSo9dq2~MUY0Vt*j@4KNIeK<*@(izne zxmKYfl9XL@4tb%B!&L^T*j^iw}3Woogybw=d+_`FGY2YdqwP{_bRrnI!5x@?8~3oeF~7oWFr zpWgn+e^AhG^y#hzmb4W@*Qmnog8dIA9KY!&7zrF7(S=QMkQMPF1f?7&o_zyP)ng=X zySm~ere(ZGBHi8~ES}zp87l5_OEf&jMvM3(HGM~Om63H(l$Psk%vvXLUi)>@ zg?%{g5%8?2{iiVSi4W~#s-b9g%=lrXB7U(3Ili*0T7?J75A+p5%fxg>us4}M3|c>L zNA>#dJ@pS$_9ojgT_TV5qtf8~E$fGxZFI*SuX>m2N$+kLRe3h%h0HDW!CLd}!tzri z>I(WZ{GppGix?VLpmjYnS^|^&#hbttd)K7MBDq0-zd=|a@+J>YN0x40z6Rhc&RxIt zn3kRey?r=Gnw-v$QjR&uptH%gsN`~^fBTFp@L8~kMY9espo33V*!com+c#xr20G4Z zLQT3h=da?R>`3F)9p)EsnK^7)ICh|)0F`;P& ziiQcA?P=u>hGfdhqAWwzGRBZ^$aU`KQweyPBWTmkp$xOYVul7)4Ib`NkX{vvQjD_N zx6|bkJ!G3E?}+v?`RQl%#qsffEDSCpV?uIlbA7m=)})AH*{nSe_ik|!E6I>%x!%Ap zX9xH*b6KJ(C+SIrvb|CLAKj&Va9Je7{^(K-c(y-=!qcOqCz5h`QX#x}8?bw~ z!ntq+p&_<#ux}RVf)E5{wPK#RCUrICk_Cj}yr`%)(O0gcEu!l#IM|O?1Ln4*2+1KU z0LC6C{fhF?{G%FY4z&%R%Q})^UXIHp9eqPhBfIgFacQ+v*^R;EG0cYCPRZTD?1w)s z(ePd)P7j7GBi)>g*x92bPUn4v8+uqPB;MNZre+VVzr|a=Y-BwbpMM1$dY{dD8Yeup zoCwu#wckVIeESe57@MFNu@Obh+Kp(}$&u%Vzv~orQKwUDiG5gFQ)05gii|wsHr;JHX~M zNj4gP-KX{rzvPzt1#S2!SzG?f*R=p+kJHxQ-#WZ#?T;0izqk=myKN{}PHEsC znW*s+s`KxAzo=l!4&L&j>_Rjy)Es>~>O0Ul8KsjTSuI}=saR!RBhbJFzA@=bx?$Qc zAtY|LY!S>K(VZB{TDiI?mMZE-Zeke?+GBa6&4bO#JtBnb)Y+_wO`{0j^#4%YlR5+9 z75GJmX_FuLYuKAjaD>zX8XylTs#IZqg$<}2z<$!0Ejq1nS0 zv7o87`L zsVS;c*=&@Ke2SQ9eo_T9wIr#TT!RhS5igu_szUK4t26d8dk|w(4bkdE-3m>kOJ!<$ zLYuTaBv@bvzsh2|yR5QJMBG0|XD5U)*(VjYl#SvlHyQHq7 zu)e#y<3?SC4gOFm4agKoM{nzcE$2$y@bn2+wU6H z@+Fo8YCv7g2@Z^xbWVFSeKUPHrWVxu&?gy9mri>0pQ;+w4JC7G`k%tGZUotjN`9^6 ziJv7-H#oAG`opE-K(cNVFz>&e)zz?Vao+URU#SzR#_Q;98D%e_JQvO_tw^5GNVY49 z9#mkUkVHd1N0kv(sGA1_IEt_LWlc26w7Yw%oDzzBaPkjb?#XX^&3u7LA^RRu#-7Kh zm9f|aku5K;`P}g|D)`8{RDOlFzE7}?a2?{3U~Oj4)LLm4M&a1 zBiwXTNB&Sr(l)8&>2ig*-CoBhUACfPE!$9=Gie6D@>4Ft{)-pA>8SVBa92WI(+i7- z?%fB`I_e-c+6ah;xLgBMh}wtQPDkZShDq{{`>mTf!*jRtgR9`nCXf+g7VYP-cqQQ&p? zB?&zYI>N#SGx~92O>D)Jl0D{kBEJKAD^A7yQ4&P9Fa|7XuY0TU19uW1g*qWt$5>Ly;?T&k z3C3W5k$|CczuhoXmMSG>SH)sS%hqVmPYSGf%`(=0SS?z&Fap1@A$m%Jx-N3RI#7QP zK{q$)#=MP#5U^lQ3^*IT9M&_M@tb9PerAwo5*SH{28 zUk!HPk_&UQOQZMb=Ec;q^1z})XctH1;Ejc0MSR-GnB+03W^RoH?Mo?&r@zkE zHCu;(ZCFaSvh7?VtGh4Jqv)p_&_NZiDE;voIL^c`0?nU0KzK=1K%i`T_-KCAr^{EQN8qkuEhNNY_9zQ-ph|sGIO0B3r>dx?})RnAJBI zQydGkue&EqHWv~4O9S7ICsxXvj%l^VY%vf{ZlWnO8T8)=R~&}52h%!F_>u|9uF};n zWnRwHg-zQOA?=?^%bB(kB@_wy_j9mUnL~AL7=l%Fg8uFNxYl3n1V}_E?|6)mv zGZg*WpxNzr8a(7fSgV4eou0p0ln^>TwqcSDrfqZ&{iT%K@$~h?bN{a~Fp0P^+Mz)T zerQZ*QV!>M1a4eQ&TjoyVp_9kFZa}7&6s+2_zBdzx%%F_u?ky`Q!bggshDIh_*hYM z>I@|fyA4ZX*sn?gI?CjTcE*?hS-hF6S|`;zkWtq{44i>cu#>a$qCa~}x<&_}3_s-P z=)mZAnc|!tu04A23Q+VxDp}^RF}qC*Kid!yEahK*fntxixfa&5w|p;|hB{=4sp~h` zZ^0T6yu~iHEtO!b%Ve=`I2IJ51xZ^~F2`mPlyAjHMMi8v@Z(tNg=i#2vvcaa1pdHg zg3S}ot7k)yXdX|K#6MGx3v3(Q){`-O4QVxDb6;~SxRC%T{wad&yxIs^^pv_ zGG_bRd{RAZMd%d;C6GfI9AI@z+Sw!vuZFXDT6N>-G-GS%7|4pG+<75|@ayh8iG7oU zp_Z)P6i@gjkS(*}atAEN5WI8MASJPy52=j`;Q;Jf;fvznr)b63Qm>H0%v`qho1aAjN1%`RDELb10Re*sstf7z5f6t;!;=;G=7RmsiI^oTwNAXZ z;)*-N5)t$~9I8l1Ka$KDG)TC8m_+Kw!!u?}GK3!C`P_EsC=)J`UeHT^HdaA|uw*qK z6&gm2;*_e@ztMF-gI{>I^a7bOMzMa#(pj0voo6X%+fW7&$=q&fWJx>PP;%E(0#<1F zAb#1D+2GwAI%(CAbNYF)QM%!pTyVz zz~uZWRl)bLqNtd0xVMbzmFo4Cit5+KfWXfC#kSH7{`zUvGEPwDt3u3}x z2W0-?Tf@KOyfD}&R-&;63U~FBl*^XG^lnTj9I#=vPUaBAzYJ$GlfTBrYT3U8oTWc9 z!0@TvFySdxyX-QlcW7l$3@9Urv^%#MMdS_|2X^1cu?;zMRYe|-2WHF}P^N!K9N%OA zH&Ffh{Lo=^n6j7t&-nch=e*I)t9wp=Vv!4$gXwMDkr6)lmJ|lOTbna(3d7I=dwBTB zOyolzJdF!OmtQ~saJ1=Gwzv!ns{%tI1M5?O4hqI(M=0rqTY{0^LhK9{MuyFXAsY;q z=tUwiVO|1UiBdDbB8d`|0WbHW}<3uem*^%@)HJkSbSYPu4rGyc+LXCqDST!?;6UF%&e8W4^ljy6vGC}yHvB4(m#;%|IG$y77>Wr?wp z6!mGfa}_Jezd!AM-u=Y+xhh#HITIZpJsmqYh(6dkh&vQBm^I`SrKn!7!CgU{Dn3<2 zS!7hGG>e_ZIh8kDa#q4UTLYy(B5oI0Po;`#4i;_iv>NrN~zxVN(*S(6zR8Ijq) zGUDqGS)8?SPTEOUCZ`>~E&DVk?9=a;?^{p)tDCKlYO!g~Fe>ei@2D|d4=+x#47bcJ z?v1e}aPnM`GpV+rFg0|3U6WdlG@rNOvaGZvv-TUmYs&4W*sNK1>f2Zg%zgzT4fZV; z_L|Nn9usfr>^)4%jxny357|yHFY%8&CZVkity}g?WK`xbuQrW&PERd3X6xsF9$TO3 z8h>s3mWH%hw2F7C$ ztIhk3S+5(ggO+77|9bk z?d-DV;xb^7;i?n3bSQNjmr3UAkhGCfchqk+z8^JZJutT|x;?d>PuodzOuJ8ONNZ1X zuyl7SeyUAQrbJ>LX+QR9Pf1c&Dpg8RT3_lzR9!;>X=dm@;6mV`zKAjAYvQUQL$kzkj5)no+^>($Ri4R91 zpg70{ba=iRS~)Llfnb$0g+H-tiD_YLc{Y_b?lmry7dCEsaO0%8)^30Dz_aS;xmtLZ zxa#+n;bL@^qix>jV9YSw@Nb9H!SO-be)lQ`q~o(PtF!gS;MeKRrrp@x>+6v_ozwDH z%G-d``!K)X=hbV^O~L9ejVe8LotN%79f$i^h{75#br3eYO_O@rzlpdCD76+IJ-2Vk z#b(9J#dt`9SRN@H6R6`G6l?yMQf;a}XVm?L5b;pdgqRw+)bXV88BGpN+C{HLNA3o0 zKYKziq_(!(CY zMfeu@Vyv5fzNExzd4Gqn6fYXJmVqi5vjm|S53?6&yH83bz`L*Z9(Gz(HAjStiq4A z)<$OHb`yOXf z+|)c2DF+j~d+pNgN)uW+wEVI;6jeo#P=QdHWc09OJ2+%REqyW;{6A_`Tt;f z|C9F2rlV$Q^rlMxy|@^OxbzN_(33ZQxK=Q1s}U5 z$czKTWx>T~&SPmRWcc62sJR&o9K94`Aokzr5R_4Ky0C|tq}$Z!kTx6}m_HzFR{|NG z3^ Date: Sun, 4 Oct 2026 10:37:24 +0200 Subject: [PATCH 4/5] fix(documents): log why a document got no thumbnail Refs #1124. The thumbnail job opened no wide event, so every annotation it made went nowhere: one instance ran it hundreds of times, left sixteen PDFs without a preview, and the log never said so. Each run is now a `job.document_thumbnail` background event. A run that should have produced a preview and did not is raised to warn and names the reason; a type that never has a preview stays at info. --- .../__tests__/native-canvas-support.test.ts | 2 +- src/lib/documents/__tests__/thumbnail.test.ts | 9 +- src/lib/documents/thumbnail.ts | 34 +++++-- .../jobs/__tests__/document-thumbnail.test.ts | 98 +++++++++++++++++++ src/lib/jobs/document-thumbnail.ts | 26 ++++- 5 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 src/lib/jobs/__tests__/document-thumbnail.test.ts diff --git a/src/lib/documents/__tests__/native-canvas-support.test.ts b/src/lib/documents/__tests__/native-canvas-support.test.ts index df978c965..df282df1d 100644 --- a/src/lib/documents/__tests__/native-canvas-support.test.ts +++ b/src/lib/documents/__tests__/native-canvas-support.test.ts @@ -115,6 +115,6 @@ describe("generateThumbnail behind a closed gate", () => { Buffer.from([0xff, 0xd8, 0xff]), "image/jpeg", ); - expect(result).toEqual({ ok: false }); + expect(result).toEqual({ ok: false, reason: "native-canvas-unsupported" }); }); }); diff --git a/src/lib/documents/__tests__/thumbnail.test.ts b/src/lib/documents/__tests__/thumbnail.test.ts index 59a03ab64..05a2630d6 100644 --- a/src/lib/documents/__tests__/thumbnail.test.ts +++ b/src/lib/documents/__tests__/thumbnail.test.ts @@ -126,19 +126,19 @@ describe("generateThumbnail", () => { it("returns { ok: false } for an unsupported MIME type", async () => { await expect( generateThumbnail(Buffer.from("plain text"), "text/plain"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "unsupported-type" }); }); it("returns { ok: false } and never throws on malformed image bytes", async () => { await expect( generateThumbnail(Buffer.from("not a real image"), "image/png"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "error" }); }); it("returns { ok: false } on a malformed PDF", async () => { await expect( generateThumbnail(Buffer.from("not a pdf"), "application/pdf"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "raster-failed" }); }); it("refuses a decompression bomb before decoding (pixel cap)", async () => { @@ -153,6 +153,7 @@ describe("generateThumbnail", () => { bomb.writeUInt32BE(30_000, 20); // height await expect(generateThumbnail(bomb, "image/png")).resolves.toEqual({ ok: false, + reason: "pixel-cap", }); }); @@ -164,6 +165,6 @@ describe("generateThumbnail", () => { loadImageMock.mockResolvedValueOnce({ width: 30_000, height: 30_000 }); await expect( generateThumbnail(Buffer.from("desynced jpeg bytes"), "image/jpeg"), - ).resolves.toEqual({ ok: false }); + ).resolves.toEqual({ ok: false, reason: "pixel-cap" }); }); }); diff --git a/src/lib/documents/thumbnail.ts b/src/lib/documents/thumbnail.ts index 8d975954f..26ed69c1b 100644 --- a/src/lib/documents/thumbnail.ts +++ b/src/lib/documents/thumbnail.ts @@ -157,9 +157,23 @@ export interface Thumbnail { height: number; } +/** + * Why no preview was made. `unsupported-type` is the expected answer for an + * Office file or a text document; every other reason means a preview should + * have existed, so the thumbnail job logs it as a warning. + */ +export type ThumbnailSkipReason = + | "unsupported-type" + | "native-canvas-unsupported" + | "pixel-cap" + | "raster-failed" + | "empty-render" + | "error"; + /** Best-effort generation outcome — never an exception. */ export type ThumbnailResult = - { ok: true; thumbnail: Thumbnail } | { ok: false }; + | { ok: true; thumbnail: Thumbnail } + | { ok: false; reason: ThumbnailSkipReason }; // Minimal structural types for the slice of `@napi-rs/canvas` this module uses. // A type-only shape (erased at build) so we never eagerly evaluate the module @@ -229,7 +243,7 @@ export async function generateThumbnail( action: { name: "documents.thumbnail.rejected" }, meta: { reason: "native_canvas_unsupported" }, }); - return { ok: false }; + return { ok: false, reason: "native-canvas-unsupported" }; } // Lazy import: keeps @napi-rs/canvas (+ transitively pdfjs on the PDF path) // out of the eager server chunk. Resolved on first generation, then cached. @@ -244,7 +258,7 @@ export async function generateThumbnail( action: { name: "documents.thumbnail.rejected" }, meta: { source: "image", reason: "pixel_cap", pixels }, }); - return { ok: false }; + return { ok: false, reason: "pixel-cap" }; } const image = await napi.loadImage(bytes); // Post-decode backstop: the header sniff returns null on any marker @@ -260,10 +274,10 @@ export async function generateThumbnail( pixels: image.width * image.height, }, }); - return { ok: false }; + return { ok: false, reason: "pixel-cap" }; } const thumb = drawScaledJpeg(napi, image); - if (!thumb) return { ok: false }; + if (!thumb) return { ok: false, reason: "empty-render" }; annotate({ action: { name: "documents.thumbnail.ok" }, meta: { source: "image", width: thumb.width, height: thumb.height }, @@ -274,11 +288,13 @@ export async function generateThumbnail( if (mimeType === "application/pdf") { // Page 1 only — a preview never renders the whole document. const raster = await rasterizePdf(bytes, 1); - if (!raster.ok || raster.images.length === 0) return { ok: false }; + if (!raster.ok || raster.images.length === 0) { + return { ok: false, reason: "raster-failed" }; + } const pageJpeg = Buffer.from(raster.images[0]!.dataBase64, "base64"); const image = await napi.loadImage(pageJpeg); const thumb = drawScaledJpeg(napi, image); - if (!thumb) return { ok: false }; + if (!thumb) return { ok: false, reason: "empty-render" }; annotate({ action: { name: "documents.thumbnail.ok" }, meta: { source: "pdf", width: thumb.width, height: thumb.height }, @@ -287,7 +303,7 @@ export async function generateThumbnail( } // Every other MIME (Office/text/TIFF/HEIC/XML/JSON) — no preview. - return { ok: false }; + return { ok: false, reason: "unsupported-type" }; } catch (err) { annotate({ action: { name: "documents.thumbnail.failed" }, @@ -297,6 +313,6 @@ export async function generateThumbnail( message: err instanceof Error ? err.message.slice(0, 300) : String(err), }, }); - return { ok: false }; + return { ok: false, reason: "error" }; } } diff --git a/src/lib/jobs/__tests__/document-thumbnail.test.ts b/src/lib/jobs/__tests__/document-thumbnail.test.ts new file mode 100644 index 000000000..4270507d1 --- /dev/null +++ b/src/lib/jobs/__tests__/document-thumbnail.test.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +/** + * GitHub #1124 — the thumbnail job ran 355 times on one instance, left + * sixteen PDFs without a preview, and the log held not one line about it: + * the job opened no wide event, so every annotation went nowhere. Pins: each + * run is a `job.document_thumbnail` background event; a run that should have + * made a preview and did not is raised to `warn` with the reason; a type that + * never has a preview stays at info. + */ + +const { evt, findFirst, upsert, generateThumbnail, withBackgroundEvent } = + vi.hoisted(() => { + const evt = { addMeta: vi.fn(), elevateLevel: vi.fn() }; + return { + evt, + findFirst: vi.fn(), + upsert: vi.fn(), + generateThumbnail: vi.fn(), + withBackgroundEvent: vi.fn( + async (_name: string, fn: (e: typeof evt) => Promise) => + fn(evt), + ), + }; + }); + +vi.mock("@/lib/db", () => ({ + prisma: { + inboundDocument: { findFirst }, + documentThumbnail: { upsert }, + }, +})); +vi.mock("@/lib/documents/store", () => ({ + decryptDocumentContent: vi.fn(() => Buffer.from("%PDF-1.7")), + encryptThumbnail: vi.fn(() => new Uint8Array([1])), +})); +vi.mock("@/lib/documents/thumbnail", () => ({ generateThumbnail })); +vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() })); +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); +vi.mock("@/lib/logging/background", () => ({ withBackgroundEvent })); + +import { runDocumentThumbnail } from "../document-thumbnail"; + +const DOC = { + id: "doc-1", + mimeType: "application/pdf", + contentEncrypted: new Uint8Array([1]), + contentCodec: "binary2", + thumbnail: null, +}; + +beforeEach(() => { + vi.clearAllMocks(); + findFirst.mockResolvedValue(DOC); +}); + +describe("runDocumentThumbnail wide event", () => { + it("runs inside a background event named for the job", async () => { + generateThumbnail.mockResolvedValue({ ok: false, reason: "raster-failed" }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(withBackgroundEvent).toHaveBeenCalledWith( + "job.document_thumbnail", + expect.any(Function), + ); + expect(evt.addMeta).toHaveBeenCalledWith("document_id", "doc-1"); + }); + + it("warns with the reason when a PDF gets no preview", async () => { + generateThumbnail.mockResolvedValue({ ok: false, reason: "error" }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(evt.elevateLevel).toHaveBeenCalledWith("warn"); + expect(evt.addMeta).toHaveBeenCalledWith( + "thumbnail_missing_reason", + "error", + ); + expect(upsert).not.toHaveBeenCalled(); + }); + + it("stays at info for a type that never has a preview", async () => { + findFirst.mockResolvedValue({ ...DOC, mimeType: "text/plain" }); + generateThumbnail.mockResolvedValue({ + ok: false, + reason: "unsupported-type", + }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(evt.elevateLevel).not.toHaveBeenCalled(); + }); + + it("stores the preview and stays at info when one is made", async () => { + generateThumbnail.mockResolvedValue({ + ok: true, + thumbnail: { jpeg: Buffer.from([0xff, 0xd8]), width: 2, height: 3 }, + }); + await runDocumentThumbnail({ userId: "user-1", documentId: "doc-1" }); + expect(upsert).toHaveBeenCalledOnce(); + expect(evt.elevateLevel).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/document-thumbnail.ts b/src/lib/jobs/document-thumbnail.ts index 91a8d44ed..1b0a1a91d 100644 --- a/src/lib/jobs/document-thumbnail.ts +++ b/src/lib/jobs/document-thumbnail.ts @@ -15,6 +15,12 @@ * kind icon), and `runDocumentThumbnail` returns cleanly so pg-boss does not * retry it. Only an unexpected throw (a DB hiccup) propagates for retry. * + * Each run is one background wide event (`job.document_thumbnail`). A run + * that should have produced a preview and did not is raised to `warn` with + * the reason, so a document left with its kind icon is explained in the log. + * Without the event every annotation below went nowhere: a whole class of + * PDFs went without previews for weeks and the log never said so. + * * The queue MUST be registered in the maintenance registrar * (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it. */ @@ -25,6 +31,7 @@ import { } from "@/lib/documents/store"; import { generateThumbnail } from "@/lib/documents/thumbnail"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; export const DOCUMENT_THUMBNAIL_QUEUE = "document-thumbnail"; @@ -49,7 +56,20 @@ export async function runDocumentThumbnail( ): Promise { const { userId, documentId } = payload; if (!userId || !documentId) return; + await withBackgroundEvent("job.document_thumbnail", async (evt) => { + evt.addMeta("document_id", documentId); + await renderThumbnail(userId, documentId, (reason) => { + evt.addMeta("thumbnail_missing_reason", reason); + evt.elevateLevel("warn"); + }); + }); +} +async function renderThumbnail( + userId: string, + documentId: string, + warn: (reason: string) => void, +): Promise { // Owner-scoped load; the blob column is selected ONLY here (the job that // renders the preview), never in the list/detail queries. Skip a document // that already has a thumbnail so a re-enqueue is a cheap no-op. @@ -84,6 +104,7 @@ export async function runDocumentThumbnail( action: { name: "documents.thumbnail.decryptFailed" }, meta: { documentId }, }); + warn("decrypt-failed"); return; } @@ -91,8 +112,11 @@ export async function runDocumentThumbnail( if (!result.ok) { annotate({ action: { name: "documents.thumbnail.run" }, - meta: { documentId, generated: false }, + meta: { documentId, generated: false, mimeType: document.mimeType }, }); + // An Office or text document has no preview by design; anything else + // was meant to get one. + if (result.reason !== "unsupported-type") warn(result.reason); return; } From 9cb06ec44d2ac322f31d234ef88997827f365c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 10:43:00 +0200 Subject: [PATCH 5/5] fix(idempotency): read a rebuilt multipart form as the same request A client that rebuilds a form for a retry picks a fresh random boundary, so the raw bytes differ and a repeated document upload ran a second time instead of replaying. The boundary named in Content-Type is replaced by a fixed token before hashing. --- src/lib/__tests__/idempotency.test.ts | 32 +++++++++++++++++++++++ src/lib/idempotency.ts | 37 +++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/lib/__tests__/idempotency.test.ts b/src/lib/__tests__/idempotency.test.ts index a93a23ee1..37efb2d1c 100644 --- a/src/lib/__tests__/idempotency.test.ts +++ b/src/lib/__tests__/idempotency.test.ts @@ -990,6 +990,38 @@ describe("request fingerprint", () => { expect(sys).not.toBe(dia); }); + it("reads a rebuilt multipart form with a fresh boundary as the same upload", () => { + const form = (boundary: string, name: string) => + enc( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${name}"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4 body\r\n--${boundary}--\r\n`, + ); + const type = (boundary: string) => + `multipart/form-data; boundary=${boundary}`; + const first = requestFingerprint( + "POST", + "/api/documents", + form("----formdata-a1", "brief.pdf"), + null, + type("----formdata-a1"), + ); + const retry = requestFingerprint( + "POST", + "/api/documents", + form("----formdata-b2", "brief.pdf"), + null, + type("----formdata-b2"), + ); + expect(retry).toBe(first); + const other = requestFingerprint( + "POST", + "/api/documents", + form("----formdata-b2", "other.pdf"), + null, + type("----formdata-b2"), + ); + expect(other).not.toBe(first); + }); + it("hashes a non-JSON or empty body as its raw bytes", () => { const text = requestFingerprint("POST", "/p", enc("not json")); expect(text).toBe(requestFingerprint("POST", "/p", enc("not json"))); diff --git a/src/lib/idempotency.ts b/src/lib/idempotency.ts index 82d2cc0b6..7b374faf3 100644 --- a/src/lib/idempotency.ts +++ b/src/lib/idempotency.ts @@ -158,20 +158,52 @@ export function canonicalJson(value: unknown): string { return JSON.stringify(value); } +/** + * Replace every occurrence of the multipart boundary named in `contentType` + * with a fixed token. Any other body is returned as it is. + */ +function withoutMultipartBoundary( + body: Uint8Array, + contentType: string | null, +): Uint8Array { + if (!contentType || !/^multipart\//i.test(contentType)) return body; + const match = /;\s*boundary="?([^";]+)"?/i.exec(contentType); + if (!match) return body; + const boundary = Buffer.from(match[1], "utf8"); + const token = Buffer.from("boundary", "utf8"); + const source = Buffer.from(body.buffer, body.byteOffset, body.byteLength); + const parts: Buffer[] = []; + let from = 0; + for (;;) { + const at = source.indexOf(boundary, from); + if (at === -1) break; + parts.push(source.subarray(from, at), token); + from = at + boundary.length; + } + if (parts.length === 0) return body; + parts.push(source.subarray(from)); + return new Uint8Array(Buffer.concat(parts)); +} + /** * The fingerprint of a request: SHA-256 hex over method, path and the body in * canonical form. A body that parses as JSON is hashed as `canonicalJson`, so * the same object serialised with a different key order is the same request; * anything else (multipart, plain text, an empty body) is hashed as its raw * bytes. `truncatedAt` marks a body cut at `FINGERPRINT_MAX_BYTES`, hashed as - * that raw prefix together with the declared length. Exported for unit tests. + * that raw prefix together with the declared length. A multipart body has its + * boundary replaced by a fixed token first: a client that rebuilds the form + * for a retry picks a fresh random boundary, and the same upload must still + * replay. Exported for unit tests. */ export function requestFingerprint( method: string, path: string, - body: Uint8Array, + rawBody: Uint8Array, truncatedAt: string | null = null, + contentType: string | null = null, ): string { + const body = withoutMultipartBoundary(rawBody, contentType); const hash = createHash("sha256") .update(method) .update("\0") @@ -237,6 +269,7 @@ async function fingerprintRequest( path, body, truncated ? (request.headers.get("content-length") ?? "unknown") : null, + request.headers.get("content-type"), ); }