From 813dc2547b3abd30be897bb04ad32bc38720d267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:32:59 +0200 Subject: [PATCH 1/8] feat(documents): add per-user automatic AI reading opt-in Add the documentsAutoAiRead per-user Boolean (migration 0231, default false) that authorises ambient AI reading of uploaded vault documents. OFF by default keeps the vault local-first for every self-hoster. - schema column + migration 0231_documents_auto_ai_read - documentAutoReadEnabled(userId) read helper - GET/PATCH /api/auth/me/documents-auto-ai-read toggle route; the ON write mints an ai_full consent receipt for the audit trail - the document consent gate short-circuits an external pick when the toggle is ON; a local pick stays ungated and OFF still requires a receipt for any external egress --- .../0231_documents_auto_ai_read/migration.sql | 17 +++ prisma/schema.prisma | 12 ++ .../auth/me/documents-auto-ai-read/route.ts | 129 ++++++++++++++++++ src/lib/ai/__tests__/consent-guard.test.ts | 56 +++++++- src/lib/ai/consent-guard.ts | 12 ++ src/lib/documents/document-settings.ts | 28 ++++ 6 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 prisma/migrations/0231_documents_auto_ai_read/migration.sql create mode 100644 src/app/api/auth/me/documents-auto-ai-read/route.ts create mode 100644 src/lib/documents/document-settings.ts diff --git a/prisma/migrations/0231_documents_auto_ai_read/migration.sql b/prisma/migrations/0231_documents_auto_ai_read/migration.sql new file mode 100644 index 000000000..5524b0512 --- /dev/null +++ b/prisma/migrations/0231_documents_auto_ai_read/migration.sql @@ -0,0 +1,17 @@ +-- Automatic AI reading of uploaded vault documents (per-user opt-in). +-- +-- One additive, defaulted column on `users`. OFF by default: the document vault +-- stays local-first — a freshly uploaded document is only read by an external AI +-- provider when the user explicitly taps a per-document action (which still +-- requires an active consent receipt). When `true`, the auto-index-on-upload job +-- may read a new document through the user's configured external provider with +-- no per-document tap; flipping the toggle on is itself the standing consent +-- (the write that sets it also mints an `ai_full` consent receipt for the audit +-- trail). +-- +-- The default `false` backfills every existing row onto the current behaviour +-- (local-first, per-document-explicit, consent-required), so this is a no-op for +-- anyone who never opts in. + +ALTER TABLE "users" + ADD COLUMN "documents_auto_ai_read" BOOLEAN NOT NULL DEFAULT false; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index cdd222bd4..dcbe9c1a2 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -302,6 +302,18 @@ model User { aiOcrBaseUrl String? @map("ai_ocr_base_url") // LOCAL OCR provider only aiOcrKeyEncrypted String? @map("ai_ocr_key_encrypted") + // Automatic AI reading of uploaded vault documents. OFF by default: the vault + // stays local-first — a newly stored document is only read externally when the + // user explicitly taps a per-document AI action (which still needs an active + // consent receipt). When `true`, the auto-index job may read a freshly + // uploaded document through the user's configured external provider with no + // per-document tap: the toggle IS the standing consent (the write that flips + // it on also mints an `ai_full` receipt for the audit trail). Mirrors + // `labsLocalOcrEnabled`; all AI provider config is per-user, so the operator + // flips it for their own account. Default `false` is the entire privacy + // contract for every other self-hoster. + documentsAutoAiRead Boolean @default(false) @map("documents_auto_ai_read") + // Onboarding onboardingCompletedAt DateTime? @map("onboarding_completed_at") // v1.4.25 W14b — last persisted onboarding wizard step. diff --git a/src/app/api/auth/me/documents-auto-ai-read/route.ts b/src/app/api/auth/me/documents-auto-ai-read/route.ts new file mode 100644 index 000000000..4bd8f1a62 --- /dev/null +++ b/src/app/api/auth/me/documents-auto-ai-read/route.ts @@ -0,0 +1,129 @@ +/** + * Per-user "read documents automatically with AI" opt-in endpoint. + * + * GET /api/auth/me/documents-auto-ai-read — current flag. + * PATCH /api/auth/me/documents-auto-ai-read — body `{ documentsAutoAiRead: boolean }`. + * + * OFF by default: the document vault stays local-first and every external AI + * egress needs an explicit per-document action + an active consent receipt. When + * `true`, the auto-index-on-upload job may read a freshly uploaded document + * through the user's configured external provider with no per-document tap. + * + * Flipping it ON is itself the standing consent act, so the write also mints an + * append-only `ai_full` consent receipt (`ensureWebAiConsentReceipt`) — the + * durable audit record that sits alongside the runtime short-circuit the + * document consent gate reads. Mirrors `auth/me/labs-local-ocr`: 60/min rate + * limit, Zod `safeParse` → 422 via `returnAllZodIssues`, audit-log row, + * field-by-field write (no mass assignment). Idempotent — always returns the + * resolved next state so the client can hard-set the optimistic update. + */ +import { z } from "zod"; + +import { apiHandler, requireAuth } from "@/lib/api-handler"; +import { + apiError, + apiSuccess, + getClientIp, + returnAllZodIssues, + safeJson, +} from "@/lib/api-response"; +import { annotate } from "@/lib/logging/context"; +import { auditLog } from "@/lib/auth/audit"; +import { ensureWebAiConsentReceipt } from "@/lib/consent/web-grant"; +import { prisma } from "@/lib/db"; +import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit"; + +const patchBodySchema = z.object({ + documentsAutoAiRead: z.boolean(), +}); + +export const dynamic = "force-dynamic"; + +const PATCH_RATE_LIMIT = 60; +const PATCH_WINDOW_MS = 60_000; + +type DocumentsAutoAiReadResponse = { + documentsAutoAiRead: boolean; +}; + +export const GET = apiHandler(async () => { + const { user } = await requireAuth(); + annotate({ action: { name: "auth.me.documentsAutoAiRead.get" } }); + + const row = await prisma.user.findUnique({ + where: { id: user.id }, + select: { documentsAutoAiRead: true }, + }); + const payload: DocumentsAutoAiReadResponse = { + documentsAutoAiRead: row?.documentsAutoAiRead ?? false, + }; + return apiSuccess(payload); +}); + +export const PATCH = apiHandler(async (req: Request) => { + const { user } = await requireAuth(); + + const rl = await checkRateLimit( + `documents-auto-ai-read:patch:${user.id}`, + PATCH_RATE_LIMIT, + PATCH_WINDOW_MS, + ); + if (!rl.allowed) { + const response = apiError("Too many requests", 429); + for (const [k, v] of Object.entries(rateLimitHeaders(rl))) { + response.headers.set(k, v); + } + return response; + } + + // The body is a single boolean — bound the parse so a malformed or oversized + // payload is rejected before it is materialised. + const { data: body, error: jsonError } = await safeJson(req, { + maxBytes: 1024, + }); + if (jsonError) return jsonError; + + const parsed = patchBodySchema.safeParse(body); + if (!parsed.success) { + annotate({ + action: { name: "auth.me.documentsAutoAiRead.patch.invalid_shape" }, + }); + return returnAllZodIssues(parsed.error, 422); + } + + const next = parsed.data.documentsAutoAiRead; + + const previous = await prisma.user.findUnique({ + where: { id: user.id }, + select: { documentsAutoAiRead: true }, + }); + + await prisma.user.update({ + where: { id: user.id }, + data: { documentsAutoAiRead: next }, + }); + + // Turning the toggle ON is the standing consent act — mint an append-only + // `ai_full` receipt (idempotent) so the durable audit trail records it, in + // addition to the runtime short-circuit the document consent gate reads. + if (next) { + await ensureWebAiConsentReceipt(user.id); + } + + await auditLog("user.documentsAutoAiRead.update", { + userId: user.id, + ipAddress: getClientIp(req), + details: { + previous: previous?.documentsAutoAiRead ?? false, + next, + }, + }); + + annotate({ + action: { name: "auth.me.documentsAutoAiRead.patch" }, + meta: { documentsAutoAiRead: next }, + }); + + const payload: DocumentsAutoAiReadResponse = { documentsAutoAiRead: next }; + return apiSuccess(payload); +}); diff --git a/src/lib/ai/__tests__/consent-guard.test.ts b/src/lib/ai/__tests__/consent-guard.test.ts index db055bbfb..cabf7f6f1 100644 --- a/src/lib/ai/__tests__/consent-guard.test.ts +++ b/src/lib/ai/__tests__/consent-guard.test.ts @@ -3,6 +3,9 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; vi.mock("@/lib/consent/receipts", () => ({ latestActiveReceipt: vi.fn(), })); +vi.mock("@/lib/documents/document-settings", () => ({ + documentAutoReadEnabled: vi.fn().mockResolvedValue(false), +})); import { ConsentRequiredError, @@ -13,10 +16,12 @@ import { isExternalDocumentEgress, } from "../consent-guard"; import { latestActiveReceipt } from "@/lib/consent/receipts"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import type { ProviderChainResolved } from "../provider-runner"; import type { ConsentKind } from "@/lib/validations/consent"; const mockedLatest = vi.mocked(latestActiveReceipt); +const mockedAutoRead = vi.mocked(documentAutoReadEnabled); /** Minimal chain entry — the gate only inspects `providerType`. */ function entry(providerType: string): ProviderChainResolved { @@ -184,7 +189,12 @@ describe("isExternalDocumentEgress", () => { }); describe("assertDocumentEgressConsent", () => { - beforeEach(() => mockedLatest.mockReset()); + beforeEach(() => { + mockedLatest.mockReset(); + mockedAutoRead.mockReset(); + // Default: the auto-read toggle is OFF (the shipped privacy posture). + mockedAutoRead.mockResolvedValue(false); + }); it("never reads consent and never throws for a LOCAL document pick", async () => { grant([]); @@ -196,6 +206,8 @@ describe("assertDocumentEgressConsent", () => { }), ).resolves.toBeUndefined(); expect(mockedLatest).not.toHaveBeenCalled(); + // A local pick short-circuits before the toggle is even consulted. + expect(mockedAutoRead).not.toHaveBeenCalled(); }); // The live gap the governance fix closes: codex was ungated for documents. @@ -233,4 +245,46 @@ describe("assertDocumentEgressConsent", () => { }), ).resolves.toBeUndefined(); }); + + // Gate A: the documentsAutoAiRead opt-in short-circuits an external pick. + it("proceeds for an external pick when documentsAutoAiRead is ON, without a receipt", async () => { + grant([]); + mockedAutoRead.mockResolvedValue(true); + for (const p of ["codex", "openai", "anthropic", "admin-openai"]) { + await expect( + assertDocumentEgressConsent({ + userId: "u1", + providerType: p, + surface: "insights", + }), + ).resolves.toBeUndefined(); + } + // The toggle short-circuits BEFORE the receipt read. + expect(mockedLatest).not.toHaveBeenCalled(); + }); + + it("STILL requires a receipt for an external pick when documentsAutoAiRead is OFF", async () => { + grant([]); + mockedAutoRead.mockResolvedValue(false); + await expect( + assertDocumentEgressConsent({ + userId: "u1", + providerType: "codex", + surface: "insights", + }), + ).rejects.toBeInstanceOf(ConsentRequiredError); + }); + + it("does NOT consult the toggle for a LOCAL pick even when it would be ON", async () => { + grant([]); + mockedAutoRead.mockResolvedValue(true); + await expect( + assertDocumentEgressConsent({ + userId: "u1", + providerType: "local", + surface: "insights", + }), + ).resolves.toBeUndefined(); + expect(mockedAutoRead).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/ai/consent-guard.ts b/src/lib/ai/consent-guard.ts index 494042b32..e6960c3aa 100644 --- a/src/lib/ai/consent-guard.ts +++ b/src/lib/ai/consent-guard.ts @@ -28,6 +28,7 @@ * mapped kind OR the superset `ai_full` grant satisfies the gate. */ import { latestActiveReceipt } from "@/lib/consent/receipts"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import type { ConsentKind } from "@/lib/validations/consent"; import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; @@ -153,6 +154,16 @@ export function isExternalDocumentEgress(providerType: string): boolean { * machine); for any external pick, throw `ConsentRequiredError` unless an active * receipt of the surface's mapped kind (or `ai_full`) is on file. * + * ONE relaxation: the per-user `documentsAutoAiRead` opt-in. Flipping it ON is + * itself the standing consent act (the toggle write also mints an `ai_full` + * receipt for the audit trail), so the gate short-circuits an external pick when + * it is ON — that is what removes the per-document friction ("upload and the AI + * just reads it, no 80 switches"). When it is OFF the branch is inert and the + * gate behaves exactly as shipped: no external egress without an explicit + * receipt. The short-circuit is checked BEFORE the receipt read and ONLY for an + * external pick — a `local` pick is already ungated above, so the toggle can + * never widen egress that was not already external. + * * Call this AFTER the document provider is picked and BEFORE the first * `generateCompletion` on it. */ @@ -162,6 +173,7 @@ export async function assertDocumentEgressConsent(args: { surface: ConsentSurface; }): Promise { if (!isExternalDocumentEgress(args.providerType)) return; + if (await documentAutoReadEnabled(args.userId)) return; if (await hasActiveConsentForSurface(args.userId, args.surface)) return; throw new ConsentRequiredError(args.surface); } diff --git a/src/lib/documents/document-settings.ts b/src/lib/documents/document-settings.ts new file mode 100644 index 000000000..22d9b4f8c --- /dev/null +++ b/src/lib/documents/document-settings.ts @@ -0,0 +1,28 @@ +/** + * Per-user document-vault AI settings. + * + * `documentAutoReadEnabled(userId)` reads the `documentsAutoAiRead` opt-in — the + * single switch that authorises ambient (no-per-document-tap) AI reading of + * uploaded documents. OFF by default: the vault stays local-first and every + * external egress needs an explicit per-document action + consent receipt. + * + * Two gates read this flag: + * - the document consent gate (`assertDocumentEgressConsent`) short-circuits an + * external pick when it is ON (the toggle IS the standing consent); + * - the auto-index-on-upload job stays strictly local when it is OFF and uses + * the document-order external pick when it is ON. + */ +import { prisma } from "@/lib/db"; + +/** + * True when the user has opted into automatic AI reading of uploaded documents. + * A missing row (deleted account raced against a job) resolves to `false` — + * fail-closed to the local-first posture, never to external egress. + */ +export async function documentAutoReadEnabled(userId: string): Promise { + const row = await prisma.user.findUnique({ + where: { id: userId }, + select: { documentsAutoAiRead: true }, + }); + return row?.documentsAutoAiRead ?? false; +} From 73a3ac9c363ededf96fd1a38fab8d453a0892ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:36:14 +0200 Subject: [PATCH 2/8] fix(documents): reconcile the auto-index job onto the document order The auto-index-on-upload job resolved over the cost-first app-wide chain (codex priority 1) and gated with the chain-based consent check, so a pure-codex chain could silently egress a freshly uploaded document to the subscription backend on upload. Reconcile it onto resolveDocumentVisionProvider (local-first, codex last) and govern external egress with the documentsAutoAiRead opt-in: a local pick always runs (no egress); an external pick runs only when the toggle is ON, otherwise the job stays strictly local. This tightens the default and makes the opt-in path deliberate. --- .../__tests__/index-document.test.ts | 94 +++++++++++++------ src/lib/documents/index-document.ts | 74 +++++++++------ 2 files changed, 109 insertions(+), 59 deletions(-) diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index 32a4e6d29..d9a145bc6 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -1,12 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; /** - * The AI-first / local-fallback content-index decision tree. Pins: a usable - * provider takes the vision path; a text-layer PDF the provider cannot read - * falls through to local; no provider / no consent / budget-out / provider - * error all fall through to the free local path; decrypt + not-found fail - * closed. No plaintext is asserted at the upsert boundary (source + ciphertext - * path is the content-index module's concern, mocked here). + * The AI-first / local-fallback content-index decision tree. Pins: the DOCUMENT + * provider order drives the pick; a LOCAL pick is always eligible; an EXTERNAL + * pick is eligible only when the `documentsAutoAiRead` toggle is ON (OFF → the + * job stays strictly local, never egresses on upload); a text-layer PDF the + * provider cannot read falls through to local; no provider / budget-out / + * provider error all fall through to the free local path; decrypt + not-found + * fail closed. No plaintext is asserted at the upsert boundary (source + + * ciphertext path is the content-index module's concern, mocked here). */ vi.mock("@/lib/documents/ai-route-support", () => ({ @@ -28,12 +30,15 @@ vi.mock("@/lib/documents/store", () => ({ vi.mock("@/lib/labs/ocr-upload", () => ({ detectOcrMimeType: vi.fn(() => "application/pdf"), })); -vi.mock("@/lib/labs/ocr-capability", () => ({ - resolveVisionProvider: vi.fn(), +vi.mock("@/lib/documents/provider-order", () => ({ + resolveDocumentVisionProvider: vi.fn(), })); +vi.mock("@/lib/documents/document-settings", () => ({ + documentAutoReadEnabled: vi.fn(), +})); +// Use the real egress classification — only "local" is non-egress. vi.mock("@/lib/ai/consent-guard", () => ({ - assertConsentForChain: vi.fn().mockResolvedValue(undefined), - ConsentRequiredError: class ConsentRequiredError extends Error {}, + isExternalDocumentEgress: (providerType: string) => providerType !== "local", })); vi.mock("@/lib/ai/coach/budget", () => ({ buildDateKey: vi.fn(() => "2026-07-07"), @@ -52,13 +57,10 @@ import { } from "@/lib/documents/ai-route-support"; import { upsertContentIndex } from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { localExtractText } from "@/lib/documents/local-extract"; import { decryptDocumentContent } from "@/lib/documents/store"; -import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; -import { - assertConsentForChain, - ConsentRequiredError, -} from "@/lib/ai/consent-guard"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget"; const DOC = { @@ -70,6 +72,7 @@ const DOC = { status: "STORED", }; +/** An EXTERNAL (Anthropic) document pick — egress, so toggle-gated. */ const PICK = { chain: [{ providerType: "anthropic", instance: {} }], pick: { @@ -79,6 +82,16 @@ const PICK = { }, }; +/** A LOCAL (self-hosted) document pick — never egresses, toggle-independent. */ +const LOCAL_PICK = { + chain: [{ providerType: "local", instance: {} }], + pick: { + entry: { providerType: "local", instance: {} }, + providerType: "local", + pdfSupported: false, + }, +}; + function visionOk() { vi.mocked(prepareVisionInput).mockReturnValue({ ok: true, @@ -99,11 +112,14 @@ beforeEach(() => { text: "glucose fasting creatinine values report", source: "local-pdf", } as never); + // Default: the auto-read toggle is ON (so the external-pick provider path is + // exercised); the OFF-specific tests override this explicitly. + vi.mocked(documentAutoReadEnabled).mockResolvedValue(true); }); describe("indexDocumentContent — provider-first path", () => { - it("indexes via the provider (vision) when configured + consented", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + it("indexes via an external provider (vision) when configured + toggle ON", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); visionOk(); vi.mocked(transcribeDocument).mockResolvedValue({ text: "haemoglobin 14.2 cholesterol 190", @@ -120,11 +136,30 @@ describe("indexDocumentContent — provider-first path", () => { // The local path is never touched when the provider succeeds. expect(localExtractText).not.toHaveBeenCalled(); }); + + it("runs a LOCAL vision pick even when the toggle is OFF (no egress)", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue( + LOCAL_PICK as never, + ); + vi.mocked(documentAutoReadEnabled).mockResolvedValue(false); + visionOk(); + vi.mocked(transcribeDocument).mockResolvedValue({ + text: "haemoglobin 14.2 cholesterol 190", + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: true, source: "vision", tokenCount: 5 }); + expect(transcribeDocument).toHaveBeenCalledTimes(1); + expect(upsertContentIndex).toHaveBeenCalledWith( + expect.objectContaining({ source: "vision", providerType: "local" }), + ); + expect(localExtractText).not.toHaveBeenCalled(); + }); }); describe("indexDocumentContent — local fallback path", () => { it("falls back to local when NO provider is configured", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue({ + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], pick: null, } as never); @@ -141,19 +176,20 @@ describe("indexDocumentContent — local fallback path", () => { ); }); - it("falls back to local when consent is not granted", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); - vi.mocked(assertConsentForChain).mockRejectedValueOnce( - new ConsentRequiredError("insights" as never), - ); + it("stays LOCAL for an external pick when the toggle is OFF (never egresses)", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(documentAutoReadEnabled).mockResolvedValue(false); const outcome = await indexDocumentContent("user-1", "doc-1"); expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + // No egress, no budget reservation, no provider call on the OFF path. + expect(prepareVisionInput).not.toHaveBeenCalled(); + expect(reserveBudget).not.toHaveBeenCalled(); expect(transcribeDocument).not.toHaveBeenCalled(); }); it("falls back to local when the provider cannot read the PDF", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); vi.mocked(prepareVisionInput).mockReturnValue({ ok: false, reason: "pdfNeedsAnthropic", @@ -166,7 +202,7 @@ describe("indexDocumentContent — local fallback path", () => { }); it("falls back to local when the daily budget is exhausted", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); visionOk(); vi.mocked(reserveBudget).mockResolvedValue({ allowed: false, @@ -179,7 +215,7 @@ describe("indexDocumentContent — local fallback path", () => { }); it("refunds and falls back to local when the provider call throws", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); visionOk(); vi.mocked(transcribeDocument).mockRejectedValueOnce(new Error("boom")); @@ -191,7 +227,7 @@ describe("indexDocumentContent — local fallback path", () => { }); it("reports local-empty for a scanned PDF with no usable provider", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue({ + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], pick: null, } as never); @@ -211,11 +247,11 @@ describe("indexDocumentContent — fail-closed", () => { vi.mocked(loadOwnedDocument).mockResolvedValue(null as never); const outcome = await indexDocumentContent("user-1", "missing"); expect(outcome).toEqual({ indexed: false, reason: "not-found" }); - expect(resolveVisionProvider).not.toHaveBeenCalled(); + expect(resolveDocumentVisionProvider).not.toHaveBeenCalled(); }); it("fails closed to decrypt-error when local decryption throws", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue({ + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], pick: null, } as never); diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 99f673cd9..193e3dd6b 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -2,19 +2,26 @@ * Content-index ONE stored document — the shared, AI-first decision tree behind * both the per-document auto-index-on-upload job and the corpus backfill. * - * Ordering (maintainer, 2026-07-07 — AI-first, local as fallback): - * 1. PROVIDER (primary). When a vision-capable AI provider is configured AND - * the user has consented, decrypt the stored original and run ONE provider - * transcription — rich, handles scanned PDFs + images via vision. Budget- - * reserved + reconciled exactly like the interactive index route; owner- - * scoped throughout. Codex-over-OAuth is subscription-covered, so this - * path carries no per-token API bill (see the research note). - * 2. LOCAL (fallback). When no provider is usable — none configured, consent - * not granted, the provider can't read this file (e.g. a text-layer PDF on - * a non-Anthropic account), the daily budget is reached, or the provider - * call fails — fall back to server-side text-layer extraction (`pdf-parse`, - * no egress, milliseconds). So content search works at a baseline even with - * no AI, and every text-layer PDF is searchable regardless of provider. + * Provider resolution follows the DOCUMENT order (local-first, codex last — + * `resolveDocumentVisionProvider`), NOT the cost-first app-wide chain, and the + * external egress is governed by the per-user `documentsAutoAiRead` opt-in: + * + * 1. LOCAL PROVIDER (a self-hosted vision model) — never egresses, so it runs + * whenever it is the document-order pick, toggle-independent. + * 2. EXTERNAL PROVIDER (codex / BYOK openai|anthropic / admin key) — reads a + * document off the machine, so the auto-index job only uses it when the + * operator opted into `documentsAutoAiRead`. OFF → the job is strictly + * local (never egresses on upload, even if an unrelated consent receipt + * exists); ON → the document-order external pick reads the original (rich, + * handles scanned PDFs + images). Budget-reserved + reconciled exactly like + * the interactive index route; owner-scoped throughout. + * 3. LOCAL TEXT-LAYER (fallback). When no provider is usable — none + * configured, the toggle is OFF for an external pick, the provider can't + * read this file (e.g. a text-layer PDF on a non-Anthropic account), the + * daily budget is reached, or the provider call fails — fall back to + * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * So content search works at a baseline even with no AI, and every + * text-layer PDF is searchable regardless of provider. * * Both paths write the SAME blind, encrypted index via `upsertContentIndex` * (AES-256-GCM text + opaque HMAC token tags) — nothing readable at rest. The @@ -24,10 +31,7 @@ import { Buffer } from "node:buffer"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; -import { - assertConsentForChain, - ConsentRequiredError, -} from "@/lib/ai/consent-guard"; +import { isExternalDocumentEgress } from "@/lib/ai/consent-guard"; import { buildDateKey, reconcileSpend, @@ -44,11 +48,12 @@ import { type ContentIndexSource, } from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { localExtractText } from "@/lib/documents/local-extract"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { decryptDocumentContent } from "@/lib/documents/store"; import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; -import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; /** * Outcome of one document's index attempt. A provider that is not usable (none @@ -71,32 +76,41 @@ export type IndexOutcome = */ export interface ResolvedIndexProvider { chain: ProviderChainResolved[]; - pick: Awaited>["pick"]; + pick: Awaited>["pick"]; consentOk: boolean; dailyCap: number; } /** - * Resolve the user's vision provider and pre-check consent once. A missing pick - * or an un-granted consent both yield an unusable context — the caller then - * falls straight to the local path. A NON-consent error propagates. + * Resolve the DOCUMENT-order vision provider and decide egress eligibility once. + * A local pick is always eligible (it never leaves the machine). An external + * pick is eligible ONLY when the operator opted into `documentsAutoAiRead` — so + * the auto-index job never silently egresses a freshly uploaded document unless + * that toggle is ON. A missing pick or an ineligible external pick both yield an + * unusable context, and the caller falls straight to the local text-layer path. */ export async function resolveIndexProvider( userId: string, ): Promise { - const { chain, pick } = await resolveVisionProvider(userId); + const { chain, pick } = await resolveDocumentVisionProvider(userId); let consentOk = false; if (pick) { - try { - await assertConsentForChain({ userId, chain, surface: "insights" }); + if (!isExternalDocumentEgress(pick.providerType)) { + // A local (self-hosted) vision pick never egresses — always eligible, + // toggle-independent. consentOk = true; - } catch (err) { - if (!(err instanceof ConsentRequiredError)) throw err; + } else { + // An external pick reads the document off the machine. The auto-index job + // only egresses on upload when the operator opted in; the toggle IS the + // standing consent (the document consent gate short-circuits on it). OFF + // → local-only, even if an unrelated receipt exists. + consentOk = await documentAutoReadEnabled(userId); } } - const dailyCap = pick - ? resolveDailyCap([{ providerType: pick.entry.providerType }]) - : 0; + const dailyCap = + pick && consentOk + ? resolveDailyCap([{ providerType: pick.entry.providerType }]) + : 0; return { chain, pick, consentOk, dailyCap }; } From d2451021bc6620e3a1a8d424eabbaea763911d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:49:17 +0200 Subject: [PATCH 3/8] feat(documents): rasterize PDFs so image-only-wire providers can read them Only Anthropic accepts a native PDF block; codex and other image-only-wire providers dropped it. Add a bounded, best-effort server-side rasterizer that renders a PDF's pages to JPEG images those providers read as ordinary image input, so a scanned/image-only PDF is legible on the auto-read path. - @napi-rs/canvas promoted to a direct dependency; its musl prebuilts + the pdfjs loader pinned into the standalone trace (next.config.ts) - rasterize-pdf.ts: lazy pdfjs import (keeps DOMMatrix out of the eager chunk), first 10 pages, 1600px long edge, JPEG q80, never throws - prepareVisionInput rasterizes a PDF for a non-Anthropic vision provider and keeps the native PDF block for Anthropic; the document capability reports PDFs readable via raster so the UI offers the read --- next.config.ts | 12 ++ package.json | 2 + pnpm-lock.yaml | 6 + .../api/documents/inbound/[id]/index/route.ts | 2 +- .../documents/inbound/[id]/suggest/route.ts | 2 +- .../documents/inbound/[id]/summary/route.ts | 2 +- .../documents/__tests__/rasterize-pdf.test.ts | 104 ++++++++++ src/lib/documents/ai-route-support.ts | 54 +++-- src/lib/documents/index-document.ts | 2 +- src/lib/documents/provider-order.ts | 8 +- src/lib/documents/rasterize-pdf.ts | 191 ++++++++++++++++++ .../jobs/document-content-index-backfill.ts | 2 +- 12 files changed, 366 insertions(+), 21 deletions(-) create mode 100644 src/lib/documents/__tests__/rasterize-pdf.test.ts create mode 100644 src/lib/documents/rasterize-pdf.ts diff --git a/next.config.ts b/next.config.ts index 63a17d195..fc2280969 100644 --- a/next.config.ts +++ b/next.config.ts @@ -254,6 +254,18 @@ const nextConfig: NextConfig = { // runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`. "./node_modules/.pnpm/pdf-parse@*/node_modules/pdf-parse/**", "./node_modules/.pnpm/pdfjs-dist@*/node_modules/pdfjs-dist/**", + // Document AI rasterization (`src/lib/documents/rasterize-pdf.ts`) renders + // a scanned/image-only PDF page to a JPEG via `@napi-rs/canvas` so an + // image-only-wire provider (codex/OAuth) can read it — the ambient + // auto-read path. `@napi-rs/canvas` loads a native `.node` binary that the + // Turbopack NFT tracer resolves dynamically and therefore misses, so pin + // the JS loader AND the two musl prebuilts (the Alpine runtime image is + // amd64 + arm64) into the standalone image explicitly. pnpm keeps them + // unhoisted under `.pnpm`. No cairo/pango apk is needed — the prebuilt + // `.node` is self-contained. + "./node_modules/.pnpm/@napi-rs+canvas@*/node_modules/@napi-rs/canvas/**", + "./node_modules/.pnpm/@napi-rs+canvas-linux-x64-musl@*/node_modules/@napi-rs/canvas-linux-x64-musl/**", + "./node_modules/.pnpm/@napi-rs+canvas-linux-arm64-musl@*/node_modules/@napi-rs/canvas-linux-arm64-musl/**", ], }, // v1.4.34 IW-A — silence the Turbopack NFT trace warnings emitted diff --git a/package.json b/package.json index 250ed1262..909cddd16 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@dnd-kit/utilities": "^3.2.2", "@hookform/resolvers": "^5.4.0", "@modelcontextprotocol/sdk": "^1.29.0", + "@napi-rs/canvas": "0.1.80", "@node-rs/argon2": "^2.0.2", "@parse/node-apn": "^8.1.0", "@prisma/adapter-pg": "^7.8.0", @@ -123,6 +124,7 @@ "knip": "^6.16.1", "lefthook": "^2.1.9", "pdf-parse": "^2.4.5", + "pdfjs-dist": "5.4.296", "pg": "^8.22.0", "prettier": "^3.9.1", "prettier-plugin-tailwindcss": "^0.8.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4772a0b7e..6c75a0116 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.29.0 version: 1.29.0(zod@4.4.3) + '@napi-rs/canvas': + specifier: 0.1.80 + version: 0.1.80 '@node-rs/argon2': specifier: ^2.0.2 version: 2.0.2 @@ -217,6 +220,9 @@ importers: pdf-parse: specifier: ^2.4.5 version: 2.4.5 + pdfjs-dist: + specifier: 5.4.296 + version: 5.4.296 pg: specifier: ^8.22.0 version: 8.22.0 diff --git a/src/app/api/documents/inbound/[id]/index/route.ts b/src/app/api/documents/inbound/[id]/index/route.ts index 0d318a235..665a93d05 100644 --- a/src/app/api/documents/inbound/[id]/index/route.ts +++ b/src/app/api/documents/inbound/[id]/index/route.ts @@ -184,7 +184,7 @@ async function handleVisionIndex( ); if (!rl.allowed) return rateLimited(rl); - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { if (vision.reason === "pdfNeedsAnthropic") { return apiError( diff --git a/src/app/api/documents/inbound/[id]/suggest/route.ts b/src/app/api/documents/inbound/[id]/suggest/route.ts index e0670d3f7..79a939fb2 100644 --- a/src/app/api/documents/inbound/[id]/suggest/route.ts +++ b/src/app/api/documents/inbound/[id]/suggest/route.ts @@ -233,7 +233,7 @@ async function handleVisionSuggest( ); if (!rl.allowed) return rateLimited(rl); - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { if (vision.reason === "pdfNeedsAnthropic") { return apiError( diff --git a/src/app/api/documents/inbound/[id]/summary/route.ts b/src/app/api/documents/inbound/[id]/summary/route.ts index 6d09219cb..dfce5bff3 100644 --- a/src/app/api/documents/inbound/[id]/summary/route.ts +++ b/src/app/api/documents/inbound/[id]/summary/route.ts @@ -258,7 +258,7 @@ async function handleVisionSummary( ); if (!rl.allowed) return rateLimited(rl); - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { if (vision.reason === "pdfNeedsAnthropic") { return apiError( diff --git a/src/lib/documents/__tests__/rasterize-pdf.test.ts b/src/lib/documents/__tests__/rasterize-pdf.test.ts new file mode 100644 index 000000000..547661932 --- /dev/null +++ b/src/lib/documents/__tests__/rasterize-pdf.test.ts @@ -0,0 +1,104 @@ +import { Buffer } from "node:buffer"; + +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() })); + +import { RASTER_MAX_PAGES, rasterizePdf } from "../rasterize-pdf"; + +/** + * Rasterization renders a PDF's pages to bounded JPEG images. Pins: a text-layer + * PDF and a synthetic image-only (graphics-only, no text layer) PDF both render + * to JPEG page images; the page cap is honoured; malformed input degrades to + * `{ ok: false }` and never throws. + * + * The fixtures are built inline (a tiny hand-assembled PDF with a correct xref) + * so the test carries no binary blobs. + */ +function buildPdf(pageCount: number, withText: boolean): Buffer { + const pageNums: number[] = []; + const perPage: { pageNum: number; contentNum: number; content: string }[] = + []; + let n = 4; + for (let p = 0; p < pageCount; p++) { + const pageNum = n++; + const contentNum = n++; + pageNums.push(pageNum); + const content = withText + ? `BT /F1 24 Tf 40 120 Td (Page ${p + 1} lab report glucose creatinine) Tj ET` + : `0 0 1 rg 40 40 120 120 re f`; + perPage.push({ pageNum, contentNum, content }); + } + + const bodies: string[] = []; + bodies[1] = `<< /Type /Catalog /Pages 2 0 R >>`; + bodies[2] = `<< /Type /Pages /Kids [${pageNums + .map((x) => `${x} 0 R`) + .join(" ")}] /Count ${pageCount} >>`; + bodies[3] = `<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>`; + for (const { pageNum, contentNum, content } of perPage) { + bodies[pageNum] = + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentNum} 0 R >>`; + bodies[contentNum] = + `<< /Length ${Buffer.byteLength(content, "latin1")} >>\nstream\n${content}\nendstream`; + } + + const total = n - 1; + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + for (let i = 1; i <= total; i++) { + offsets[i] = Buffer.byteLength(out, "latin1"); + out += `${i} 0 obj\n${bodies[i]}\nendobj\n`; + } + const xrefOffset = Buffer.byteLength(out, "latin1"); + out += `xref\n0 ${total + 1}\n0000000000 65535 f \n`; + for (let i = 1; i <= total; i++) { + out += `${String(offsets[i]).padStart(10, "0")} 00000 n \n`; + } + out += `trailer\n<< /Size ${total + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF`; + return Buffer.from(out, "latin1"); +} + +/** JPEG magic bytes. */ +function isJpeg(base64: string): boolean { + const head = Buffer.from(base64, "base64").subarray(0, 2).toString("hex"); + return head === "ffd8"; +} + +describe("rasterizePdf", () => { + it("renders a text-layer PDF's pages to JPEG images", async () => { + const result = await rasterizePdf(buildPdf(3, true)); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(3); + for (const img of result.images) { + expect(img.mediaType).toBe("image/jpeg"); + expect(isJpeg(img.dataBase64)).toBe(true); + } + }); + + it("renders a synthetic image-only (no text layer) PDF", async () => { + const result = await rasterizePdf(buildPdf(1, false)); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(1); + expect(isJpeg(result.images[0]!.dataBase64)).toBe(true); + }); + + it("caps the number of rendered pages at RASTER_MAX_PAGES", async () => { + const result = await rasterizePdf(buildPdf(RASTER_MAX_PAGES + 4, true)); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.images).toHaveLength(RASTER_MAX_PAGES); + }); + + it("returns { ok: false } and never throws on malformed input", async () => { + await expect( + rasterizePdf(Buffer.from("this is not a pdf")), + ).resolves.toEqual({ ok: false }); + }); + + it("returns { ok: false } on an empty buffer", async () => { + await expect(rasterizePdf(Buffer.alloc(0))).resolves.toEqual({ ok: false }); + }); +}); diff --git a/src/lib/documents/ai-route-support.ts b/src/lib/documents/ai-route-support.ts index 64cbb6221..a974ad5ab 100644 --- a/src/lib/documents/ai-route-support.ts +++ b/src/lib/documents/ai-route-support.ts @@ -10,6 +10,7 @@ import { Buffer } from "node:buffer"; import { prisma } from "@/lib/db"; +import { rasterizePdf } from "@/lib/documents/rasterize-pdf"; import { decryptDocumentContent } from "@/lib/documents/store"; import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; @@ -75,13 +76,24 @@ export type VisionInput = /** * Decrypt the stored original, re-derive its MIME from the bytes (never trust * the stored label for a provider call), and split it into the images/documents - * arrays a vision call takes. Fails closed on a decrypt error and rejects a PDF - * on a provider that cannot read PDFs natively. + * arrays a vision call takes. Fails closed on a decrypt error. + * + * PDF handling by provider capability: + * - `pdfSupported` (Anthropic) → native `document` block, highest fidelity. + * - otherwise (codex / any image-only-wire provider) → rasterize the pages to + * JPEG `input_image` parts so the provider can still read the PDF. Egress is + * already authorised upstream (the consent gate / the `documentsAutoAiRead` + * toggle) before this is reached. If rasterization fails (malformed / + * encrypted / unrenderable), fall back to the shipped `pdfNeedsAnthropic` + * outcome so the job degrades to the local text-layer path and an + * interactive route surfaces the same actionable error. + * + * Images (jpg/png/webp) flow straight to `input_image`, unchanged. */ -export function prepareVisionInput( +export async function prepareVisionInput( document: LoadedDocument, pdfSupported: boolean, -): VisionInput { +): Promise { let buffer: Buffer; try { buffer = decryptDocumentContent( @@ -94,17 +106,31 @@ export function prepareVisionInput( const mime = detectOcrMimeType(buffer); if (!mime) return { ok: false, reason: "fileType" }; - if (mime === "application/pdf" && !pdfSupported) { + + if (mime === "application/pdf") { + if (pdfSupported) { + return { + ok: true, + images: [], + documents: [ + { + mediaType: "application/pdf", + dataBase64: buffer.toString("base64"), + }, + ], + }; + } + // Non-Anthropic vision provider — render the PDF pages to images it can read. + const raster = await rasterizePdf(buffer); + if (raster.ok) { + return { ok: true, images: raster.images, documents: [] }; + } return { ok: false, reason: "pdfNeedsAnthropic" }; } - const dataBase64 = buffer.toString("base64"); - if (mime === "application/pdf") { - return { - ok: true, - images: [], - documents: [{ mediaType: "application/pdf", dataBase64 }], - }; - } - return { ok: true, images: [{ mediaType: mime, dataBase64 }], documents: [] }; + return { + ok: true, + images: [{ mediaType: mime, dataBase64: buffer.toString("base64") }], + documents: [], + }; } diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 193e3dd6b..319dae074 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -128,7 +128,7 @@ async function tryProviderIndex( const { pick } = provider; if (!pick || !provider.consentOk) return null; - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { // A decrypt failure is terminal (local would fail the same way); anything // else (fileType / pdfNeedsAnthropic) falls through to the local path — a diff --git a/src/lib/documents/provider-order.ts b/src/lib/documents/provider-order.ts index e27678d50..5107dcef3 100644 --- a/src/lib/documents/provider-order.ts +++ b/src/lib/documents/provider-order.ts @@ -25,6 +25,7 @@ * external egress from happening without an active receipt. */ import { isExternalDocumentEgress } from "@/lib/ai/consent-guard"; +import { RASTERIZATION_AVAILABLE } from "@/lib/documents/rasterize-pdf"; import { resolveTextProvider, resolveVisionProvider, @@ -119,13 +120,16 @@ export async function resolveDocumentAiCapability( await resolveDocumentVisionProvider(userId); // A vision-capable provider is available — the read runs directly over the - // stored original. Egress follows the picked provider. + // stored original. Egress follows the picked provider. PDFs are readable + // whenever the picked provider natively supports them (Anthropic) OR the + // server-side rasterizer is available (every other vision provider reads a + // PDF via rasterized page images), so the UI offers a PDF read on codex too. if (pick) { return { available: true, mode: "vision", reason: null, - pdfSupported: pick.pdfSupported, + pdfSupported: pick.pdfSupported || RASTERIZATION_AVAILABLE, egress: documentEgressClass(pick.providerType), }; } diff --git a/src/lib/documents/rasterize-pdf.ts b/src/lib/documents/rasterize-pdf.ts new file mode 100644 index 000000000..b880441c9 --- /dev/null +++ b/src/lib/documents/rasterize-pdf.ts @@ -0,0 +1,191 @@ +/** + * Server-side PDF rasterization for the document vault. + * + * The OAuth/subscription wire (codex) — and every other image-only-wire provider + * — cannot receive a native PDF `document` block; only Anthropic accepts one. + * This module renders a PDF's pages to raster JPEG images so those providers can + * still READ a scanned/image-only PDF: each page image goes out as an ordinary + * `input_image` part the codex client already handles, with no client-interface + * change. Only invoked (on the auto-read path) when the toggle authorises the + * external egress and the picked provider is NOT Anthropic (Anthropic keeps its + * higher-fidelity native PDF block; a text-layer PDF is still read locally for + * free before we ever rasterize). + * + * pdfjs is loaded LAZILY via a runtime `import()` — its module top-level + * references the browser-only `DOMMatrix` global, so a STATIC import would make + * the Turbopack server chunk evaluate that reference at instantiation and throw + * `DOMMatrix is not defined`, taking down every route/worker sharing the chunk. + * The lazy import (the exact pattern `local-extract.ts` documents) keeps pdfjs + + * `@napi-rs/canvas` (its Node canvas backend) out of the eager chunk. Rendering + * is pure local compute — no network, nothing written to disk. + * + * Bounds (denial-of-service + vision token cost): render at most the first + * `RASTER_MAX_PAGES` pages, each capped to `RASTER_TARGET_LONG_EDGE` on its long + * edge (never upscaled), JPEG at `RASTER_JPEG_QUALITY`. A discharge letter / lab + * is 1-4 pages; the cap keeps a pathological 200-page PDF from draining the + * subscription allowance in one job. + * + * NEVER throws: a malformed / encrypted / unrenderable PDF, a missing native + * binary, or any render error resolves to `{ ok: false }`, and the caller falls + * back to the local text-layer path or leaves the document un-indexed. This + * preserves the "a bad document never aborts an upload/batch" contract. + */ +import { Buffer } from "node:buffer"; + +import { annotate } from "@/lib/logging/context"; + +/** + * Whether the PDF rasterizer is available in this build. The native + * `@napi-rs/canvas` binary is compiled in and traced into the standalone image + * (see `next.config.ts`), so rasterization is a standing capability — every + * vision provider can read a PDF (Anthropic natively, all others via raster). + * The document capability DTO reads this so the UI offers a PDF read for a + * non-Anthropic provider; a runtime render failure still degrades gracefully + * (`prepareVisionInput` → `pdfNeedsAnthropic` → local text or un-indexed). + */ +export const RASTERIZATION_AVAILABLE = true; + +/** Render at most the first N pages — the token-cost / DoS governor. */ +export const RASTER_MAX_PAGES = 10; + +/** + * Longest-edge pixel cap per page. Tighter than the client upload downscale + * ceiling because these feed a token-metered vision model; ~1600px is legible + * for a document read without blowing up the tile count. + */ +export const RASTER_TARGET_LONG_EDGE = 1600; + +/** JPEG quality (0-100). ~80 keeps text legible at a fraction of PNG's bytes. */ +export const RASTER_JPEG_QUALITY = 80; + +/** One rasterized page, shaped as a vision `input_image` part. */ +export interface RasterImage { + mediaType: "image/jpeg"; + dataBase64: string; +} + +/** Best-effort rasterization outcome — never an exception. */ +export type RasterResult = { ok: true; images: RasterImage[] } | { ok: false }; + +// Minimal structural types for the slice of the pdfjs API this module uses. A +// type-only shape (erased at build) so we never eagerly evaluate the pdfjs +// module for its types; the real module is pulled at runtime via `import()`. +interface PdfViewport { + width: number; + height: number; +} +interface PdfCanvas { + width: number; + height: number; + toBuffer(mime: "image/jpeg", quality?: number): Buffer; +} +interface PdfCanvasAndContext { + canvas: PdfCanvas; + context: unknown; +} +interface PdfCanvasFactory { + create(width: number, height: number): PdfCanvasAndContext; + destroy?(target: PdfCanvasAndContext): void; +} +interface PdfPage { + getViewport(opts: { scale: number }): PdfViewport; + render(opts: { + canvasContext: unknown; + viewport: PdfViewport; + canvas: unknown; + }): { promise: Promise }; + cleanup(): void; +} +interface PdfDocument { + numPages: number; + canvasFactory: PdfCanvasFactory; + getPage(pageNumber: number): Promise; + destroy(): Promise; +} +interface PdfjsModule { + getDocument(opts: { + data: Uint8Array; + verbosity?: number; + isEvalSupported?: boolean; + useSystemFonts?: boolean; + }): { promise: Promise }; +} + +/** + * Render the first `RASTER_MAX_PAGES` pages of a PDF to bounded JPEG images. + * Returns `{ ok: false }` on any failure (malformed PDF, render throw, missing + * binary, or zero renderable pages) — the caller degrades to local text or + * un-indexed. Never throws. + */ +export async function rasterizePdf(buffer: Buffer): Promise { + let doc: PdfDocument | null = null; + try { + // Lazy import: keeps pdfjs (DOMMatrix) + @napi-rs/canvas out of the eager + // server chunk. Resolved at first rasterization, then module-cached. + const pdfjs = + (await import("pdfjs-dist/legacy/build/pdf.mjs")) as unknown as PdfjsModule; + + const task = pdfjs.getDocument({ + data: new Uint8Array(buffer), + verbosity: 0, // VerbosityLevel.ERRORS — no console spam on odd PDFs. + isEvalSupported: false, // never eval font programs (defence in depth). + useSystemFonts: true, + }); + doc = await task.promise; + + const pageCount = Math.min(doc.numPages, RASTER_MAX_PAGES); + const images: RasterImage[] = []; + + for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { + const page = await doc.getPage(pageNumber); + try { + const base = page.getViewport({ scale: 1 }); + const longEdge = Math.max(base.width, base.height); + // Cap the long edge, never upscale (scale ≤ 1). + const scale = + longEdge > 0 ? Math.min(1, RASTER_TARGET_LONG_EDGE / longEdge) : 1; + const viewport = page.getViewport({ scale }); + + const factory = doc.canvasFactory; + const cc = factory.create( + Math.ceil(viewport.width), + Math.ceil(viewport.height), + ); + await page.render({ + canvasContext: cc.context, + viewport, + canvas: cc.canvas, + }).promise; + const jpeg = cc.canvas.toBuffer("image/jpeg", RASTER_JPEG_QUALITY); + images.push({ + mediaType: "image/jpeg", + dataBase64: jpeg.toString("base64"), + }); + factory.destroy?.(cc); + } finally { + page.cleanup(); + } + } + + if (images.length === 0) return { ok: false }; + annotate({ + action: { name: "documents.rasterize.ok" }, + meta: { pages: images.length, cappedFrom: doc.numPages }, + }); + return { ok: true, images }; + } catch (err) { + annotate({ + action: { name: "documents.rasterize.failed" }, + meta: { reason: err instanceof Error ? err.name : "unknown" }, + }); + return { ok: false }; + } finally { + if (doc) { + try { + await doc.destroy(); + } catch { + // Best-effort teardown; a destroy failure must not mask the result. + } + } + } +} diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index a9e78f413..fa5f17f05 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -137,7 +137,7 @@ export async function runContentIndexBackfillForUser( await reconcileSpend(userId, reservation.reserved, 0, dateKey); continue; } - const vision = prepareVisionInput(document, pick.pdfSupported); + const vision = await prepareVisionInput(document, pick.pdfSupported); if (!vision.ok) { // Not vision-indexable (should be filtered by MIME, but guard) — refund // and move the cursor past it so the walk keeps converging. From b3ae35b0552a3a309b02fafa8b84ec661e2d46b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:55:01 +0200 Subject: [PATCH 4/8] feat(documents): add the AI-settings toggle to read documents automatically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a single switch in AI settings — off by default — that turns on ambient AI reading of uploaded vault documents. Turning it on reveals a once-shown, vendor-blind honesty confirm (subscription providers may use the content to improve their models; a self-hosted local model never egresses) that must be acknowledged before the setting is written. Copy is translated across all six locales. --- messages/de.json | 9 +- messages/en.json | 9 +- messages/es.json | 9 +- messages/fr.json | 9 +- messages/it.json | 9 +- messages/pl.json | 9 +- .../settings/ai/ai-insights-card.tsx | 4 + src/components/settings/ai/auto-read-card.tsx | 144 ++++++++++++++++++ src/lib/query-keys/documents.ts | 6 + 9 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 src/components/settings/ai/auto-read-card.tsx diff --git a/messages/de.json b/messages/de.json index 4955e2990..65d74b3dc 100644 --- a/messages/de.json +++ b/messages/de.json @@ -4731,7 +4731,14 @@ }, "ocrModelPlaceholder": "z. B. gpt-4o", "ocrVisionHint": "Dieses Modell kann möglicherweise keine Bilder lesen. Wähle für den Dokumentenscan ein Modell mit Bildverständnis.", - "ocrKeyLabel": "API-Key" + "ocrKeyLabel": "API-Key", + "autoRead": { + "title": "Dokumente automatisch mit KI lesen", + "subLabel": "Neu hochgeladene Dokumente werden von deinem konfigurierten KI-Anbieter gelesen und indiziert, ohne dass du jedes Dokument einzeln bestätigen musst.", + "honesty": "Wenn aktiviert, wird jedes hochgeladene Dokument an deinen konfigurierten KI-Anbieter gesendet, um gelesen, beschrieben und für die Suche indiziert zu werden – ohne Bestätigung pro Dokument. Das geschieht auf dem Server, und der Inhalt des Dokuments verlässt dabei diese Maschine in Richtung dieses Anbieters. Wenn dein konfigurierter Anbieter eine Abo-Verbindung ist (ein angemeldetes KI-Konto statt eines API-Schlüssels), gelten dessen eigene Datenschutzeinstellungen – Abo-Anbieter dürfen den Inhalt zur Verbesserung ihrer Modelle verwenden, und kein Auftragsverarbeitungsvertrag deckt das ab. Dokumente, die von einem selbst gehosteten lokalen Modell gelesen werden, verlassen deinen Server nie. Standardmäßig aus; aktiviere es nur, wenn du diesen Kompromiss für die von dir hochgeladenen Dokumente akzeptierst.", + "confirm": "Ich verstehe – automatisches KI-Lesen aktivieren", + "cancel": "Abbrechen" + } }, "dashboard": { "units": { diff --git a/messages/en.json b/messages/en.json index 6902df151..c7fac3b9b 100644 --- a/messages/en.json +++ b/messages/en.json @@ -4731,7 +4731,14 @@ }, "ocrModelPlaceholder": "e.g. gpt-4o", "ocrVisionHint": "This model may not be able to read images. Choose a vision-capable model for document scanning.", - "ocrKeyLabel": "API key" + "ocrKeyLabel": "API key", + "autoRead": { + "title": "Read documents automatically with AI", + "subLabel": "Newly uploaded documents are read and indexed by your configured AI provider without a per-document tap.", + "honesty": "When on, each document you upload is sent to your configured AI provider to be read, described, and indexed for search — with no per-document confirmation. This happens on the server and the document's contents leave this machine to that provider. If your configured provider is a subscription connection (a signed-in AI account rather than an API key), that provider's own data settings apply — subscription providers may use the content to improve their models, and no data-processing agreement covers it. Documents read by a self-hosted local model never leave your server. Off by default; turn it on only if you accept this trade for the documents you upload.", + "confirm": "I understand — turn on automatic AI reading", + "cancel": "Cancel" + } }, "dashboard": { "units": { diff --git a/messages/es.json b/messages/es.json index 87f2950db..17567f082 100644 --- a/messages/es.json +++ b/messages/es.json @@ -4731,7 +4731,14 @@ }, "ocrModelPlaceholder": "p. ej. gpt-4o", "ocrVisionHint": "Puede que este modelo no pueda leer imágenes. Elige un modelo con capacidad de visión para el escaneo de documentos.", - "ocrKeyLabel": "Clave de API" + "ocrKeyLabel": "Clave de API", + "autoRead": { + "title": "Leer documentos automáticamente con IA", + "subLabel": "Los documentos recién subidos son leídos e indexados por tu proveedor de IA configurado, sin confirmar cada documento.", + "honesty": "Cuando está activado, cada documento que subes se envía a tu proveedor de IA configurado para leerlo, describirlo e indexarlo para la búsqueda, sin confirmación por documento. Esto ocurre en el servidor y el contenido del documento sale de esta máquina hacia ese proveedor. Si tu proveedor configurado es una conexión de suscripción (una cuenta de IA con sesión iniciada en lugar de una clave de API), se aplican sus propios ajustes de datos: los proveedores de suscripción pueden usar el contenido para mejorar sus modelos, y ningún acuerdo de tratamiento de datos lo cubre. Los documentos leídos por un modelo local autoalojado nunca salen de tu servidor. Desactivado de forma predeterminada; actívalo solo si aceptas este compromiso para los documentos que subes.", + "confirm": "Lo entiendo: activar la lectura automática con IA", + "cancel": "Cancelar" + } }, "dashboard": { "units": { diff --git a/messages/fr.json b/messages/fr.json index 5eaf31839..b15587733 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -4731,7 +4731,14 @@ }, "ocrModelPlaceholder": "p. ex. gpt-4o", "ocrVisionHint": "Ce modèle ne peut peut-être pas lire les images. Choisissez un modèle capable de vision pour la numérisation de documents.", - "ocrKeyLabel": "Clé API" + "ocrKeyLabel": "Clé API", + "autoRead": { + "title": "Lire les documents automatiquement avec l'IA", + "subLabel": "Les documents nouvellement téléversés sont lus et indexés par votre fournisseur d'IA configuré, sans validation document par document.", + "honesty": "Lorsque cette option est activée, chaque document que vous téléversez est envoyé à votre fournisseur d'IA configuré pour être lu, décrit et indexé pour la recherche, sans confirmation par document. Cela se produit sur le serveur et le contenu du document quitte cette machine vers ce fournisseur. Si votre fournisseur configuré est une connexion par abonnement (un compte d'IA connecté plutôt qu'une clé API), ses propres paramètres de données s'appliquent : les fournisseurs par abonnement peuvent utiliser le contenu pour améliorer leurs modèles, et aucun accord de traitement des données ne le couvre. Les documents lus par un modèle local auto-hébergé ne quittent jamais votre serveur. Désactivé par défaut ; activez-le uniquement si vous acceptez ce compromis pour les documents que vous téléversez.", + "confirm": "Je comprends — activer la lecture automatique par l'IA", + "cancel": "Annuler" + } }, "dashboard": { "units": { diff --git a/messages/it.json b/messages/it.json index 8f2cd55ca..71a0a2c87 100644 --- a/messages/it.json +++ b/messages/it.json @@ -4731,7 +4731,14 @@ }, "ocrModelPlaceholder": "es. gpt-4o", "ocrVisionHint": "Questo modello potrebbe non essere in grado di leggere le immagini. Scegli un modello con capacità di visione per la scansione dei documenti.", - "ocrKeyLabel": "Chiave API" + "ocrKeyLabel": "Chiave API", + "autoRead": { + "title": "Leggi i documenti automaticamente con l'IA", + "subLabel": "I documenti appena caricati vengono letti e indicizzati dal provider di IA configurato, senza confermare ogni documento.", + "honesty": "Quando è attivo, ogni documento che carichi viene inviato al provider di IA configurato per essere letto, descritto e indicizzato per la ricerca, senza conferma per singolo documento. Questo avviene sul server e il contenuto del documento lascia questa macchina verso quel provider. Se il provider configurato è una connessione in abbonamento (un account IA con accesso effettuato anziché una chiave API), si applicano le sue impostazioni sui dati: i provider in abbonamento possono usare il contenuto per migliorare i propri modelli e nessun accordo sul trattamento dei dati lo copre. I documenti letti da un modello locale self-hosted non lasciano mai il tuo server. Disattivato per impostazione predefinita; attivalo solo se accetti questo compromesso per i documenti che carichi.", + "confirm": "Ho capito: attiva la lettura automatica con l'IA", + "cancel": "Annulla" + } }, "dashboard": { "units": { diff --git a/messages/pl.json b/messages/pl.json index 5feb87735..ad032cec2 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -4731,7 +4731,14 @@ }, "ocrModelPlaceholder": "np. gpt-4o", "ocrVisionHint": "Ten model może nie odczytywać obrazów. Wybierz model obsługujący obraz do skanowania dokumentów.", - "ocrKeyLabel": "Klucz API" + "ocrKeyLabel": "Klucz API", + "autoRead": { + "title": "Automatyczne czytanie dokumentów przez AI", + "subLabel": "Nowo przesłane dokumenty są czytane i indeksowane przez skonfigurowanego dostawcę AI bez potwierdzania każdego dokumentu.", + "honesty": "Po włączeniu każdy przesłany dokument jest wysyłany do skonfigurowanego dostawcy AI, aby go odczytać, opisać i zaindeksować do wyszukiwania — bez potwierdzania każdego dokumentu. Dzieje się to na serwerze, a treść dokumentu opuszcza to urządzenie i trafia do tego dostawcy. Jeśli skonfigurowany dostawca to połączenie subskrypcyjne (zalogowane konto AI zamiast klucza API), obowiązują jego własne ustawienia danych — dostawcy subskrypcyjni mogą wykorzystywać treść do ulepszania swoich modeli i nie obejmuje tego żadna umowa powierzenia przetwarzania danych. Dokumenty odczytywane przez samodzielnie hostowany model lokalny nigdy nie opuszczają Twojego serwera. Domyślnie wyłączone; włącz je tylko wtedy, gdy akceptujesz ten kompromis dla przesyłanych dokumentów.", + "confirm": "Rozumiem — włącz automatyczne czytanie przez AI", + "cancel": "Anuluj" + } }, "dashboard": { "units": { diff --git a/src/components/settings/ai/ai-insights-card.tsx b/src/components/settings/ai/ai-insights-card.tsx index e22a1d6b2..89821d364 100644 --- a/src/components/settings/ai/ai-insights-card.tsx +++ b/src/components/settings/ai/ai-insights-card.tsx @@ -31,6 +31,7 @@ import { queryKeys } from "@/lib/query-keys"; import { AdminOpenAIProviderForm } from "./admin-openai-provider-form"; import { AnthropicProviderForm } from "./anthropic-provider-form"; +import { AutoReadCard } from "./auto-read-card"; import { CodexProviderForm } from "./codex-provider-form"; import { DocumentScanCard } from "./document-scan-card"; import { FallbackChainCard } from "./fallback-chain-card"; @@ -181,6 +182,9 @@ export function AiInsightsCard({ {/* v1.22 (#90) — dedicated document-scan (Lab-OCR) provider. */} + {/* Read uploaded vault documents automatically with AI (opt-in). */} + + ({ + queryKey: queryKeys.documentsAutoAiRead(), + queryFn: () => apiGet("/api/auth/me/documents-auto-ai-read"), + staleTime: 60_000, + }); + + const enabled = data?.documentsAutoAiRead ?? false; + + const save = useMutation({ + mutationFn: (next: boolean) => + apiPatch("/api/auth/me/documents-auto-ai-read", { + documentsAutoAiRead: next, + }), + onSuccess: (result) => { + queryClient.setQueryData(queryKeys.documentsAutoAiRead(), result); + // Flipping the toggle changes whether an ambient/per-document read + // egresses without a per-document consent step — refresh the capability + // probe the vault UI reads. + queryClient.invalidateQueries({ + queryKey: queryKeys.inboundDocumentAiCapability(), + }); + setPendingEnable(false); + }, + }); + + function onSwitch(next: boolean) { + if (next) { + // Off → on: reveal the honesty confirm; do NOT write yet. + if (!enabled) setPendingEnable(true); + return; + } + // On → off (or cancel a pending enable): write immediately, hide confirm. + setPendingEnable(false); + if (enabled) save.mutate(false); + } + + const busy = isLoading || save.isPending; + const checked = enabled || pendingEnable; + + return ( +
+
+
+

+ {t("settings.ai.autoRead.title")} +

+

+ {t("settings.ai.autoRead.subLabel")} +

+
+ +
+ + {pendingEnable ? ( +
+
+ +

{t("settings.ai.autoRead.honesty")}

+
+
+ + +
+
+ ) : null} + + {save.isError ? ( +

+ {t("settings.ai.errorGeneric")} +

+ ) : null} +
+ ); +} diff --git a/src/lib/query-keys/documents.ts b/src/lib/query-keys/documents.ts index b69fb6660..40df66e96 100644 --- a/src/lib/query-keys/documents.ts +++ b/src/lib/query-keys/documents.ts @@ -76,4 +76,10 @@ export const documentKeys = { */ inboundDocumentChat: (documentId: string) => ["documents", "inbound", documentId, "chat"] as const, + /** + * The per-user "read documents automatically with AI" opt-in + * (`GET/PATCH /api/auth/me/documents-auto-ai-read`). Drives the AI-settings + * toggle; a flip invalidates the document AI capability probe. + */ + documentsAutoAiRead: () => ["documents", "auto-ai-read"] as const, }; From 4f145eadd7db30b68ccbac06146c450ca6dafa9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:58:15 +0200 Subject: [PATCH 5/8] docs(documents): document the automatic AI reading opt-in; add e2e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ai-providers.md: an off-by-default 'Automatic document reading' section (what ON sends where, the subscription-training / no-BAA honesty, local = no egress, regulated users → BYOK no-train API or local) and a vault local-first caveat - e2e: the settings honesty-confirm flow (switch → confirm once → write) and the vault per-egress-notice / explicit-read-action contract, with the document capability endpoint mocked on every path --- docs/integrations/ai-providers.md | 40 ++++++ e2e/documents-auto-read.spec.ts | 209 ++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 e2e/documents-auto-read.spec.ts diff --git a/docs/integrations/ai-providers.md b/docs/integrations/ai-providers.md index 2cdb85c7b..bda355c87 100644 --- a/docs/integrations/ai-providers.md +++ b/docs/integrations/ai-providers.md @@ -198,6 +198,46 @@ The Coach drawer surfaces a "Local" badge when generation resolves to the local provider, so users who pinned a local endpoint can see at a glance that their context stayed on the host. +The document vault is local-first by default: an uploaded document is +only read by an external AI provider when you tap a per-document AI +action, and every text-layer PDF is indexed for search locally with no +egress at all — **unless you opt into automatic AI reading in AI +settings** (next section). + +## Automatic document reading (opt-in) + +**Off by default.** With the switch off, nothing about the vault +changes: a stored document is read externally only when you explicitly +tap "Read with AI" on it, and that tap still needs an active AI-consent +receipt for any provider that leaves the machine. Text-layer PDFs stay +searchable through local extraction with no egress. + +Turn **"Read documents automatically with AI"** on (AI settings) and +each document you upload is read, described, and indexed by your +configured AI provider with no per-document confirmation — the +"upload and it just gets read" flow. What that means, plainly: + +- **The document's contents leave this machine** to your configured + provider. The read happens on the server, right after upload. +- **A subscription connection** — a signed-in AI account rather than an + API key — applies its own consumer data settings. **Subscription + providers may use the content to improve their models, and no + data-processing agreement covers it.** Turning the toggle on is the + standing consent for that trade; it is recorded in the consent audit + trail. +- **A self-hosted local model never egresses.** Documents read by a + local provider stay on your server whether the toggle is on or off. +- **Scanned / image-only PDFs** are rendered to page images server-side + so a provider that only accepts images (a subscription connection) can + read them too; a native-PDF provider reads the PDF directly. Only the + first pages of a long document are sent, to bound cost. + +Regulated or privacy-sensitive deployments should either leave this off +and read documents per-document on demand, or configure a **BYOK +no-training API key** or a **local model** as the provider — both avoid +the subscription-training trade. The setting is per-user: in a +multi-user deployment each account opts in for its own documents. + ## What the Coach is — and isn't The Coach is an informational assistant that works from your own diff --git a/e2e/documents-auto-read.spec.ts b/e2e/documents-auto-read.spec.ts new file mode 100644 index 000000000..a5f03705b --- /dev/null +++ b/e2e/documents-auto-read.spec.ts @@ -0,0 +1,209 @@ +/** + * Automatic AI document reading — the per-user opt-in acceptance suite. + * + * Two surfaces: + * (1) AI settings — the single switch reveals a once-shown honesty confirm + * BEFORE writing the setting (off → on), and the confirm is what commits + * `documentsAutoAiRead: true`. Turning it off is immediate. + * (2) The vault detail sheet — the per-egress "leaves your machine" notice is + * shown once and the per-document "Read with AI" action is the explicit + * path; reading through it succeeds with no per-document consent prompt. + * + * Everything is `page.route`-mocked so the suite mutates no shared demo state. + * The document-scoped capability endpoint is mocked on every path — an unmocked + * / changed capability endpoint is the shipped regression that breaks the vault + * AI e2e, so it is pinned here too. Assertions target stable data-slots, never + * viewport-dependent text. + */ +import { expect, test, type Page } from "@playwright/test"; + +import { STORAGE_STATE_PATH } from "./setup/global-setup"; +import { + ensureVaultFixture, + ensureVaultAiFixture, + AI_PROBE_DOC_ID, +} from "./setup/vault-fixture"; + +async function fulfilJson( + route: Parameters[1]>[0], + body: unknown, +) { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(body), + }); +} + +/** Mock the document capability probe so the vault AI section renders. */ +async function mockDocumentCapability( + page: Page, + opts: { egress: "local" | "external"; pdfSupported?: boolean }, +): Promise { + await page.route("**/api/documents/inbound/capability", (route) => + fulfilJson(route, { + data: { + available: true, + mode: "vision", + reason: null, + pdfSupported: opts.pdfSupported ?? true, + egress: opts.egress, + }, + error: null, + }), + ); +} + +test.describe("automatic AI reading — settings toggle", () => { + test.use({ storageState: STORAGE_STATE_PATH }); + + test.beforeEach(({}, testInfo) => { + // The confirm logic is viewport-independent; run the settings flow once on + // desktop to keep the heavy AI-settings page off the mobile matrix. + test.skip( + testInfo.project.name !== "chromium-desktop", + "settings toggle flow runs on desktop only", + ); + test.slow(); + }); + + test("turning it on reveals a once-shown honesty confirm before writing", async ({ + page, + }) => { + // The toggle starts OFF; capture the write so we can prove it only fires on + // confirm (never on the raw switch flip). + let patchBody: unknown = null; + await page.route("**/api/auth/me/documents-auto-ai-read", (route) => { + if (route.request().method() === "PATCH") { + patchBody = JSON.parse(route.request().postData() ?? "{}"); + return fulfilJson(route, { + data: { documentsAutoAiRead: true }, + error: null, + }); + } + return fulfilJson(route, { + data: { documentsAutoAiRead: false }, + error: null, + }); + }); + + await page.goto("/settings/ai"); + const card = page.locator('[data-slot="documents-auto-read-card"]'); + await expect(card).toBeVisible(); + + // Flip the switch ON — the honesty confirm appears and NOTHING is written. + await card.getByTestId("documents-auto-read-enable").click(); + const confirm = card.locator('[data-slot="documents-auto-read-confirm"]'); + await expect(confirm).toBeVisible(); + expect(patchBody).toBeNull(); + + // Acknowledge — now (and only now) the write fires with the true flag. + await confirm + .locator('[data-slot="documents-auto-read-confirm-cta"]') + .click(); + await expect.poll(() => patchBody).toEqual({ documentsAutoAiRead: true }); + await expect(confirm).not.toBeVisible(); + }); + + test("cancelling the confirm writes nothing", async ({ page }) => { + let patchCalls = 0; + await page.route("**/api/auth/me/documents-auto-ai-read", (route) => { + if (route.request().method() === "PATCH") { + patchCalls += 1; + return fulfilJson(route, { + data: { documentsAutoAiRead: true }, + error: null, + }); + } + return fulfilJson(route, { + data: { documentsAutoAiRead: false }, + error: null, + }); + }); + + await page.goto("/settings/ai"); + const card = page.locator('[data-slot="documents-auto-read-card"]'); + await expect(card).toBeVisible(); + await card.getByTestId("documents-auto-read-enable").click(); + const confirm = card.locator('[data-slot="documents-auto-read-confirm"]'); + await expect(confirm).toBeVisible(); + await confirm.getByRole("button", { name: /cancel/i }).click(); + await expect(confirm).not.toBeVisible(); + await page.waitForTimeout(300); + expect(patchCalls).toBe(0); + }); +}); + +test.describe("automatic AI reading — vault per-document contract", () => { + test.use({ storageState: STORAGE_STATE_PATH }); + + test.beforeAll(async () => { + await ensureVaultFixture(); + await ensureVaultAiFixture(); + }); + + test.beforeEach(() => test.slow()); + + test("the vault shows the per-egress notice once and the explicit read action", async ({ + page, + }) => { + // External egress (the codex/OAuth or any non-local provider case). + await mockDocumentCapability(page, { egress: "external" }); + + await page.goto(`/documents?doc=${AI_PROBE_DOC_ID}`); + const sheet = page.getByRole("dialog"); + await expect(sheet).toBeVisible(); + + // The vendor-blind "leaves your machine" notice is shown exactly once. + await expect( + sheet.locator('[data-slot="document-ai-egress-notice"]'), + ).toHaveCount(1); + // The per-document read is the explicit path — the action is present and the + // user must tap it (auto-read never removes the affordance, only the tap). + await expect(sheet.locator('[data-slot="document-read-ai"]')).toBeVisible(); + }); + + test("reading through the action succeeds with no per-document consent prompt", async ({ + page, + }) => { + await mockDocumentCapability(page, { egress: "external" }); + let indexCalls = 0; + await page.route( + `**/api/documents/inbound/${AI_PROBE_DOC_ID}/index`, + (route) => { + indexCalls += 1; + return fulfilJson(route, { + data: { documentId: AI_PROBE_DOC_ID, indexed: true, tokenCount: 9 }, + error: null, + }); + }, + ); + + await page.goto(`/documents?doc=${AI_PROBE_DOC_ID}`); + const sheet = page.getByRole("dialog"); + await expect(sheet).toBeVisible(); + + const read = sheet.locator('[data-slot="document-read-ai"]'); + await expect(read).toBeVisible(); + await read.click(); + + await expect(page.getByText("The AI read your document.")).toBeVisible(); + expect(indexCalls).toBe(1); + }); + + test("a local-egress read shows no external notice", async ({ page }) => { + // A self-hosted local model never leaves the machine — no egress notice. + await mockDocumentCapability(page, { + egress: "local", + pdfSupported: false, + }); + + await page.goto(`/documents?doc=${AI_PROBE_DOC_ID}`); + const sheet = page.getByRole("dialog"); + await expect(sheet).toBeVisible(); + + await expect( + sheet.locator('[data-slot="document-ai-egress-notice"]'), + ).toHaveCount(0); + }); +}); From f2263361b699f955c53bd2b53ab0915eacc7a875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 11:02:20 +0200 Subject: [PATCH 6/8] style(documents): format document-settings read helper --- src/lib/documents/document-settings.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/documents/document-settings.ts b/src/lib/documents/document-settings.ts index 22d9b4f8c..348bd504f 100644 --- a/src/lib/documents/document-settings.ts +++ b/src/lib/documents/document-settings.ts @@ -19,7 +19,9 @@ import { prisma } from "@/lib/db"; * A missing row (deleted account raced against a job) resolves to `false` — * fail-closed to the local-first posture, never to external egress. */ -export async function documentAutoReadEnabled(userId: string): Promise { +export async function documentAutoReadEnabled( + userId: string, +): Promise { const row = await prisma.user.findUnique({ where: { id: userId }, select: { documentsAutoAiRead: true }, From 1a43880ab4756a55e297c5770a9f93b375e69b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 11:17:07 +0200 Subject: [PATCH 7/8] =?UTF-8?q?chore(release):=20v1.28.2=20=E2=80=94=20aut?= =?UTF-8?q?omatic=20AI=20document=20reading=20(opt-in)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One opt-in in AI settings reads every uploaded document with the configured provider — no per-document tap. Off by default; the vault stays local-first otherwise. Scanned PDFs and photos are read too (a PDF's pages are rendered to images so an image-only provider can read them). The background indexer now follows the same opt-in, so nothing leaves the machine automatically unless it is enabled. --- CHANGELOG.md | 10 ++++++++++ docs/api/openapi.yaml | 2 +- knip.json | 3 ++- package.json | 2 +- public/sw.js | 2 +- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1bf08364..6a669c910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +## [1.28.2] — 2026-07-09 — Automatic AI document reading (opt-in) + +One opt-in in AI settings: read every uploaded document automatically with your +configured AI provider — no per-document tap, the same way a lab photo is read. +Off by default; the vault stays local-first for everyone who does not turn it +on. When on, scanned PDFs and photos are read too — a PDF's pages are rendered +to images so a provider that only accepts images can still read them — and the +turn-on records a standing consent entry. The background indexer now follows the +same opt-in, so nothing leaves the machine automatically unless you enable it. + ## [1.28.1] — 2026-07-09 — Document vault load fix Fixes documents failing to load — old and newly uploaded alike — on diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 61dcb3525..870b9d6b6 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.28.1 + version: 1.28.2 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/knip.json b/knip.json index f5d9ec71d..90d913d63 100644 --- a/knip.json +++ b/knip.json @@ -62,7 +62,8 @@ "shadcn", "tailwindcss", "testcontainers", - "tw-animate-css" + "tw-animate-css", + "@napi-rs/canvas" ], "ignoreBinaries": ["docker", "docker-compose", "pg_isready", "tsx", "cosign"], "ignoreExportsUsedInFile": true, diff --git a/package.json b/package.json index 909cddd16..783cd92c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.28.1", + "version": "1.28.2", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index c7cd66bcf..b47ec4239 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.28.1"; + /* @sw-version-fallback */ "v1.28.2"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` From 23309acd6a93518d3bcd11915039cccc808f9ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 11:35:03 +0200 Subject: [PATCH 8/8] test(documents): mock the full AI-availability surface in the auto-read e2e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-read vault specs mocked only the document capability probe, leaving the usage endpoint (assist + content-index flags) unmocked — so the read action rendered but stayed disabled and the click timed out. Mock the same three endpoints the working document-AI e2e does (usage, labs OCR capability, document capability) so the action is enabled. --- e2e/documents-auto-read.spec.ts | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/e2e/documents-auto-read.spec.ts b/e2e/documents-auto-read.spec.ts index a5f03705b..bce3a9870 100644 --- a/e2e/documents-auto-read.spec.ts +++ b/e2e/documents-auto-read.spec.ts @@ -35,18 +35,45 @@ async function fulfilJson( }); } -/** Mock the document capability probe so the vault AI section renders. */ +/** + * Mock the FULL AI-availability surface the vault section probes so the read + * action renders AND is enabled: `usage` (assist + content-index flags — this is + * what un-disables the action), the labs OCR capability, and the document-scoped + * capability (provider order + vendor-blind egress class). Mirrors + * `mockAiEnabled` in documents-ai.spec.ts; egress-specific tests set the class. + */ async function mockDocumentCapability( page: Page, opts: { egress: "local" | "external"; pdfSupported?: boolean }, ): Promise { + const pdfSupported = opts.pdfSupported ?? true; + await page.route("**/api/documents/inbound/usage", (route) => + fulfilJson(route, { + data: { + usedBytes: 4096, + quotaBytes: 1_073_741_824, + maxFileBytes: 26_214_400, + acceptedExtensions: [".pdf", ".png", ".jpg", ".jpeg", ".webp"], + linkedEpisodes: [], + assistAvailable: true, + contentIndex: { enabled: true, indexedCount: 0, totalCount: 1 }, + }, + error: null, + }), + ); + await page.route("**/api/labs/ocr/capability", (route) => + fulfilJson(route, { + data: { available: true, mode: "vision", reason: null, pdfSupported }, + error: null, + }), + ); await page.route("**/api/documents/inbound/capability", (route) => fulfilJson(route, { data: { available: true, mode: "vision", reason: null, - pdfSupported: opts.pdfSupported ?? true, + pdfSupported, egress: opts.egress, }, error: null,