From 73a3ac9c363ededf96fd1a38fab8d453a0892ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 9 Jul 2026 10:36:14 +0200 Subject: [PATCH] fix(documents): reconcile the auto-index job onto the document order The auto-index-on-upload job resolved over the cost-first app-wide chain (codex priority 1) and gated with the chain-based consent check, so a pure-codex chain could silently egress a freshly uploaded document to the subscription backend on upload. Reconcile it onto resolveDocumentVisionProvider (local-first, codex last) and govern external egress with the documentsAutoAiRead opt-in: a local pick always runs (no egress); an external pick runs only when the toggle is ON, otherwise the job stays strictly local. This tightens the default and makes the opt-in path deliberate. --- .../__tests__/index-document.test.ts | 94 +++++++++++++------ src/lib/documents/index-document.ts | 74 +++++++++------ 2 files changed, 109 insertions(+), 59 deletions(-) diff --git a/src/lib/documents/__tests__/index-document.test.ts b/src/lib/documents/__tests__/index-document.test.ts index 32a4e6d29..d9a145bc6 100644 --- a/src/lib/documents/__tests__/index-document.test.ts +++ b/src/lib/documents/__tests__/index-document.test.ts @@ -1,12 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; /** - * The AI-first / local-fallback content-index decision tree. Pins: a usable - * provider takes the vision path; a text-layer PDF the provider cannot read - * falls through to local; no provider / no consent / budget-out / provider - * error all fall through to the free local path; decrypt + not-found fail - * closed. No plaintext is asserted at the upsert boundary (source + ciphertext - * path is the content-index module's concern, mocked here). + * The AI-first / local-fallback content-index decision tree. Pins: the DOCUMENT + * provider order drives the pick; a LOCAL pick is always eligible; an EXTERNAL + * pick is eligible only when the `documentsAutoAiRead` toggle is ON (OFF → the + * job stays strictly local, never egresses on upload); a text-layer PDF the + * provider cannot read falls through to local; no provider / budget-out / + * provider error all fall through to the free local path; decrypt + not-found + * fail closed. No plaintext is asserted at the upsert boundary (source + + * ciphertext path is the content-index module's concern, mocked here). */ vi.mock("@/lib/documents/ai-route-support", () => ({ @@ -28,12 +30,15 @@ vi.mock("@/lib/documents/store", () => ({ vi.mock("@/lib/labs/ocr-upload", () => ({ detectOcrMimeType: vi.fn(() => "application/pdf"), })); -vi.mock("@/lib/labs/ocr-capability", () => ({ - resolveVisionProvider: vi.fn(), +vi.mock("@/lib/documents/provider-order", () => ({ + resolveDocumentVisionProvider: vi.fn(), })); +vi.mock("@/lib/documents/document-settings", () => ({ + documentAutoReadEnabled: vi.fn(), +})); +// Use the real egress classification — only "local" is non-egress. vi.mock("@/lib/ai/consent-guard", () => ({ - assertConsentForChain: vi.fn().mockResolvedValue(undefined), - ConsentRequiredError: class ConsentRequiredError extends Error {}, + isExternalDocumentEgress: (providerType: string) => providerType !== "local", })); vi.mock("@/lib/ai/coach/budget", () => ({ buildDateKey: vi.fn(() => "2026-07-07"), @@ -52,13 +57,10 @@ import { } from "@/lib/documents/ai-route-support"; import { upsertContentIndex } from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { localExtractText } from "@/lib/documents/local-extract"; import { decryptDocumentContent } from "@/lib/documents/store"; -import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; -import { - assertConsentForChain, - ConsentRequiredError, -} from "@/lib/ai/consent-guard"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget"; const DOC = { @@ -70,6 +72,7 @@ const DOC = { status: "STORED", }; +/** An EXTERNAL (Anthropic) document pick — egress, so toggle-gated. */ const PICK = { chain: [{ providerType: "anthropic", instance: {} }], pick: { @@ -79,6 +82,16 @@ const PICK = { }, }; +/** A LOCAL (self-hosted) document pick — never egresses, toggle-independent. */ +const LOCAL_PICK = { + chain: [{ providerType: "local", instance: {} }], + pick: { + entry: { providerType: "local", instance: {} }, + providerType: "local", + pdfSupported: false, + }, +}; + function visionOk() { vi.mocked(prepareVisionInput).mockReturnValue({ ok: true, @@ -99,11 +112,14 @@ beforeEach(() => { text: "glucose fasting creatinine values report", source: "local-pdf", } as never); + // Default: the auto-read toggle is ON (so the external-pick provider path is + // exercised); the OFF-specific tests override this explicitly. + vi.mocked(documentAutoReadEnabled).mockResolvedValue(true); }); describe("indexDocumentContent — provider-first path", () => { - it("indexes via the provider (vision) when configured + consented", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + it("indexes via an external provider (vision) when configured + toggle ON", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); visionOk(); vi.mocked(transcribeDocument).mockResolvedValue({ text: "haemoglobin 14.2 cholesterol 190", @@ -120,11 +136,30 @@ describe("indexDocumentContent — provider-first path", () => { // The local path is never touched when the provider succeeds. expect(localExtractText).not.toHaveBeenCalled(); }); + + it("runs a LOCAL vision pick even when the toggle is OFF (no egress)", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue( + LOCAL_PICK as never, + ); + vi.mocked(documentAutoReadEnabled).mockResolvedValue(false); + visionOk(); + vi.mocked(transcribeDocument).mockResolvedValue({ + text: "haemoglobin 14.2 cholesterol 190", + } as never); + + const outcome = await indexDocumentContent("user-1", "doc-1"); + expect(outcome).toEqual({ indexed: true, source: "vision", tokenCount: 5 }); + expect(transcribeDocument).toHaveBeenCalledTimes(1); + expect(upsertContentIndex).toHaveBeenCalledWith( + expect.objectContaining({ source: "vision", providerType: "local" }), + ); + expect(localExtractText).not.toHaveBeenCalled(); + }); }); describe("indexDocumentContent — local fallback path", () => { it("falls back to local when NO provider is configured", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue({ + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], pick: null, } as never); @@ -141,19 +176,20 @@ describe("indexDocumentContent — local fallback path", () => { ); }); - it("falls back to local when consent is not granted", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); - vi.mocked(assertConsentForChain).mockRejectedValueOnce( - new ConsentRequiredError("insights" as never), - ); + it("stays LOCAL for an external pick when the toggle is OFF (never egresses)", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(documentAutoReadEnabled).mockResolvedValue(false); const outcome = await indexDocumentContent("user-1", "doc-1"); expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" }); + // No egress, no budget reservation, no provider call on the OFF path. + expect(prepareVisionInput).not.toHaveBeenCalled(); + expect(reserveBudget).not.toHaveBeenCalled(); expect(transcribeDocument).not.toHaveBeenCalled(); }); it("falls back to local when the provider cannot read the PDF", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); vi.mocked(prepareVisionInput).mockReturnValue({ ok: false, reason: "pdfNeedsAnthropic", @@ -166,7 +202,7 @@ describe("indexDocumentContent — local fallback path", () => { }); it("falls back to local when the daily budget is exhausted", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); visionOk(); vi.mocked(reserveBudget).mockResolvedValue({ allowed: false, @@ -179,7 +215,7 @@ describe("indexDocumentContent — local fallback path", () => { }); it("refunds and falls back to local when the provider call throws", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never); + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue(PICK as never); visionOk(); vi.mocked(transcribeDocument).mockRejectedValueOnce(new Error("boom")); @@ -191,7 +227,7 @@ describe("indexDocumentContent — local fallback path", () => { }); it("reports local-empty for a scanned PDF with no usable provider", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue({ + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], pick: null, } as never); @@ -211,11 +247,11 @@ describe("indexDocumentContent — fail-closed", () => { vi.mocked(loadOwnedDocument).mockResolvedValue(null as never); const outcome = await indexDocumentContent("user-1", "missing"); expect(outcome).toEqual({ indexed: false, reason: "not-found" }); - expect(resolveVisionProvider).not.toHaveBeenCalled(); + expect(resolveDocumentVisionProvider).not.toHaveBeenCalled(); }); it("fails closed to decrypt-error when local decryption throws", async () => { - vi.mocked(resolveVisionProvider).mockResolvedValue({ + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ chain: [], pick: null, } as never); diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 99f673cd9..193e3dd6b 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -2,19 +2,26 @@ * Content-index ONE stored document — the shared, AI-first decision tree behind * both the per-document auto-index-on-upload job and the corpus backfill. * - * Ordering (maintainer, 2026-07-07 — AI-first, local as fallback): - * 1. PROVIDER (primary). When a vision-capable AI provider is configured AND - * the user has consented, decrypt the stored original and run ONE provider - * transcription — rich, handles scanned PDFs + images via vision. Budget- - * reserved + reconciled exactly like the interactive index route; owner- - * scoped throughout. Codex-over-OAuth is subscription-covered, so this - * path carries no per-token API bill (see the research note). - * 2. LOCAL (fallback). When no provider is usable — none configured, consent - * not granted, the provider can't read this file (e.g. a text-layer PDF on - * a non-Anthropic account), the daily budget is reached, or the provider - * call fails — fall back to server-side text-layer extraction (`pdf-parse`, - * no egress, milliseconds). So content search works at a baseline even with - * no AI, and every text-layer PDF is searchable regardless of provider. + * Provider resolution follows the DOCUMENT order (local-first, codex last — + * `resolveDocumentVisionProvider`), NOT the cost-first app-wide chain, and the + * external egress is governed by the per-user `documentsAutoAiRead` opt-in: + * + * 1. LOCAL PROVIDER (a self-hosted vision model) — never egresses, so it runs + * whenever it is the document-order pick, toggle-independent. + * 2. EXTERNAL PROVIDER (codex / BYOK openai|anthropic / admin key) — reads a + * document off the machine, so the auto-index job only uses it when the + * operator opted into `documentsAutoAiRead`. OFF → the job is strictly + * local (never egresses on upload, even if an unrelated consent receipt + * exists); ON → the document-order external pick reads the original (rich, + * handles scanned PDFs + images). Budget-reserved + reconciled exactly like + * the interactive index route; owner-scoped throughout. + * 3. LOCAL TEXT-LAYER (fallback). When no provider is usable — none + * configured, the toggle is OFF for an external pick, the provider can't + * read this file (e.g. a text-layer PDF on a non-Anthropic account), the + * daily budget is reached, or the provider call fails — fall back to + * server-side text-layer extraction (`pdf-parse`, no egress, milliseconds). + * So content search works at a baseline even with no AI, and every + * text-layer PDF is searchable regardless of provider. * * Both paths write the SAME blind, encrypted index via `upsertContentIndex` * (AES-256-GCM text + opaque HMAC token tags) — nothing readable at rest. The @@ -24,10 +31,7 @@ import { Buffer } from "node:buffer"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; -import { - assertConsentForChain, - ConsentRequiredError, -} from "@/lib/ai/consent-guard"; +import { isExternalDocumentEgress } from "@/lib/ai/consent-guard"; import { buildDateKey, reconcileSpend, @@ -44,11 +48,12 @@ import { type ContentIndexSource, } from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; +import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; import { localExtractText } from "@/lib/documents/local-extract"; +import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { decryptDocumentContent } from "@/lib/documents/store"; import { detectOcrMimeType } from "@/lib/labs/ocr-upload"; import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; -import { resolveVisionProvider } from "@/lib/labs/ocr-capability"; /** * Outcome of one document's index attempt. A provider that is not usable (none @@ -71,32 +76,41 @@ export type IndexOutcome = */ export interface ResolvedIndexProvider { chain: ProviderChainResolved[]; - pick: Awaited>["pick"]; + pick: Awaited>["pick"]; consentOk: boolean; dailyCap: number; } /** - * Resolve the user's vision provider and pre-check consent once. A missing pick - * or an un-granted consent both yield an unusable context — the caller then - * falls straight to the local path. A NON-consent error propagates. + * Resolve the DOCUMENT-order vision provider and decide egress eligibility once. + * A local pick is always eligible (it never leaves the machine). An external + * pick is eligible ONLY when the operator opted into `documentsAutoAiRead` — so + * the auto-index job never silently egresses a freshly uploaded document unless + * that toggle is ON. A missing pick or an ineligible external pick both yield an + * unusable context, and the caller falls straight to the local text-layer path. */ export async function resolveIndexProvider( userId: string, ): Promise { - const { chain, pick } = await resolveVisionProvider(userId); + const { chain, pick } = await resolveDocumentVisionProvider(userId); let consentOk = false; if (pick) { - try { - await assertConsentForChain({ userId, chain, surface: "insights" }); + if (!isExternalDocumentEgress(pick.providerType)) { + // A local (self-hosted) vision pick never egresses — always eligible, + // toggle-independent. consentOk = true; - } catch (err) { - if (!(err instanceof ConsentRequiredError)) throw err; + } else { + // An external pick reads the document off the machine. The auto-index job + // only egresses on upload when the operator opted in; the toggle IS the + // standing consent (the document consent gate short-circuits on it). OFF + // → local-only, even if an unrelated receipt exists. + consentOk = await documentAutoReadEnabled(userId); } } - const dailyCap = pick - ? resolveDailyCap([{ providerType: pick.entry.providerType }]) - : 0; + const dailyCap = + pick && consentOk + ? resolveDailyCap([{ providerType: pick.entry.providerType }]) + : 0; return { chain, pick, consentOk, dailyCap }; }