feat(documents): auto-index uploads for content search, provider-first with local fallback

Index a document for content search automatically the moment it is stored,
instead of requiring a manual index action. A fire-and-forget per-document
job runs an AI-first decision tree: when a vision-capable provider is
configured and consented, transcribe the stored original (rich, handles
scanned PDFs and images), budget-reserved exactly like the interactive index
route; otherwise fall back to server-side text-layer extraction via pdf-parse
(no provider, no egress, milliseconds) so content search works at a baseline
even with no AI. Budget exhaustion, a provider that cannot read the file
(a text-layer PDF on a non-Anthropic account), and provider errors all fall
through to the free local path rather than leave a document unsearchable.

Both paths write the same blind, encrypted content index (AES-256-GCM text +
opaque HMAC token tags) — nothing readable at rest. The local path adds the
'local-pdf' / 'local-ocr' source values (free String column, no migration).
Server-side OCR for scanned images and scanned PDFs is left as a documented
seam. Pin pdf-parse + pdfjs-dist into the standalone trace so the wasm text
extractor ships in the runtime image.
This commit is contained in:
Marc-André Bombeck committed 2026-07-08 20:50:30 +02:00
1 parent a3276dcfeb
commit 5e253313fc
12 files changed
+880 -4

No files matched your search

@@ -62,6 +62,10 @@ vi.mock("@/lib/modules/gate", () => ({
requireModuleEnabled: vi.fn().mockResolvedValue({ enabled: true }),
}));
vi.mock("@/lib/jobs/document-index", () => ({
enqueueDocumentIndex: vi.fn().mockResolvedValue({ enqueued: true }),
}));
vi.mock("@/lib/rate-limit", () => ({
checkRateLimit: vi.fn().mockResolvedValue({
allowed: true,
@@ -94,6 +98,7 @@ import { POST, GET } from "../route";
import { prisma } from "@/lib/db";
import { getSession } from "@/lib/auth/session";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
const post = POST as unknown as (r: Request) => Promise<Response>;
const get = GET as unknown as (r: Request) => Promise<Response>;
@@ -203,6 +208,9 @@ describe("POST /api/documents/inbound (vault upload)", () => {
expect(stored.equals(PNG_BYTES)).toBe(false);
// The create never round-trips the blob back out.
expect(arg.omit).toEqual({ contentEncrypted: true });
// A fresh store auto-enqueues the per-document index job (owner + id).
expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1");
});
it("rejects an unidentifiable payload with 415 + reason unsupportedType", async () => {
@@ -264,6 +272,8 @@ describe("POST /api/documents/inbound (vault upload)", () => {
expect(where.contentSha256).toBe(PNG_SHA256);
expect(where.deletedAt).toBeNull();
expect(txCreate).not.toHaveBeenCalled();
// A duplicate short-circuits before the store — no index job is enqueued.
expect(enqueueDocumentIndex).not.toHaveBeenCalled();
});
it("pre-links the caller's episodes and refuses a foreign episode id", async () => {
+8
View File
@@ -41,6 +41,7 @@ import {
serialiseDocument,
type SerialisableDocument,
} from "@/lib/documents/store";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import {
detectDocumentType,
resolveDocumentLimits,
@@ -361,6 +362,13 @@ async function postUpload(request: Request): Promise<Response> {
},
});
// Auto-index the freshly stored document for content search: enqueue a
// fire-and-forget background job (provider-first, local text-layer fallback).
// The upload response never blocks on or fails because of indexing; a missing
// boss (worker not up) is a silent no-op. Only fresh inserts enqueue — a
// duplicate upload returns early above and never reaches here.
await enqueueDocumentIndex(user.id, document.id);
const links = await loadConditionLinks(user.id, [document.id]);
return apiSuccess(
serialiseDocument(