Merge pull request #439 from MBombeck/release/v1.27.29

Release v1.27.29 — automatic document indexing and AI reading
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-07-09 00:49:17 +02:00
commit 023dc4ee4c
46 files changed
+1621 -229

No files matched your search

+11
View File
@@ -2,6 +2,17 @@
## [Unreleased]
## [1.27.29] — 2026-07-08 — Automatic document indexing and AI reading
### Added
- Every uploaded document is made searchable automatically — no "index" button. On upload, a background job reads the document (the upload finishes instantly and never fails because of indexing) and adds it to the private, encrypted search index. It picks the best reader: an AI provider transcribes the original (including scanned pages and photos) when one is configured and consented; otherwise HealthLog extracts a PDF's embedded text layer locally, in milliseconds, with no AI and nothing leaving the machine — so digitally-generated PDFs (lab reports, letters) become searchable even with no AI.
- The document detail sheet gains a prominent "Read with AI" block with a status pill — Read by AI, Searchable, Making searchable…, or Not searchable yet — plus the existing suggest-details / summarise / show-text actions, consolidated. With no provider it shows a calm pointer to AI settings, and any locally-indexed document stays searchable. The old manual per-document index buttons are gone (indexing is automatic).
### Fixed
- An upload whose background index job failed to enqueue (a transient database hiccup) could return a 500 even though the file was stored; enqueuing is now fire-and-forget and never fails the upload.
## [1.27.28] — 2026-07-08 — Share a document from the document
### Added
+25 -3
View File
@@ -1,7 +1,7 @@
openapi: 3.1.0
info:
title: HealthLog API
version: 1.27.28
version: 1.27.29
description: >-
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
@@ -26193,6 +26193,15 @@ components:
- attachment
hasContentIndex:
type: boolean
contentIndexSource:
anyOf:
- type: string
enum:
- vision
- text-ocr
- local-pdf
- local-ocr
- type: "null"
createdAt:
type: string
updatedAt:
@@ -26214,6 +26223,7 @@ components:
- conditionLinks
- servingClass
- hasContentIndex
- contentIndexSource
- createdAt
- updatedAt
additionalProperties: false
@@ -26222,8 +26232,10 @@ components:
(plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until
extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or
download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the
document has a content-search index (drives the re-index affordance). Treat an unknown `kind` value as OTHER
when decoding.
document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was
produced — `vision` means an AI provider read the original, the other values are provider-free local
extractions, and it is null when `hasContentIndex` is false. Treat an unknown `kind` value as OTHER when
decoding.
DocumentConditionLink:
type: object
properties:
@@ -26448,6 +26460,15 @@ components:
- attachment
hasContentIndex:
type: boolean
contentIndexSource:
anyOf:
- type: string
enum:
- vision
- text-ocr
- local-pdf
- local-ocr
- type: "null"
createdAt:
type: string
updatedAt:
@@ -26473,6 +26494,7 @@ components:
- conditionLinks
- servingClass
- hasContentIndex
- contentIndexSource
- createdAt
- updatedAt
- facts
+86
View File
@@ -104,6 +104,13 @@ test.describe("document vault — AI assist + content search", () => {
await ensureVaultAiFixture();
});
// These AI flows drive several mocked provider round-trips per test; on a
// loaded CI runner they race the default 30s, so the whole group earns the
// tripled timeout rather than flaking green paths.
test.beforeEach(() => {
test.slow();
});
// ── (a) Review-first assist: prefill → edit → save, nothing auto-committed ──
test("assist prefills an editable draft that only saves on commit", async ({
@@ -298,6 +305,85 @@ test.describe("document vault — AI assist + content search", () => {
await expect(sheet.locator('[data-slot="assist-suggest"]')).toHaveCount(0);
});
// ── (g) "Read with AI": the prominent action maps to the index endpoint ──
test('the "Read with AI" action reads a document and confirms', async ({
page,
}) => {
await mockAiEnabled(page, { mode: "vision" });
// AI_PROBE_DOC_ID carries no content index → the action reads "Read with AI"
// (not "Read again") and the status pill is the calm "not searchable yet".
let indexCalls = 0;
let indexHadJsonBody = false;
await page.route(
`**/api/documents/inbound/${AI_PROBE_DOC_ID}/index`,
(route) => {
indexCalls += 1;
// VISION mode posts NO JSON body — the endpoint 422s a text body without
// the local-OCR opt-in, so the UI must never send one on this path.
indexHadJsonBody = Boolean(route.request().postData());
return fulfilJson(route, {
data: { documentId: AI_PROBE_DOC_ID, indexed: true, tokenCount: 12 },
error: null,
});
},
);
await page.goto(`/documents?doc=${AI_PROBE_DOC_ID}`);
const sheet = page.getByRole("dialog");
await expect(sheet).toBeVisible();
// The prominent action is present and labelled for a first read.
const read = sheet.locator('[data-slot="document-read-ai"]');
await expect(read).toBeVisible();
await expect(read).toHaveText(/Read with AI/);
// The status pill starts at the calm "not searchable yet".
const status = sheet.locator('[data-slot="content-search-status"]');
await expect(status).toHaveAttribute("data-state", "none");
await read.click();
await expect(page.getByText("The AI read your document.")).toBeVisible();
expect(indexCalls).toBe(1);
expect(indexHadJsonBody).toBe(false);
});
// ── (h) Provenance: a provider-read document is marked "Read by AI" ──────
test("a vision-indexed document surfaces the AI-read provenance", async ({
page,
}, testInfo) => {
// CONTENT_DOC_ID is seeded with source "vision" (an AI provider read it).
// No mocks: the real list/detail GET threads the provenance through.
await page.goto(`/documents?doc=${CONTENT_DOC_ID}`);
const sheet = page.getByRole("dialog");
await expect(sheet).toBeVisible();
// The detail status pill reflects the AI-read source — asserted on every
// project (this is the provenance contract; it holds on desktop + mobile).
const status = sheet.locator('[data-slot="content-search-status"]');
await expect(status).toHaveAttribute("data-state", "ai-read");
await expect(status).toHaveText(/Read by AI/);
// The timeline card carries the same AI-read marker — desktop only. The
// vault timeline is virtualized; on the narrow mobile viewport the seeded
// card renders outside the initial virtual window (not in the DOM), so a
// card-level attribute read there tests the virtualizer, not the marker.
// The marker's render logic is viewport-independent and covered by the SSR
// card test + this desktop assertion; mobile provenance is already proven
// by the status pill above.
if (testInfo.project.name !== "chromium-mobile") {
await page.keyboard.press("Escape");
const marker = page
.locator('[data-slot="document-card"]', { hasText: "Radiology note" })
.locator('[data-slot="document-searchable"]')
.first();
await expect(marker).toHaveAttribute("data-source", "ai-read", {
timeout: 20_000,
});
}
});
// ── (f) Text-mode refuses a non-image before any OCR/upload ──────────────
test("text-mode assist refuses a PDF client-side before any request", async ({
+1
View File
@@ -32,6 +32,7 @@ test.describe("medication wizard — bi-weekly", () => {
test("creates an every-2-weeks Monday medication end-to-end", async ({
page,
}) => {
test.slow();
await stubDashboardAnalytics(page);
const capture = mockMedicationsApi(page, {
id: STUB_MEDICATION_ID,
+1
View File
@@ -33,6 +33,7 @@ test.describe("medication wizard — compose-mode", () => {
test.use({ storageState: STORAGE_STATE_PATH });
test("creates a 2-schedule medication end-to-end", async ({ page }) => {
test.slow();
await stubDashboardAnalytics(page);
const listEntry = {
+5 -1
View File
@@ -221,11 +221,15 @@ export async function expectStep(
// stepper landed, so it is hidden on the desktop e2e viewport. Assert the
// viewport-independent root data-attributes the dialog always carries.
const root = page.locator('[data-slot="medication-wizard-dialog"]');
// Each step advance waits on per-step form validation; on a loaded CI runner
// that settle can exceed the default 5s expect timeout (the dialog sits at
// the prior step when the assertion first samples). Give the step-transition
// assertion room so a slow runner doesn't read a mid-transition frame.
await expect(root).toHaveAttribute(
"data-display-step",
String(displayIndex),
{
timeout: 5_000,
timeout: 20_000,
},
);
if (totalSteps != null) {
+1
View File
@@ -30,6 +30,7 @@ test.describe("medication wizard — monthly", () => {
test("creates a monthly-on-day-15 medication end-to-end", async ({
page,
}) => {
test.slow();
await stubDashboardAnalytics(page);
const capture = mockMedicationsApi(page, {
id: STUB_MEDICATION_ID,
+1
View File
@@ -34,6 +34,7 @@ test.describe("medication wizard — one-shot", () => {
test.use({ storageState: STORAGE_STATE_PATH });
test("creates a single-dose medication end-to-end", async ({ page }) => {
test.slow();
await stubDashboardAnalytics(page);
const capture = mockMedicationsApi(page, {
id: STUB_MEDICATION_ID,
+1
View File
@@ -33,6 +33,7 @@ test.describe("medication wizard — rolling", () => {
test("creates an every-7-days-from-last-intake medication end-to-end", async ({
page,
}) => {
test.slow();
await stubDashboardAnalytics(page);
const capture = mockMedicationsApi(page, {
id: STUB_MEDICATION_ID,
+1
View File
@@ -29,6 +29,7 @@ test.describe("medication wizard — weekdays", () => {
test.use({ storageState: STORAGE_STATE_PATH });
test("creates a Mo/Mi/Fr medication end-to-end", async ({ page }) => {
test.slow();
await stubDashboardAnalytics(page);
const capture = mockMedicationsApi(page, {
id: STUB_MEDICATION_ID,
+14 -7
View File
@@ -8085,7 +8085,8 @@
"attachmentBadge": "Nur Download",
"uploading": "Wird hochgeladen…",
"dismissFailed": "Fehlgeschlagenen Upload ausblenden",
"searchableBadge": "Inhalt durchsuchbar"
"searchableBadge": "Inhalt durchsuchbar",
"aiReadBadge": "Von KI gelesen"
},
"error": {
"fileTooLarge": "Diese Datei ist größer als das Limit von {maxSize}.",
@@ -8206,15 +8207,21 @@
"notSaved": "Nicht gespeichert · keine Diagnose. Nur einmalig für diese Ansicht angezeigt.",
"close": "Ausblenden"
},
"ai": {
"blockTitle": "Mit KI auslesen",
"available": "Lass die KI dieses Dokument lesen — Titel, Art und Datum vorschlagen, zusammenfassen oder den ganzen Text durchsuchbar machen.",
"read": "Mit KI auslesen",
"reread": "Erneut lesen",
"reading": "Liest…",
"statusAiRead": "Von KI gelesen",
"statusSearchable": "Durchsuchbar",
"statusIndexing": "Wird durchsuchbar gemacht…",
"statusNotYet": "Noch nicht durchsuchbar",
"readDone": "Die KI hat dein Dokument gelesen."
},
"contentIndex": {
"searchHint": "Die Suche durchsucht jetzt auch den Inhalt deiner Dokumente (ganze Wörter).",
"indexPrompt": "Dokumentinhalte durchsuchbar machen.",
"searchable": "Inhalt durchsuchbar",
"notSearchable": "Inhalt noch nicht durchsuchbar",
"indexAction": "Für Suche indizieren",
"reindexAction": "Neu indizieren",
"indexing": "Wird indiziert…",
"indexed": "Für die Suche indiziert.",
"indexAll": "Alle für Suche indizieren",
"indexAllPending": "Wird gestartet…",
"indexAllQueued": "{count} Dokument(e) werden im Hintergrund indiziert.",
+14 -7
View File
@@ -8085,7 +8085,8 @@
"attachmentBadge": "Download only",
"uploading": "Uploading…",
"dismissFailed": "Dismiss failed upload",
"searchableBadge": "Contents searchable"
"searchableBadge": "Contents searchable",
"aiReadBadge": "Read by AI"
},
"error": {
"fileTooLarge": "This file is larger than the {maxSize} limit.",
@@ -8206,15 +8207,21 @@
"notSaved": "Not saved · not a diagnosis. Shown once for this view only.",
"close": "Hide"
},
"ai": {
"blockTitle": "Read with AI",
"available": "Let the AI read this document — suggest a title, type and date, summarise it, or make its full text searchable.",
"read": "Read with AI",
"reread": "Read again",
"reading": "Reading…",
"statusAiRead": "Read by AI",
"statusSearchable": "Searchable",
"statusIndexing": "Making searchable…",
"statusNotYet": "Not searchable yet",
"readDone": "The AI read your document."
},
"contentIndex": {
"searchHint": "Search now looks inside your documents too (whole words).",
"indexPrompt": "Make document contents searchable.",
"searchable": "Contents searchable",
"notSearchable": "Contents not searchable yet",
"indexAction": "Index for search",
"reindexAction": "Re-index",
"indexing": "Indexing…",
"indexed": "Indexed for search.",
"indexAll": "Index all for search",
"indexAllPending": "Starting…",
"indexAllQueued": "Indexing {count} document(s) in the background.",
+14 -7
View File
@@ -8085,7 +8085,8 @@
"attachmentBadge": "Solo descarga",
"uploading": "Subiendo…",
"dismissFailed": "Descartar subida fallida",
"searchableBadge": "Contenido buscable"
"searchableBadge": "Contenido buscable",
"aiReadBadge": "Leído por IA"
},
"error": {
"fileTooLarge": "Este archivo supera el límite de {maxSize}.",
@@ -8206,15 +8207,21 @@
"notSaved": "No se guarda · no es un diagnóstico. Se muestra solo una vez en esta vista.",
"close": "Ocultar"
},
"ai": {
"blockTitle": "Leer con IA",
"available": "Deja que la IA lea este documento: sugerir un título, tipo y fecha, resumirlo o hacer que todo su texto sea buscable.",
"read": "Leer con IA",
"reread": "Leer de nuevo",
"reading": "Leyendo…",
"statusAiRead": "Leído por IA",
"statusSearchable": "Buscable",
"statusIndexing": "Haciéndolo buscable…",
"statusNotYet": "Aún no buscable",
"readDone": "La IA leyó tu documento."
},
"contentIndex": {
"searchHint": "La búsqueda ahora también mira dentro de tus documentos (palabras completas).",
"indexPrompt": "Haz que el contenido de los documentos sea buscable.",
"searchable": "Contenido buscable",
"notSearchable": "Contenido aún no buscable",
"indexAction": "Indexar para búsqueda",
"reindexAction": "Reindexar",
"indexing": "Indexando…",
"indexed": "Indexado para búsqueda.",
"indexAll": "Indexar todo para búsqueda",
"indexAllPending": "Iniciando…",
"indexAllQueued": "Indexando {count} documento(s) en segundo plano.",
+14 -7
View File
@@ -8085,7 +8085,8 @@
"attachmentBadge": "Téléchargement uniquement",
"uploading": "Téléversement…",
"dismissFailed": "Ignorer le téléversement échoué",
"searchableBadge": "Contenu consultable"
"searchableBadge": "Contenu consultable",
"aiReadBadge": "Lu par l’IA"
},
"error": {
"fileTooLarge": "Ce fichier dépasse la limite de {maxSize}.",
@@ -8206,15 +8207,21 @@
"notSaved": "Non enregistré · pas un diagnostic. Affiché une seule fois pour cette vue.",
"close": "Masquer"
},
"ai": {
"blockTitle": "Lire avec l’IA",
"available": "Laissez l’IA lire ce document — suggérer un titre, un type et une date, le résumer ou rendre tout son texte consultable.",
"read": "Lire avec l’IA",
"reread": "Relire",
"reading": "Lecture…",
"statusAiRead": "Lu par l’IA",
"statusSearchable": "Consultable",
"statusIndexing": "Rendu consultable…",
"statusNotYet": "Pas encore consultable",
"readDone": "L’IA a lu votre document."
},
"contentIndex": {
"searchHint": "La recherche explore désormais aussi le contenu de vos documents (mots entiers).",
"indexPrompt": "Rendre le contenu des documents consultable.",
"searchable": "Contenu consultable",
"notSearchable": "Contenu pas encore consultable",
"indexAction": "Indexer pour la recherche",
"reindexAction": "Réindexer",
"indexing": "Indexation…",
"indexed": "Indexé pour la recherche.",
"indexAll": "Tout indexer pour la recherche",
"indexAllPending": "Démarrage…",
"indexAllQueued": "Indexation de {count} document(s) en arrière-plan.",
+14 -7
View File
@@ -8085,7 +8085,8 @@
"attachmentBadge": "Solo download",
"uploading": "Caricamento…",
"dismissFailed": "Ignora caricamento non riuscito",
"searchableBadge": "Contenuto ricercabile"
"searchableBadge": "Contenuto ricercabile",
"aiReadBadge": "Letto dall’IA"
},
"error": {
"fileTooLarge": "Questo file supera il limite di {maxSize}.",
@@ -8206,15 +8207,21 @@
"notSaved": "Non salvato · non è una diagnosi. Mostrato una sola volta per questa vista.",
"close": "Nascondi"
},
"ai": {
"blockTitle": "Leggi con l’IA",
"available": "Lascia che l’IA legga questo documento: suggerire un titolo, un tipo e una data, riassumerlo o rendere ricercabile tutto il testo.",
"read": "Leggi con l’IA",
"reread": "Leggi di nuovo",
"reading": "Lettura…",
"statusAiRead": "Letto dall’IA",
"statusSearchable": "Ricercabile",
"statusIndexing": "Rendo ricercabile…",
"statusNotYet": "Non ancora ricercabile",
"readDone": "L’IA ha letto il tuo documento."
},
"contentIndex": {
"searchHint": "La ricerca ora esamina anche il contenuto dei tuoi documenti (parole intere).",
"indexPrompt": "Rendi ricercabile il contenuto dei documenti.",
"searchable": "Contenuto ricercabile",
"notSearchable": "Contenuto non ancora ricercabile",
"indexAction": "Indicizza per la ricerca",
"reindexAction": "Reindicizza",
"indexing": "Indicizzazione…",
"indexed": "Indicizzato per la ricerca.",
"indexAll": "Indicizza tutto per la ricerca",
"indexAllPending": "Avvio…",
"indexAllQueued": "Indicizzazione di {count} documento/i in background.",
+14 -7
View File
@@ -8085,7 +8085,8 @@
"attachmentBadge": "Tylko do pobrania",
"uploading": "Przesyłanie…",
"dismissFailed": "Odrzuć nieudane przesyłanie",
"searchableBadge": "Treść przeszukiwalna"
"searchableBadge": "Treść przeszukiwalna",
"aiReadBadge": "Odczytane przez AI"
},
"error": {
"fileTooLarge": "Ten plik przekracza limit {maxSize}.",
@@ -8206,15 +8207,21 @@
"notSaved": "Nie zapisano · to nie diagnoza. Wyświetlane tylko raz w tym widoku.",
"close": "Ukryj"
},
"ai": {
"blockTitle": "Odczytaj z AI",
"available": "Pozwól AI odczytać ten dokument — zaproponować tytuł, typ i datę, streścić go lub udostępnić cały tekst do wyszukiwania.",
"read": "Odczytaj z AI",
"reread": "Odczytaj ponownie",
"reading": "Odczytywanie…",
"statusAiRead": "Odczytane przez AI",
"statusSearchable": "Wyszukiwalne",
"statusIndexing": "Udostępnianie do wyszukiwania…",
"statusNotYet": "Jeszcze nie do wyszukania",
"readDone": "AI odczytało Twój dokument."
},
"contentIndex": {
"searchHint": "Wyszukiwanie przeszukuje teraz także treść Twoich dokumentów (całe słowa).",
"indexPrompt": "Udostępnij treść dokumentów do wyszukiwania.",
"searchable": "Treść przeszukiwalna",
"notSearchable": "Treść jeszcze nieprzeszukiwalna",
"indexAction": "Zindeksuj do wyszukiwania",
"reindexAction": "Zindeksuj ponownie",
"indexing": "Indeksowanie…",
"indexed": "Zindeksowano do wyszukiwania.",
"indexAll": "Zindeksuj wszystko do wyszukiwania",
"indexAllPending": "Uruchamianie…",
"indexAllQueued": "Indeksowanie {count} dokumentu/ów w tle.",
+13 -1
View File
@@ -242,7 +242,19 @@ const nextConfig: NextConfig = {
// files explicitly keeps them in the runtime image alongside the
// bundled module.
outputFileTracingIncludes: {
"*": ["./src/lib/ai/prompts/safety-contracts.*.yaml"],
"*": [
"./src/lib/ai/prompts/safety-contracts.*.yaml",
// Document AI local text extraction runs `pdf-parse` (→ pdfjs-dist)
// server-side in the content-index job — the provider-free fallback that
// keeps content search working with no AI configured. Both are pure JS +
// wasm; there is NO native `.node` on the text-layer path (`@napi-rs/canvas`
// is only pulled for image rendering, which `getText()` never invokes).
// The standalone tracer can miss pdfjs-dist's wasm decoders (they resolve
// via `new URL(..., import.meta.url)`), so pin both packages into the
// runtime image explicitly. pnpm keeps pdfjs-dist unhoisted under `.pnpm`.
"./node_modules/.pnpm/pdf-parse@*/node_modules/pdf-parse/**",
"./node_modules/.pnpm/pdfjs-dist@*/node_modules/pdfjs-dist/**",
],
},
// v1.4.34 IW-A — silence the Turbopack NFT trace warnings emitted
// during `next build`. The tracer follows the `MAXMIND_LICENSE_KEY`
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "healthlog",
"version": "1.27.28",
"version": "1.27.29",
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
"license": "PolyForm-Noncommercial-1.0.0",
"homepage": "https://healthlog.dev",
+9 -2
View File
@@ -19,10 +19,17 @@ export default defineConfig({
testIgnore: ["setup/**"],
globalSetup: "./e2e/setup/global-setup.ts",
timeout: 30_000,
expect: { timeout: 5_000 },
expect: { timeout: 10_000 },
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 1 : 0,
// Two retries on CI: under a loaded shared runner, DOM-settle-gated
// assertions (wizard step transitions, list refetch → card provenance,
// disclosure toggles) intermittently sample a mid-transition frame. A
// rotating single test failed each run while 211 passed; a second retry
// absorbs that transient contention without masking a real failure (a true
// break fails all three attempts). The default expect timeout is also
// lifted from 5s → 10s for the same settle headroom.
retries: process.env.CI ? 2 : 0,
workers: process.env.CI ? 2 : undefined,
reporter: process.env.CI
? [["github"], ["html", { open: "never" }]]
+4 -1
View File
@@ -5965,7 +5965,10 @@ model DocumentContentIndex {
/// (raw SQL in the migration; Prisma cannot express a GIN index).
searchTokens String[] @default([]) @map("search_tokens")
/// Provenance of the indexed text: "vision" (provider transcription of the
/// stored original) or "text-ocr" (browser-OCR text posted by the client).
/// stored original), "text-ocr" (browser-OCR text posted by the client),
/// "local-pdf" (server-side text-layer extraction, no provider) or "local-ocr"
/// (server-side OCR of a scanned document — deferred follow-up). Free String;
/// adding a source value needs no migration.
source String
/// Provider type that produced the text (diagnostic only), or null on the
/// text-ocr path.
+1 -1
View File
@@ -36,7 +36,7 @@ try {
// v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild.
const CACHE_VERSION =
(typeof self !== "undefined" && self.__APP_VERSION__) ||
/* @sw-version-fallback */ "v1.27.28";
/* @sw-version-fallback */ "v1.27.29";
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
// v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`
@@ -22,6 +22,9 @@ vi.mock("@/lib/db", () => ({
deleteMany: vi.fn(),
createMany: vi.fn(),
},
documentContentIndex: {
findUnique: vi.fn(),
},
illnessEpisode: {
findMany: vi.fn(),
},
@@ -131,6 +134,9 @@ beforeEach(() => {
vi.mocked(prisma.documentConditionLink.createMany).mockResolvedValue({
count: 0,
} as never);
vi.mocked(prisma.documentContentIndex.findUnique).mockResolvedValue(
null as never,
);
});
describe("PATCH /api/documents/inbound/[id]", () => {
+15 -3
View File
@@ -27,7 +27,10 @@ import { serialiseDocumentDetail } from "@/lib/documents/store";
import { annotate } from "@/lib/logging/context";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import { documentUpdateSchema } from "@/lib/validations/inbound-documents";
import {
documentUpdateSchema,
toContentIndexSource,
} from "@/lib/validations/inbound-documents";
import type { Prisma } from "@/generated/prisma/client";
type RouteParams = { params: Promise<{ id: string }> };
@@ -80,7 +83,7 @@ export const GET = apiHandler(
loadConditionLinks(user.id, [document.id]),
prisma.documentContentIndex.findUnique({
where: { documentId: document.id },
select: { id: true },
select: { source: true },
}),
]);
@@ -95,6 +98,7 @@ export const GET = apiHandler(
document.facts,
links.get(document.id) ?? [],
contentIndex !== null,
toContentIndexSource(contentIndex?.source),
),
);
},
@@ -173,7 +177,13 @@ export const PATCH = apiHandler(
omit: { contentEncrypted: true },
include: { facts: { orderBy: { createdAt: "asc" } } },
});
const links = await loadConditionLinks(user.id, [document.id]);
const [links, contentIndex] = await Promise.all([
loadConditionLinks(user.id, [document.id]),
prisma.documentContentIndex.findUnique({
where: { documentId: document.id },
select: { source: true },
}),
]);
const touched = [
...Object.keys(data),
@@ -199,6 +209,8 @@ export const PATCH = apiHandler(
document,
document.facts,
links.get(document.id) ?? [],
contentIndex !== null,
toContentIndexSource(contentIndex?.source),
),
);
},
@@ -62,6 +62,10 @@ vi.mock("@/lib/modules/gate", () => ({
requireModuleEnabled: vi.fn().mockResolvedValue({ enabled: true }),
}));
vi.mock("@/lib/jobs/document-index", () => ({
enqueueDocumentIndex: vi.fn().mockResolvedValue({ enqueued: true }),
}));
vi.mock("@/lib/rate-limit", () => ({
checkRateLimit: vi.fn().mockResolvedValue({
allowed: true,
@@ -94,6 +98,7 @@ import { POST, GET } from "../route";
import { prisma } from "@/lib/db";
import { getSession } from "@/lib/auth/session";
import { requireModuleEnabled } from "@/lib/modules/gate";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
const post = POST as unknown as (r: Request) => Promise<Response>;
const get = GET as unknown as (r: Request) => Promise<Response>;
@@ -203,6 +208,9 @@ describe("POST /api/documents/inbound (vault upload)", () => {
expect(stored.equals(PNG_BYTES)).toBe(false);
// The create never round-trips the blob back out.
expect(arg.omit).toEqual({ contentEncrypted: true });
// A fresh store auto-enqueues the per-document index job (owner + id).
expect(enqueueDocumentIndex).toHaveBeenCalledWith("user-1", "doc-1");
});
it("rejects an unidentifiable payload with 415 + reason unsupportedType", async () => {
@@ -264,6 +272,8 @@ describe("POST /api/documents/inbound (vault upload)", () => {
expect(where.contentSha256).toBe(PNG_SHA256);
expect(where.deletedAt).toBeNull();
expect(txCreate).not.toHaveBeenCalled();
// A duplicate short-circuits before the store — no index job is enqueued.
expect(enqueueDocumentIndex).not.toHaveBeenCalled();
});
it("pre-links the caller's episodes and refuses a foreign episode id", async () => {
+18 -6
View File
@@ -41,6 +41,7 @@ import {
serialiseDocument,
type SerialisableDocument,
} from "@/lib/documents/store";
import { enqueueDocumentIndex } from "@/lib/jobs/document-index";
import {
detectDocumentType,
resolveDocumentLimits,
@@ -54,6 +55,7 @@ import { checkRateLimit, rateLimitHeaders } from "@/lib/rate-limit";
import {
documentCreateSchema,
documentListQuerySchema,
toContentIndexSource,
} from "@/lib/validations/inbound-documents";
import type { Prisma } from "@/generated/prisma/client";
@@ -361,6 +363,14 @@ async function postUpload(request: Request): Promise<Response> {
},
});
// Auto-index the freshly stored document for content search: enqueue a
// fire-and-forget background job (provider-first, local text-layer fallback).
// The upload response never blocks on or fails because of indexing — the
// enqueue is not awaited and swallows its own errors (a missing boss or a
// transient send failure is a silent no-op). Only fresh inserts enqueue — a
// duplicate upload returns early above and never reaches here.
void enqueueDocumentIndex(user.id, document.id);
const links = await loadConditionLinks(user.id, [document.id]);
return apiSuccess(
serialiseDocument(
@@ -498,15 +508,16 @@ export const GET = apiHandler(async (request: Request) => {
page.map((d) => d.id),
);
// Which of the page's documents have a content index (drives `hasContentIndex`
// + the re-index affordance). One indexed grouped query; never the ciphertext.
const indexedIds = new Set<string>();
// Which of the page's documents have a content index (drives the searchable
// status + the provenance the UI reads to tell an AI-read document from a
// locally-indexed one). One grouped query; never the ciphertext.
const indexSources = new Map<string, string>();
if (page.length > 0) {
const indexed = await prisma.documentContentIndex.findMany({
where: { userId: user.id, documentId: { in: page.map((d) => d.id) } },
select: { documentId: true },
select: { documentId: true, source: true },
});
for (const row of indexed) indexedIds.add(row.documentId);
for (const row of indexed) indexSources.set(row.documentId, row.source);
}
annotate({
@@ -525,7 +536,8 @@ export const GET = apiHandler(async (request: Request) => {
doc,
factCounts.get(doc.id) ?? { factCount: 0, pendingCount: 0 },
linkMap.get(doc.id) ?? [],
indexedIds.has(doc.id),
indexSources.has(doc.id),
toContentIndexSource(indexSources.get(doc.id)),
),
),
nextCursor,
@@ -6,7 +6,7 @@ import type { DocumentSuggestionDto } from "@/lib/validations/inbound-documents"
import {
AiUnavailableHint,
AssistSuggestionReview,
ContentIndexStatus,
ContentSearchStatus,
DocumentSummaryPanel,
} from "../document-ai-panels";
@@ -175,25 +175,47 @@ describe("<DocumentSummaryPanel>", () => {
});
});
describe("<ContentIndexStatus>", () => {
it("offers indexing when the document is not yet searchable", () => {
describe("<ContentSearchStatus>", () => {
it("shows a calm not-searchable-yet pill before the auto-index lands", () => {
const html = render(
<ContentIndexStatus
<ContentSearchStatus
hasContentIndex={false}
source={null}
isPending={false}
onIndex={noop}
/>,
);
expect(html).toContain('data-slot="content-index-status"');
expect(html).toContain("Contents not searchable yet");
expect(html).toContain("Index for search");
expect(html).toContain('data-slot="content-search-status"');
expect(html).toContain('data-state="none"');
expect(html).toContain("Not searchable yet");
// A status, never a chore button.
expect(html).not.toContain("<button");
});
it("offers a re-index when the document is already searchable", () => {
const html = render(
<ContentIndexStatus hasContentIndex isPending={false} onIndex={noop} />,
it("highlights an AI-read document distinctly from a locally-indexed one", () => {
const aiRead = render(
<ContentSearchStatus hasContentIndex source="vision" isPending={false} />,
);
expect(html).toContain("Contents searchable");
expect(html).toContain("Re-index");
expect(aiRead).toContain('data-state="ai-read"');
expect(aiRead).toContain("Read by AI");
expect(aiRead).toContain("text-primary");
const local = render(
<ContentSearchStatus
hasContentIndex
source="local-pdf"
isPending={false}
/>,
);
expect(local).toContain('data-state="searchable"');
expect(local).toContain("Searchable");
expect(local).not.toContain("Read by AI");
});
it("reflects a running read as an indexing state", () => {
const html = render(
<ContentSearchStatus hasContentIndex={false} source={null} isPending />,
);
expect(html).toContain('data-state="indexing"');
expect(html).toContain("Making searchable…");
});
});
@@ -6,10 +6,11 @@ import type { DocumentSuggestionDto } from "@/lib/validations/inbound-documents"
import { DocumentAiSection } from "../document-ai-section";
/**
* The detail sheet's AI area is availability-gated: with a provider it shows the
* assist / summary toolbar; without one it shows ONLY the calm settings pointer
* (never an action the endpoint would 422). The summary panel, when open, keeps
* the session-only note visible.
* The detail sheet's "Read with AI" block is availability-gated: with a provider
* it offers the prominent "Read with AI" action alongside suggest / summary;
* without one the actions collapse to the calm settings pointer (never an action
* the endpoint would 422). The searchable status pill reflects the auto-index in
* both states. The summary panel, when open, keeps the session-only note visible.
*/
function render(node: React.ReactNode) {
@@ -52,6 +53,7 @@ function base(
summaryErrorKey: null,
onCloseSummary: noop,
hasContentIndex: false,
contentIndexSource: null,
indexPending: false,
onIndex: noop,
...overrides,
@@ -60,25 +62,59 @@ function base(
}
describe("<DocumentAiSection>", () => {
it("offers the assist + summary toolbar when a provider is available", () => {
it("leads with the prominent Read-with-AI action when a provider is available", () => {
const html = render(base({ aiEnabled: true }));
expect(html).toContain('data-slot="document-read-ai"');
expect(html).toContain("Read with AI");
expect(html).toContain('data-slot="assist-suggest"');
expect(html).toContain("Suggest details");
expect(html).toContain("Summarise");
expect(html).toContain("Show extracted text");
// No unavailable pointer when the affordance is offered.
// No unavailable pointer when the actions are offered.
expect(html).not.toContain('data-slot="assist-unavailable"');
});
it("shows ONLY the calm settings pointer when no provider is available", () => {
it("labels the action as a re-read once a provider has already read it", () => {
const html = render(
base({ aiEnabled: false, unavailableReason: "no-provider" }),
base({ hasContentIndex: true, contentIndexSource: "vision" }),
);
expect(html).toContain('data-slot="document-read-ai"');
expect(html).toContain("Read again");
// The status pill reflects the AI-read provenance.
expect(html).toContain('data-state="ai-read"');
expect(html).toContain("Read by AI");
});
it("shows the calm settings pointer (no 422 actions) when no provider is available", () => {
const html = render(
base({
aiEnabled: false,
indexEnabled: false,
unavailableReason: "no-provider",
}),
);
expect(html).toContain('data-slot="assist-unavailable"');
expect(html).toContain('href="/settings/ai"');
// The assist / index actions must be absent — never a 422-guaranteed tap.
// Every AI action must be absent — never a 422-guaranteed tap.
expect(html).not.toContain('data-slot="assist-suggest"');
expect(html).not.toContain('data-slot="content-index-status"');
expect(html).not.toContain('data-slot="document-read-ai"');
// But the searchable status pill still renders — auto-index is honest here.
expect(html).toContain('data-slot="content-search-status"');
});
it("keeps a locally-indexed document honestly searchable without a provider", () => {
const html = render(
base({
aiEnabled: false,
indexEnabled: false,
unavailableReason: "no-provider",
hasContentIndex: true,
contentIndexSource: "local-pdf",
}),
);
expect(html).toContain('data-state="searchable"');
expect(html).toContain("Searchable");
expect(html).not.toContain("Read by AI");
});
it("renders the reviewed suggestion drafts when present", () => {
@@ -106,9 +142,9 @@ describe("<DocumentAiSection>", () => {
expect(html).toContain("Not saved");
});
it("hides the content-index row when indexing is unavailable", () => {
it("hides the Read-with-AI action when indexing is unavailable but keeps suggest", () => {
const html = render(base({ aiEnabled: true, indexEnabled: false }));
expect(html).toContain('data-slot="assist-suggest"');
expect(html).not.toContain('data-slot="content-index-status"');
expect(html).not.toContain('data-slot="document-read-ai"');
});
});
@@ -32,6 +32,7 @@ function doc(overrides: Partial<InboundDocumentDto> = {}): InboundDocumentDto {
conditionLinks: [{ episodeId: "ep-knee", name: "Knie" }],
servingClass: "inline",
hasContentIndex: false,
contentIndexSource: null,
createdAt: "2025-10-05T08:00:00.000Z",
updatedAt: "2025-10-05T08:00:00.000Z",
...overrides,
@@ -95,9 +96,25 @@ describe("<DocumentCard>", () => {
/>,
);
expect(html).toContain('data-slot="document-searchable"');
expect(html).toContain('data-source="local"');
expect(html).toContain("Contents searchable");
});
it("marks an AI-read document with the highlighted read-by-AI badge", () => {
const html = render(
<DocumentCard
document={doc({ hasContentIndex: true, contentIndexSource: "vision" })}
selected={false}
onToggleSelected={noop}
onOpen={noop}
highlighted={false}
/>,
);
expect(html).toContain('data-source="ai-read"');
expect(html).toContain("Read by AI");
expect(html).toContain("text-primary");
});
it("shows no searchable marker when the document is not indexed", () => {
const html = render(
<DocumentCard
@@ -52,6 +52,7 @@ function doc(id: string, title: string): InboundDocumentDto {
conditionLinks: [{ episodeId: "ep1", name: "Knee" }],
servingClass: "inline",
hasContentIndex: false,
contentIndexSource: null,
createdAt: "2025-10-05T08:00:00.000Z",
updatedAt: "2025-10-05T08:00:00.000Z",
};
@@ -33,6 +33,7 @@ function doc(overrides: Partial<InboundDocumentDto> = {}): InboundDocumentDto {
conditionLinks: [],
servingClass: "inline",
hasContentIndex: false,
contentIndexSource: null,
createdAt: "2026-03-11T08:00:00.000Z",
updatedAt: "2026-03-11T08:00:00.000Z",
...overrides,
+78 -37
View File
@@ -6,29 +6,39 @@
* the session-only note are pinned by static-render tests.
*
* - `AiUnavailableHint` — the calm "set up an AI provider" pointer shown in
* place of the AI toolbar when no provider is configured (never an error).
* place of the AI actions when no provider is configured (never an error).
* - `AssistSuggestionReview` — the reviewed DRAFT card: suggested title / type
* / date, each applied only by an explicit tap. Nothing is written until the
* user applies it (and the title lands in the editable field, not on disk).
* - `DocumentSummaryPanel` — the transient summary / extracted-text panel with
* the persistent "not saved · not a diagnosis" note.
* - `ContentIndexStatus` — the per-document searchable state + index action.
* - `ContentSearchStatus` — the per-document searchable pill reflecting the
* auto-index: "Read by AI" (provider read the original), "Searchable"
* (locally indexed), "Making searchable…" (a read is running), or the calm
* "not searchable yet". A STATUS, never a chore button.
*/
import { Check, FileText, ScanSearch, Sparkles, X } from "lucide-react";
import { Loader2, ScanSearch, Sparkles } from "lucide-react";
import { Check, FileText, X } from "lucide-react";
import Link from "next/link";
import { Button } from "@/components/ui/button";
import { Skeleton } from "@/components/ui/skeleton";
import { useTranslations } from "@/lib/i18n/context";
import { cn } from "@/lib/utils";
import type {
DocumentSuggestionDto,
DocumentSummaryMode,
import {
isAiReadSource,
type DocumentContentIndexSourceValue,
type DocumentSuggestionDto,
type DocumentSummaryMode,
} from "@/lib/validations/inbound-documents";
import type { DocumentDescribeResult } from "./use-document-assist";
/** Calm pointer to the AI settings, shown when assist is unavailable. */
/**
* Calm pointer to the AI settings, shown when assist is unavailable. When the
* document is already searchable (auto-indexed locally) the copy stays honest —
* it is searchable, an AI provider only adds a richer read.
*/
export function AiUnavailableHint({
reason,
}: {
@@ -280,44 +290,75 @@ export function DocumentSummaryPanel({
}
/**
* The per-document content-search state: whether the body is searchable, and an
* explicit index / re-index action. Shown only when indexing is available.
* The per-document searchable status pill. Reflects the auto-index (indexing is
* automatic on upload, so this is a state, never a to-do): a document is "Read
* by AI" when a provider read the original, "Searchable" when it is only locally
* indexed, "Making searchable…" while a read runs, and a calm "not searchable
* yet" otherwise. The AI-read pill is highlighted (primary) — that read is the
* richer one and worth surfacing.
*/
export function ContentIndexStatus({
export function ContentSearchStatus({
hasContentIndex,
source,
isPending,
onIndex,
}: {
hasContentIndex: boolean;
source: DocumentContentIndexSourceValue | null;
/** A read/index is running right now. */
isPending: boolean;
onIndex: () => void;
}) {
const { t } = useTranslations();
return (
<div
data-slot="content-index-status"
className="flex items-center justify-between gap-2"
>
<p className="text-muted-foreground inline-flex items-center gap-1.5 text-xs">
<ScanSearch className="size-3.5 shrink-0" aria-hidden />
{hasContentIndex
? t("documents.contentIndex.searchable")
: t("documents.contentIndex.notSearchable")}
</p>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 shrink-0 px-2.5 text-xs"
onClick={onIndex}
disabled={isPending}
if (isPending) {
return (
<span
data-slot="content-search-status"
data-state="indexing"
className="text-muted-foreground inline-flex items-center gap-1.5 rounded-full px-2 py-0.5 text-xs"
>
{isPending
? t("documents.contentIndex.indexing")
: hasContentIndex
? t("documents.contentIndex.reindexAction")
: t("documents.contentIndex.indexAction")}
</Button>
</div>
<Loader2
className="size-3.5 shrink-0 animate-spin motion-reduce:animate-none"
aria-hidden
/>
{t("documents.ai.statusIndexing")}
</span>
);
}
if (!hasContentIndex) {
return (
<span
data-slot="content-search-status"
data-state="none"
className="text-muted-foreground inline-flex items-center gap-1.5 rounded-full px-2 py-0.5 text-xs"
>
<ScanSearch className="size-3.5 shrink-0" aria-hidden />
{t("documents.ai.statusNotYet")}
</span>
);
}
if (isAiReadSource(source)) {
return (
<span
data-slot="content-search-status"
data-state="ai-read"
className="bg-primary/10 text-primary inline-flex items-center gap-1.5 rounded-full px-2 py-0.5 text-xs font-medium"
>
<Sparkles className="size-3.5 shrink-0" aria-hidden />
{t("documents.ai.statusAiRead")}
</span>
);
}
return (
<span
data-slot="content-search-status"
data-state="searchable"
className="text-muted-foreground inline-flex items-center gap-1.5 rounded-full px-2 py-0.5 text-xs"
>
<ScanSearch className="size-3.5 shrink-0" aria-hidden />
{t("documents.ai.statusSearchable")}
</span>
);
}
+141 -87
View File
@@ -1,29 +1,43 @@
"use client";
/**
* v1.27.22 (Document vault P2) — the AI area of the document detail sheet,
* lifted out of the sheet so the availability gate (toolbar vs. calm pointer)
* and the review-first wiring render without the sheet's dialog portal and are
* pinned by static-render tests.
* The "Read with AI" block of the document detail sheet — one coherent, inviting
* home for every AI reading action, lifted out of the sheet so the availability
* gate and the review-first wiring render without the sheet's dialog portal and
* are pinned by static-render tests.
*
* Indexing is AUTOMATIC on upload, so this block never nags a "please index"
* chore. It shows the document's searchable status (auto-indexed) as a pill, and
* offers AI reading as a capability the user reaches for when they want more:
*
* - "Read with AI" (prominent) runs a provider read of the document — the
* richer pass that also refreshes the searchable index. Labelled "Read
* again" once a provider has already read it.
* - "Suggest details" drafts a title / type / date to review before saving.
* - "Summarise" / "Show text" surface a transient, session-only read-out.
*
* Presentational: the sheet owns the mutations + capability probe and passes
* their state + handlers down. When `aiEnabled` is false the whole area is the
* calm "set up an AI provider" pointer — never an error, and the manual form
* below stays fully usable.
* their state + handlers down. When `aiEnabled` is false the actions collapse to
* the calm "set up an AI provider" pointer (never an error) — the searchable
* status pill stays, honestly reflecting any local auto-index, and the manual
* form below stays fully usable.
*/
import { FileText, Sparkles } from "lucide-react";
import { FileText, ScanText, WandSparkles } from "lucide-react";
import { Button } from "@/components/ui/button";
import { useTranslations } from "@/lib/i18n/context";
import type {
DocumentSuggestionDto,
DocumentSummaryMode,
import { cn } from "@/lib/utils";
import {
isAiReadSource,
type DocumentContentIndexSourceValue,
type DocumentSuggestionDto,
type DocumentSummaryMode,
} from "@/lib/validations/inbound-documents";
import {
AiUnavailableHint,
AssistSuggestionReview,
ContentIndexStatus,
ContentSearchStatus,
DocumentSummaryPanel,
} from "./document-ai-panels";
import type { DocumentDescribeResult } from "./use-document-assist";
@@ -51,6 +65,7 @@ export function DocumentAiSection({
summaryErrorKey,
onCloseSummary,
hasContentIndex,
contentIndexSource,
indexPending,
onIndex,
}: {
@@ -77,92 +92,131 @@ export function DocumentAiSection({
summaryErrorKey: string | null;
onCloseSummary: () => void;
hasContentIndex: boolean;
contentIndexSource: DocumentContentIndexSourceValue | null;
indexPending: boolean;
onIndex: () => void;
}) {
const { t } = useTranslations();
if (!aiEnabled) {
return (
<div className="space-y-3" data-slot="document-ai-section">
<AiUnavailableHint reason={unavailableReason} />
</div>
);
}
const aiRead = hasContentIndex && isAiReadSource(contentIndexSource);
const readLabel = indexPending
? t("documents.ai.reading")
: aiRead
? t("documents.ai.reread")
: t("documents.ai.read");
return (
<div className="space-y-3" data-slot="document-ai-section">
<div className="flex flex-wrap items-center gap-2">
<Button
type="button"
variant="outline"
size="sm"
data-slot="assist-suggest"
onClick={onSuggest}
disabled={suggestPending || actionsDisabled}
>
<Sparkles className="size-4" aria-hidden />
{suggestPending
? t("documents.assist.suggesting")
: t("documents.assist.suggest")}
</Button>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => onSummarise("summary")}
disabled={summaryPending || actionsDisabled}
>
<FileText className="size-4" aria-hidden />
{t("documents.summary.summarise")}
</Button>
<Button
type="button"
variant="ghost"
size="sm"
onClick={() => onSummarise("text")}
disabled={summaryPending || actionsDisabled}
>
{t("documents.summary.showText")}
</Button>
<div
data-slot="document-ai-section"
className="border-border bg-muted/30 space-y-3 rounded-lg border p-3 md:p-4"
>
<div className="flex items-start justify-between gap-2">
<div className="flex min-w-0 items-center gap-2">
<WandSparkles
className="text-foreground size-5 shrink-0"
aria-hidden
/>
<p className="text-sm font-semibold">
{t("documents.ai.blockTitle")}
</p>
</div>
<ContentSearchStatus
hasContentIndex={hasContentIndex}
source={contentIndexSource}
isPending={indexPending}
/>
</div>
{suggestErrorKey ? (
<p role="alert" className="text-destructive text-sm">
{t(suggestErrorKey)}
</p>
) : null}
{aiEnabled ? (
<>
<p className="text-muted-foreground text-xs">
{t("documents.ai.available")}
</p>
{suggestion ? (
<AssistSuggestionReview
suggestion={suggestion}
kindLabel={suggestionKindLabel}
dateLabel={suggestionDateLabel}
applied={appliedFields}
onUseTitle={onUseTitle}
onUseKind={onUseKind}
onUseDate={onUseDate}
onDismiss={onDismissSuggestion}
/>
) : null}
<div className="flex flex-wrap items-center gap-2">
{indexEnabled ? (
<Button
type="button"
size="sm"
data-slot="document-read-ai"
onClick={onIndex}
disabled={indexPending || actionsDisabled}
>
<ScanText
className={cn(
"size-4",
indexPending && "animate-pulse motion-reduce:animate-none",
)}
aria-hidden
/>
{readLabel}
</Button>
) : null}
<Button
type="button"
variant="outline"
size="sm"
data-slot="assist-suggest"
onClick={onSuggest}
disabled={suggestPending || actionsDisabled}
>
<WandSparkles className="size-4" aria-hidden />
{suggestPending
? t("documents.assist.suggesting")
: t("documents.assist.suggest")}
</Button>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => onSummarise("summary")}
disabled={summaryPending || actionsDisabled}
>
<FileText className="size-4" aria-hidden />
{t("documents.summary.summarise")}
</Button>
<Button
type="button"
variant="ghost"
size="sm"
onClick={() => onSummarise("text")}
disabled={summaryPending || actionsDisabled}
>
{t("documents.summary.showText")}
</Button>
</div>
{summaryOutput ? (
<DocumentSummaryPanel
output={summaryOutput}
result={summaryResult}
isPending={summaryPending}
errorKey={summaryErrorKey}
onClose={onCloseSummary}
/>
) : null}
{suggestErrorKey ? (
<p role="alert" className="text-destructive text-sm">
{t(suggestErrorKey)}
</p>
) : null}
{indexEnabled ? (
<ContentIndexStatus
hasContentIndex={hasContentIndex}
isPending={indexPending}
onIndex={onIndex}
/>
) : null}
{suggestion ? (
<AssistSuggestionReview
suggestion={suggestion}
kindLabel={suggestionKindLabel}
dateLabel={suggestionDateLabel}
applied={appliedFields}
onUseTitle={onUseTitle}
onUseKind={onUseKind}
onUseDate={onUseDate}
onDismiss={onDismissSuggestion}
/>
) : null}
{summaryOutput ? (
<DocumentSummaryPanel
output={summaryOutput}
result={summaryResult}
isPending={summaryPending}
errorKey={summaryErrorKey}
onClose={onCloseSummary}
/>
) : null}
</>
) : (
<AiUnavailableHint reason={unavailableReason} />
)}
</div>
);
}
+33 -13
View File
@@ -13,7 +13,7 @@
* hover / focus / while selected. The whole card is clickable through an
* invisible overlay button; the checkbox floats above it.
*/
import { Download, ScanSearch, X } from "lucide-react";
import { Download, ScanSearch, Sparkles, X } from "lucide-react";
import { useRef } from "react";
import { Badge } from "@/components/ui/badge";
@@ -22,7 +22,10 @@ import { Card, CardContent } from "@/components/ui/card";
import { Checkbox } from "@/components/ui/checkbox";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { cn } from "@/lib/utils";
import type { InboundDocumentDto } from "@/lib/validations/inbound-documents";
import {
isAiReadSource,
type InboundDocumentDto,
} from "@/lib/validations/inbound-documents";
import { DOCUMENT_KIND_ICONS } from "./document-kind-meta";
import type { UploadQueueItem } from "./use-document-upload";
import { documentDateKey, formatBytes } from "./vault-utils";
@@ -140,18 +143,35 @@ export function DocumentCard({
</Badge>
) : null}
{document.hasContentIndex ? (
// Subtle "contents searchable" marker — icon-only to keep the
// row calm; the label rides an accessible name.
<span
data-slot="document-searchable"
className="text-muted-foreground inline-flex items-center"
title={t("documents.card.searchableBadge")}
>
<ScanSearch className="size-3.5" aria-hidden />
<span className="sr-only">
{t("documents.card.searchableBadge")}
// Subtle searchable marker — icon-only to keep the row calm; the
// label rides an accessible name. An AI-read document gets the
// highlighted Sparkles (primary) so the richer read is legible at
// a glance; a locally-indexed one keeps the quiet muted glass.
isAiReadSource(document.contentIndexSource) ? (
<span
data-slot="document-searchable"
data-source="ai-read"
className="text-primary inline-flex items-center"
title={t("documents.card.aiReadBadge")}
>
<Sparkles className="size-3.5" aria-hidden />
<span className="sr-only">
{t("documents.card.aiReadBadge")}
</span>
</span>
</span>
) : (
<span
data-slot="document-searchable"
data-source="local"
className="text-muted-foreground inline-flex items-center"
title={t("documents.card.searchableBadge")}
>
<ScanSearch className="size-3.5" aria-hidden />
<span className="sr-only">
{t("documents.card.searchableBadge")}
</span>
</span>
)
) : null}
</div>
) : null}
@@ -365,7 +365,7 @@ export function DocumentDetailSheet({
indexDoc.mutate(
{ mode: aiMode, target: aiTarget },
{
onSuccess: () => toast.success(t("documents.contentIndex.indexed")),
onSuccess: () => toast.success(t("documents.ai.readDone")),
onError: (error) => toast.error(t(documentAiErrorKey(error))),
},
);
@@ -531,6 +531,7 @@ export function DocumentDetailSheet({
summary.reset();
}}
hasContentIndex={doc.hasContentIndex}
contentIndexSource={doc.contentIndexSource}
indexPending={indexDoc.isPending}
onIndex={runIndex}
/>
@@ -0,0 +1,230 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
/**
* The AI-first / local-fallback content-index decision tree. Pins: a usable
* provider takes the vision path; a text-layer PDF the provider cannot read
* falls through to local; no provider / no consent / budget-out / provider
* error all fall through to the free local path; decrypt + not-found fail
* closed. No plaintext is asserted at the upsert boundary (source + ciphertext
* path is the content-index module's concern, mocked here).
*/
vi.mock("@/lib/documents/ai-route-support", () => ({
loadOwnedDocument: vi.fn(),
prepareVisionInput: vi.fn(),
}));
vi.mock("@/lib/documents/content-index", () => ({
upsertContentIndex: vi.fn().mockResolvedValue({ tokenCount: 5 }),
}));
vi.mock("@/lib/documents/describe", () => ({
transcribeDocument: vi.fn(),
}));
vi.mock("@/lib/documents/local-extract", () => ({
localExtractText: vi.fn(),
}));
vi.mock("@/lib/documents/store", () => ({
decryptDocumentContent: vi.fn(() => Buffer.from([1, 2, 3])),
}));
vi.mock("@/lib/labs/ocr-upload", () => ({
detectOcrMimeType: vi.fn(() => "application/pdf"),
}));
vi.mock("@/lib/labs/ocr-capability", () => ({
resolveVisionProvider: vi.fn(),
}));
vi.mock("@/lib/ai/consent-guard", () => ({
assertConsentForChain: vi.fn().mockResolvedValue(undefined),
ConsentRequiredError: class ConsentRequiredError extends Error {},
}));
vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-07-07"),
reserveBudget: vi.fn(),
reconcileSpend: vi.fn().mockResolvedValue(undefined),
resolveDailyCap: vi.fn(() => 1000),
}));
vi.mock("@/lib/ai/ai-budgets", () => ({
AI_BUDGETS: { documentTranscribe: { temperature: 0, maxTokens: 4000 } },
}));
import { indexDocumentContent } from "../index-document";
import {
loadOwnedDocument,
prepareVisionInput,
} from "@/lib/documents/ai-route-support";
import { upsertContentIndex } from "@/lib/documents/content-index";
import { transcribeDocument } from "@/lib/documents/describe";
import { localExtractText } from "@/lib/documents/local-extract";
import { decryptDocumentContent } from "@/lib/documents/store";
import { resolveVisionProvider } from "@/lib/labs/ocr-capability";
import {
assertConsentForChain,
ConsentRequiredError,
} from "@/lib/ai/consent-guard";
import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget";
const DOC = {
id: "doc-1",
kind: "OTHER",
contentEncrypted: new Uint8Array([1, 2, 3]),
contentCodec: "binary2",
mimeType: "application/pdf",
status: "STORED",
};
const PICK = {
chain: [{ providerType: "anthropic", instance: {} }],
pick: {
entry: { providerType: "anthropic", instance: {} },
providerType: "anthropic",
pdfSupported: true,
},
};
function visionOk() {
vi.mocked(prepareVisionInput).mockReturnValue({
ok: true,
images: [],
documents: [{ mediaType: "application/pdf", dataBase64: "AA==" }],
} as never);
}
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(loadOwnedDocument).mockResolvedValue(DOC as never);
vi.mocked(reserveBudget).mockResolvedValue({
allowed: true,
reserved: 1,
} as never);
vi.mocked(localExtractText).mockResolvedValue({
ok: true,
text: "glucose fasting creatinine values report",
source: "local-pdf",
} as never);
});
describe("indexDocumentContent — provider-first path", () => {
it("indexes via the provider (vision) when configured + consented", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never);
visionOk();
vi.mocked(transcribeDocument).mockResolvedValue({
text: "haemoglobin 14.2 cholesterol 190",
} as never);
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: true, source: "vision", tokenCount: 5 });
expect(transcribeDocument).toHaveBeenCalledTimes(1);
expect(upsertContentIndex).toHaveBeenCalledWith(
expect.objectContaining({ source: "vision", providerType: "anthropic" }),
);
// Budget reserved then reconciled at the full reserved amount (charged).
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 1, 1, "2026-07-07");
// The local path is never touched when the provider succeeds.
expect(localExtractText).not.toHaveBeenCalled();
});
});
describe("indexDocumentContent — local fallback path", () => {
it("falls back to local when NO provider is configured", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue({
chain: [],
pick: null,
} as never);
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({
indexed: true,
source: "local-pdf",
tokenCount: 5,
});
expect(transcribeDocument).not.toHaveBeenCalled();
expect(upsertContentIndex).toHaveBeenCalledWith(
expect.objectContaining({ source: "local-pdf", providerType: null }),
);
});
it("falls back to local when consent is not granted", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never);
vi.mocked(assertConsentForChain).mockRejectedValueOnce(
new ConsentRequiredError("insights" as never),
);
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" });
expect(transcribeDocument).not.toHaveBeenCalled();
});
it("falls back to local when the provider cannot read the PDF", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never);
vi.mocked(prepareVisionInput).mockReturnValue({
ok: false,
reason: "pdfNeedsAnthropic",
} as never);
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" });
expect(reserveBudget).not.toHaveBeenCalled();
expect(transcribeDocument).not.toHaveBeenCalled();
});
it("falls back to local when the daily budget is exhausted", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never);
visionOk();
vi.mocked(reserveBudget).mockResolvedValue({
allowed: false,
reserved: 0,
} as never);
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" });
expect(transcribeDocument).not.toHaveBeenCalled();
});
it("refunds and falls back to local when the provider call throws", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue(PICK as never);
visionOk();
vi.mocked(transcribeDocument).mockRejectedValueOnce(new Error("boom"));
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toMatchObject({ indexed: true, source: "local-pdf" });
// Reservation refunded to zero spend.
expect(reconcileSpend).toHaveBeenCalledWith("user-1", 1, 0, "2026-07-07");
expect(localExtractText).toHaveBeenCalledTimes(1);
});
it("reports local-empty for a scanned PDF with no usable provider", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue({
chain: [],
pick: null,
} as never);
vi.mocked(localExtractText).mockResolvedValue({
ok: false,
reason: "empty",
} as never);
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "local-empty" });
expect(upsertContentIndex).not.toHaveBeenCalled();
});
});
describe("indexDocumentContent — fail-closed", () => {
it("returns not-found for an unknown / non-owned document", async () => {
vi.mocked(loadOwnedDocument).mockResolvedValue(null as never);
const outcome = await indexDocumentContent("user-1", "missing");
expect(outcome).toEqual({ indexed: false, reason: "not-found" });
expect(resolveVisionProvider).not.toHaveBeenCalled();
});
it("fails closed to decrypt-error when local decryption throws", async () => {
vi.mocked(resolveVisionProvider).mockResolvedValue({
chain: [],
pick: null,
} as never);
vi.mocked(decryptDocumentContent).mockImplementationOnce(() => {
throw new Error("bad key");
});
const outcome = await indexDocumentContent("user-1", "doc-1");
expect(outcome).toEqual({ indexed: false, reason: "decrypt-error" });
expect(upsertContentIndex).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,82 @@
import { describe, expect, it, vi } from "vitest";
import { jsPDF } from "jspdf";
/**
* Local, provider-free text extraction. Pins: a real text-layer PDF yields its
* text with source "local-pdf" (no provider, no OCR); a PDF below the min-chars
* gate is treated as empty (scanned); an image is the deferred-OCR seam
* (unsupported); malformed bytes fail closed to "error" without throwing.
*/
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
import {
extractPdfText,
localExtractText,
LOCAL_TEXT_MIN_CHARS,
} from "../local-extract";
/** Build a real text-layer PDF (jsPDF is isomorphic; Helvetica text layer). */
function textLayerPdf(lines: string[]): Buffer {
const doc = new jsPDF();
let y = 10;
for (const line of lines) {
doc.text(line, 10, y);
y += 10;
}
return Buffer.from(doc.output("arraybuffer"));
}
describe("extractPdfText", () => {
it("reads a text-layer PDF's embedded text as source local-pdf", async () => {
const pdf = textLayerPdf([
"Haemoglobin 14.2 g/dL Cholesterol total 190 mg/dL",
"Patient report creatinine glucose fasting values",
]);
const result = await extractPdfText(pdf);
expect(result.ok).toBe(true);
if (result.ok) {
expect(result.source).toBe("local-pdf");
expect(result.text).toContain("Haemoglobin");
expect(result.text).toContain("creatinine");
// The default "-- N of M --" page marker must not leak into the text.
expect(result.text).not.toMatch(/-- \d+ of \d+ --/);
}
});
it("treats a PDF below the min-chars gate as empty (scanned proxy)", async () => {
const pdf = textLayerPdf(["x"]);
expect("x".length).toBeLessThan(LOCAL_TEXT_MIN_CHARS);
const result = await extractPdfText(pdf);
expect(result).toEqual({ ok: false, reason: "empty" });
});
it("fails closed to error on malformed bytes without throwing", async () => {
const result = await extractPdfText(Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]));
expect(result).toEqual({ ok: false, reason: "error" });
});
});
describe("localExtractText", () => {
it("routes application/pdf through the text-layer reader", async () => {
const pdf = textLayerPdf([
"Discharge letter diagnosis medication dosage instructions",
]);
const result = await localExtractText(pdf, "application/pdf");
expect(result.ok).toBe(true);
if (result.ok) expect(result.source).toBe("local-pdf");
});
it("returns unsupported for images (deferred server-OCR seam)", async () => {
const result = await localExtractText(
Buffer.from([0xff, 0xd8]),
"image/jpeg",
);
expect(result).toEqual({ ok: false, reason: "unsupported" });
});
it("returns unsupported for any other MIME", async () => {
const result = await localExtractText(Buffer.from([0]), "text/plain");
expect(result).toEqual({ ok: false, reason: "unsupported" });
});
});
+11 -2
View File
@@ -240,8 +240,17 @@ export function decryptIndexText(buf: Uint8Array): string {
return decryptFromBytes(buf);
}
/** Provenance of the indexed text. */
export type ContentIndexSource = "vision" | "text-ocr";
/**
* Provenance of the indexed text.
* - `vision` → an AI provider transcribed the stored original (image or PDF,
* incl. scanned) — the AI-first primary path.
* - `text-ocr` → browser-OCR text posted by the client (opt-in local OCR).
* - `local-pdf` → server-side text-layer extraction of a PDF (`pdf-parse`), no
* provider, no egress — the fallback when no provider is usable.
* - `local-ocr` → server-side OCR of a scanned image/PDF (deferred follow-up).
*/
export type ContentIndexSource =
"vision" | "text-ocr" | "local-pdf" | "local-ocr";
export interface UpsertContentIndexArgs {
userId: string;
+233
View File
@@ -0,0 +1,233 @@
/**
* Content-index ONE stored document — the shared, AI-first decision tree behind
* both the per-document auto-index-on-upload job and the corpus backfill.
*
* Ordering (maintainer, 2026-07-07 — AI-first, local as fallback):
* 1. PROVIDER (primary). When a vision-capable AI provider is configured AND
* the user has consented, decrypt the stored original and run ONE provider
* transcription — rich, handles scanned PDFs + images via vision. Budget-
* reserved + reconciled exactly like the interactive index route; owner-
* scoped throughout. Codex-over-OAuth is subscription-covered, so this
* path carries no per-token API bill (see the research note).
* 2. LOCAL (fallback). When no provider is usable — none configured, consent
* not granted, the provider can't read this file (e.g. a text-layer PDF on
* a non-Anthropic account), the daily budget is reached, or the provider
* call fails — fall back to server-side text-layer extraction (`pdf-parse`,
* no egress, milliseconds). So content search works at a baseline even with
* no AI, and every text-layer PDF is searchable regardless of provider.
*
* Both paths write the SAME blind, encrypted index via `upsertContentIndex`
* (AES-256-GCM text + opaque HMAC token tags) — nothing readable at rest. The
* function NEVER throws for an expected outcome; it returns a tagged result the
* caller logs, so a bad document can never abort an upload or a batch.
*/
import { Buffer } from "node:buffer";
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
import {
assertConsentForChain,
ConsentRequiredError,
} from "@/lib/ai/consent-guard";
import {
buildDateKey,
reconcileSpend,
reserveBudget,
resolveDailyCap,
} from "@/lib/ai/coach/budget";
import {
loadOwnedDocument,
prepareVisionInput,
type LoadedDocument,
} from "@/lib/documents/ai-route-support";
import {
upsertContentIndex,
type ContentIndexSource,
} from "@/lib/documents/content-index";
import { transcribeDocument } from "@/lib/documents/describe";
import { localExtractText } from "@/lib/documents/local-extract";
import { decryptDocumentContent } from "@/lib/documents/store";
import { detectOcrMimeType } from "@/lib/labs/ocr-upload";
import type { ProviderChainResolved } from "@/lib/ai/provider-runner";
import { resolveVisionProvider } from "@/lib/labs/ocr-capability";
/**
* Outcome of one document's index attempt. A provider that is not usable (none
* configured, no consent, budget exhausted, cannot read the file, or errored)
* is never itself a terminal outcome — the tree always falls through to the
* free local path, so the terminal reasons are only the local ones.
*/
export type IndexOutcome =
| { indexed: true; source: ContentIndexSource; tokenCount: number }
| {
indexed: false;
reason:
"not-found" | "local-empty" | "local-unsupported" | "decrypt-error";
};
/**
* A resolved, consent-checked provider context. Resolve ONCE per user and reuse
* across a batch so the corpus backfill does not re-read the provider chain per
* document. `usable` is true only when a vision pick exists AND consent passed.
*/
export interface ResolvedIndexProvider {
chain: ProviderChainResolved[];
pick: Awaited<ReturnType<typeof resolveVisionProvider>>["pick"];
consentOk: boolean;
dailyCap: number;
}
/**
* Resolve the user's vision provider and pre-check consent once. A missing pick
* or an un-granted consent both yield an unusable context — the caller then
* falls straight to the local path. A NON-consent error propagates.
*/
export async function resolveIndexProvider(
userId: string,
): Promise<ResolvedIndexProvider> {
const { chain, pick } = await resolveVisionProvider(userId);
let consentOk = false;
if (pick) {
try {
await assertConsentForChain({ userId, chain, surface: "insights" });
consentOk = true;
} catch (err) {
if (!(err instanceof ConsentRequiredError)) throw err;
}
}
const dailyCap = pick
? resolveDailyCap([{ providerType: pick.entry.providerType }])
: 0;
return { chain, pick, consentOk, dailyCap };
}
/**
* Try the PROVIDER (vision) path for one already-loaded document. Returns a
* terminal outcome on success/budget/provider-error, or `null` to signal
* "fall through to local" (no usable provider, or the provider cannot read this
* particular file — e.g. a PDF on a non-Anthropic account).
*/
async function tryProviderIndex(
userId: string,
document: LoadedDocument,
provider: ResolvedIndexProvider,
): Promise<IndexOutcome | null> {
const { pick } = provider;
if (!pick || !provider.consentOk) return null;
const vision = prepareVisionInput(document, pick.pdfSupported);
if (!vision.ok) {
// A decrypt failure is terminal (local would fail the same way); anything
// else (fileType / pdfNeedsAnthropic) falls through to the local path — a
// text-layer PDF on a non-Anthropic account is read locally for free.
if (vision.reason === "decryptFailed") {
return { indexed: false, reason: "decrypt-error" };
}
return null;
}
const dateKey = buildDateKey();
const reservation = await reserveBudget(
userId,
AI_BUDGETS.documentTranscribe.maxTokens,
dateKey,
provider.dailyCap,
);
// Budget exhausted → fall through to the free local path rather than stall;
// a text-layer PDF stays searchable even once the AI allowance is spent.
if (!reservation.allowed) return null;
try {
const { text } = await transcribeDocument({
provider: pick.entry.instance,
providerType: pick.providerType,
images: vision.images,
documents: vision.documents,
});
await reconcileSpend(
userId,
reservation.reserved,
reservation.reserved,
dateKey,
);
const { tokenCount } = await upsertContentIndex({
userId,
documentId: document.id,
text,
source: "vision",
providerType: pick.providerType,
});
return { indexed: true, source: "vision", tokenCount };
} catch {
// Refund the reservation and let the caller fall through to local — a
// transient provider miss must never leave a text-layer PDF unsearchable.
await reconcileSpend(userId, reservation.reserved, 0, dateKey);
return null;
}
}
/** Try the LOCAL (provider-free) text-layer path for one loaded document. */
async function tryLocalIndex(
userId: string,
document: LoadedDocument,
): Promise<IndexOutcome> {
let buffer: Buffer;
try {
buffer = decryptDocumentContent(
document.contentEncrypted,
document.contentCodec,
);
} catch {
return { indexed: false, reason: "decrypt-error" };
}
// Re-derive the MIME from the bytes (never trust the stored label), matching
// the provider path's posture; fall back to the stored MIME if unrecognised.
const mime = detectOcrMimeType(buffer) ?? document.mimeType;
const result = await localExtractText(buffer, mime);
if (result.ok) {
const { tokenCount } = await upsertContentIndex({
userId,
documentId: document.id,
text: result.text,
source: result.source,
providerType: null,
});
return { indexed: true, source: result.source, tokenCount };
}
if (result.reason === "unsupported") {
return { indexed: false, reason: "local-unsupported" };
}
if (result.reason === "error") {
return { indexed: false, reason: "decrypt-error" };
}
return { indexed: false, reason: "local-empty" };
}
/**
* Index one already-loaded document: provider-first, local-fallback. Reuses a
* pre-resolved provider context so a batch resolves the chain once.
*/
export async function indexLoadedDocument(
userId: string,
document: LoadedDocument,
provider: ResolvedIndexProvider,
): Promise<IndexOutcome> {
const providerOutcome = await tryProviderIndex(userId, document, provider);
if (providerOutcome) return providerOutcome;
return tryLocalIndex(userId, document);
}
/**
* Index ONE document by id (owner-scoped): resolve the provider, load the
* document, then run the provider-first/local-fallback tree. This is the entry
* point the per-document auto-index-on-upload job calls.
*/
export async function indexDocumentContent(
userId: string,
documentId: string,
): Promise<IndexOutcome> {
const document = await loadOwnedDocument(userId, documentId);
if (!document) return { indexed: false, reason: "not-found" };
const provider = await resolveIndexProvider(userId);
return indexLoadedDocument(userId, document, provider);
}
+116
View File
@@ -0,0 +1,116 @@
/**
* Local (provider-free) text extraction for the document vault.
*
* When NO AI provider is configured — or the user has not consented to one —
* content search still needs a baseline. This module extracts a PDF's embedded
* text layer SERVER-SIDE with `pdf-parse` (already a direct dependency; its
* `getText()` touches only the pure-JS/wasm pdfjs core, never the native
* `@napi-rs/canvas`), in milliseconds, with no OCR and no third-party egress.
* The extracted text feeds the SAME blind, encrypted content index the provider
* path writes (`upsertContentIndex`), so nothing readable is stored at rest.
*
* Digitally-generated PDFs (lab reports, discharge letters, referrals — the
* clinical common case) carry a real embedded text layer this path reads.
* Scanned/image-only PDFs and uploaded images have NO text layer: local OCR
* (server-side tesseract + `deu+eng` traineddata baked into the image, plus
* `@napi-rs/canvas` rasterisation for scanned PDFs) is a deferred follow-up, NOT
* built here — it carries the only real Alpine/standalone-tracing + CPU cost.
* Until it lands, those documents stay un-indexed on the local path and are
* covered by the provider (vision) path whenever one is configured.
*/
import { Buffer } from "node:buffer";
import { PDFParse } from "pdf-parse";
import { annotate } from "@/lib/logging/context";
/**
* Minimum joined text length for a PDF to count as a real text-layer document.
* Below this a PDF is treated as scanned/empty — a few stray glyphs from a form
* field or a page number are not a usable index.
*/
export const LOCAL_TEXT_MIN_CHARS = 24;
/** Cap on pages parsed — bounds CPU/time on a pathological many-page PDF. */
const LOCAL_MAX_PAGES = 200;
/** Provenance tag written to `DocumentContentIndex.source` on the local path. */
export type LocalExtractSource = "local-pdf" | "local-ocr";
/**
* Outcome of a local extraction attempt.
* - `ok` → usable text was recovered (index it).
* - `empty` → a valid document with no usable text layer (scanned PDF);
* a provider vision pass is the only way to read it.
* - `unsupported` → a MIME the local path cannot read without OCR (images) or
* at all; leave un-indexed locally.
* - `error` → the file was malformed / unreadable; treat as un-indexed.
*/
export type LocalExtractResult =
| { ok: true; text: string; source: LocalExtractSource }
| { ok: false; reason: "empty" | "unsupported" | "error" };
/**
* Extract a text-layer PDF's embedded text server-side. Returns `empty` for a
* scanned/no-text-layer PDF and `error` for a malformed/encrypted one — never
* throws, so a bad document can never abort the caller (the upload/backfill job
* just leaves it un-indexed).
*/
export async function extractPdfText(
buffer: Buffer,
): Promise<LocalExtractResult> {
let parser: PDFParse | null = null;
try {
parser = new PDFParse({ data: new Uint8Array(buffer) });
const result = await parser.getText({
// Bound the page walk and drop the default "-- N of M --" page markers so
// only real document text is tokenised.
first: LOCAL_MAX_PAGES,
pageJoiner: "",
});
const text = (result.text ?? "").trim();
if (text.length < LOCAL_TEXT_MIN_CHARS) {
return { ok: false, reason: "empty" };
}
return { ok: true, text, source: "local-pdf" };
} catch {
return { ok: false, reason: "error" };
} finally {
if (parser) {
try {
await parser.destroy();
} catch {
// Best-effort teardown; a destroy failure must not mask the result.
}
}
}
}
/**
* Dispatch local extraction by MIME. PDFs go through the text-layer reader;
* images are the deferred server-OCR seam (see the module header) and return
* `unsupported` so the caller falls through to the provider path or leaves the
* document un-indexed.
*/
export async function localExtractText(
buffer: Buffer,
mime: string,
): Promise<LocalExtractResult> {
if (mime === "application/pdf") {
return extractPdfText(buffer);
}
if (mime.startsWith("image/")) {
// SEAM (NOT DONE): server-side tesseract OCR for scanned images/PDFs is a
// deferred follow-up. It needs the `deu+eng` traineddata baked into the
// image and (for scanned PDFs) `@napi-rs/canvas` traced into the standalone
// build — the only real deployment cost in this area. Until it ships, a
// no-provider image stays un-indexed locally; a configured vision provider
// covers it on the AI-first path.
annotate({
action: { name: "documents.localIndex.ocrNotImplemented" },
meta: { mime },
});
return { ok: false, reason: "unsupported" };
}
return { ok: false, reason: "unsupported" };
}
+5
View File
@@ -19,6 +19,7 @@ import type {
import { servingClassFor } from "@/lib/documents/upload-policy";
import type {
DocumentConditionLinkDto,
DocumentContentIndexSourceValue,
ExtractedFactDto,
FactData,
FactProvenance,
@@ -135,6 +136,7 @@ export function serialiseDocument(
counts: { factCount: number; pendingCount: number },
conditionLinks: DocumentConditionLinkDto[] = [],
hasContentIndex = false,
contentIndexSource: DocumentContentIndexSourceValue | null = null,
): InboundDocumentDto {
return {
id: doc.id,
@@ -157,6 +159,7 @@ export function serialiseDocument(
conditionLinks,
servingClass: servingClassFor(doc.mimeType),
hasContentIndex,
contentIndexSource: hasContentIndex ? contentIndexSource : null,
createdAt: doc.createdAt.toISOString(),
updatedAt: doc.updatedAt.toISOString(),
};
@@ -183,6 +186,7 @@ export function serialiseDocumentDetail(
facts: ExtractedFact[],
conditionLinks: DocumentConditionLinkDto[] = [],
hasContentIndex = false,
contentIndexSource: DocumentContentIndexSourceValue | null = null,
): InboundDocumentDetailDto {
const pendingCount = facts.filter((f) => f.status === "PENDING").length;
// `factCount` excludes REJECTED facts (a rejected fact is discarded, not part
@@ -194,6 +198,7 @@ export function serialiseDocumentDetail(
{ factCount, pendingCount },
conditionLinks,
hasContentIndex,
contentIndexSource,
),
facts: facts.map(serialiseFact),
};
@@ -0,0 +1,72 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
/**
* Per-document auto-index job. Pins: the enqueue coalesces per document via a
* singletonKey and no-ops when the boss is absent; the worker delegates to the
* shared decision tree and never throws for an expected outcome.
*/
vi.mock("@/lib/documents/index-document", () => ({
indexDocumentContent: vi.fn(),
}));
vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: vi.fn() }));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
import {
DOCUMENT_INDEX_QUEUE,
enqueueDocumentIndex,
runDocumentIndex,
} from "../document-index";
import { indexDocumentContent } from "@/lib/documents/index-document";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
beforeEach(() => {
vi.clearAllMocks();
});
describe("enqueueDocumentIndex", () => {
it("coalesces per document via a singletonKey", async () => {
const send = vi.fn().mockResolvedValue("job-1");
vi.mocked(getGlobalBoss).mockReturnValue({ send } as never);
const result = await enqueueDocumentIndex("user-1", "doc-1");
expect(result).toEqual({ enqueued: true });
expect(send).toHaveBeenCalledWith(
DOCUMENT_INDEX_QUEUE,
expect.objectContaining({ userId: "user-1", documentId: "doc-1" }),
expect.objectContaining({ singletonKey: "document-index|doc-1" }),
);
});
it("no-ops when the boss is not running", async () => {
vi.mocked(getGlobalBoss).mockReturnValue(null as never);
const result = await enqueueDocumentIndex("user-1", "doc-1");
expect(result).toEqual({ enqueued: false });
});
it("swallows a boss.send failure to a no-op (never fails the upload)", async () => {
const send = vi.fn().mockRejectedValue(new Error("db down"));
vi.mocked(getGlobalBoss).mockReturnValue({ send } as never);
// Must resolve, never reject — the upload has already committed.
await expect(enqueueDocumentIndex("user-1", "doc-1")).resolves.toEqual({
enqueued: false,
});
});
});
describe("runDocumentIndex", () => {
it("delegates to the shared decision tree", async () => {
vi.mocked(indexDocumentContent).mockResolvedValue({
indexed: true,
source: "local-pdf",
tokenCount: 3,
} as never);
await runDocumentIndex({ userId: "user-1", documentId: "doc-1" });
expect(indexDocumentContent).toHaveBeenCalledWith("user-1", "doc-1");
});
it("ignores a payload missing ids without calling the tree", async () => {
await runDocumentIndex({ userId: "", documentId: "" });
expect(indexDocumentContent).not.toHaveBeenCalled();
});
});
+83
View File
@@ -0,0 +1,83 @@
/**
* Automatic per-document content indexing, enqueued the moment a document is
* stored (`POST /api/documents/inbound`). Fire-and-forget: the upload response
* never blocks on or fails because of indexing.
*
* The job runs the shared AI-first decision tree (`indexDocumentContent`):
* provider (vision) first when one is configured + consented, else local
* text-layer extraction. Owner-scoped; the provider path is consent + budget
* gated; the local path needs neither (no egress). Serial concurrency keeps the
* provider calls + PDF parsing off the request pool.
*
* The queue MUST be registered in the maintenance registrar
* (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it.
*/
import { indexDocumentContent } from "@/lib/documents/index-document";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { annotate } from "@/lib/logging/context";
export const DOCUMENT_INDEX_QUEUE = "document-index";
/** Serial concurrency — provider calls + PDF parse, kept off the request pool. */
export const DOCUMENT_INDEX_CONCURRENCY = 1;
export interface DocumentIndexPayload {
userId: string;
documentId: string;
enqueuedAt?: string;
}
/**
* Index one document (owner-scoped, provider-first/local-fallback). Never
* throws for an expected outcome — the decision tree returns a tagged result —
* so pg-boss does not retry a document that simply has no indexable text.
*/
export async function runDocumentIndex(
payload: DocumentIndexPayload,
): Promise<void> {
const { userId, documentId } = payload;
if (!userId || !documentId) return;
const outcome = await indexDocumentContent(userId, documentId);
annotate({
action: { name: "documents.autoIndex.run" },
meta: outcome.indexed
? { documentId, indexed: true, source: outcome.source }
: { documentId, indexed: false, reason: outcome.reason },
});
}
/**
* Enqueue a per-document index job. Coalesced by `singletonKey` per document so
* a duplicate/idempotent re-upload cannot pile up parallel runs. Fire-and-
* forget — a missing boss (worker not up) is a no-op, and a `boss.send` failure
* (a transient DB hiccup) is swallowed to a no-op too, so enqueue can NEVER fail
* an already-stored upload. Indexing is a background nicety; the stored document
* is the contract. A dropped enqueue is recoverable via the corpus backfill.
*/
export async function enqueueDocumentIndex(
userId: string,
documentId: string,
): Promise<{ enqueued: boolean }> {
const boss = getGlobalBoss();
if (!boss) return { enqueued: false };
const payload: DocumentIndexPayload = {
userId,
documentId,
enqueuedAt: new Date().toISOString(),
};
try {
const jobId = await boss.send(DOCUMENT_INDEX_QUEUE, payload, {
retryLimit: 2,
retryDelay: 30,
retryBackoff: true,
singletonKey: `document-index|${documentId}`,
});
return { enqueued: Boolean(jobId) };
} catch (err) {
annotate({
action: { name: "documents.autoIndex.enqueueFailed" },
meta: { documentId, reason: err instanceof Error ? err.name : "unknown" },
});
return { enqueued: false };
}
}
@@ -75,6 +75,12 @@ import {
runContentIndexBackfillForUser,
type ContentIndexBackfillPayload,
} from "@/lib/jobs/document-content-index-backfill";
import {
DOCUMENT_INDEX_QUEUE,
DOCUMENT_INDEX_CONCURRENCY,
runDocumentIndex,
type DocumentIndexPayload,
} from "@/lib/jobs/document-index";
import {
ENCRYPTION_KEY_ROTATE_QUEUE,
ENCRYPTION_KEY_ROTATE_CONCURRENCY,
@@ -310,6 +316,11 @@ const allQueues = [
// Without this entry pg-boss never provisions the queue and the trigger
// silently no-ops.
CONTENT_INDEX_BACKFILL_QUEUE,
// Document AI — automatic per-document content indexing, enqueued on upload.
// Provider-first (vision) when configured + consented, else local text-layer
// extraction. Without this entry pg-boss never provisions the queue and every
// upload enqueue silently no-ops.
DOCUMENT_INDEX_QUEUE,
];
const schedules: ScheduleEntry[] = [
@@ -682,6 +693,32 @@ export async function registerMaintenanceQueues(
},
);
// Document AI — automatic per-document content indexing. Enqueued on upload
// (one job per stored document); provider-first (vision) when configured +
// consented, else local text-layer extraction. Serial concurrency so the
// provider calls + PDF parse never crowd the request pool.
await boss.work<DocumentIndexPayload>(
DOCUMENT_INDEX_QUEUE,
{ localConcurrency: DOCUMENT_INDEX_CONCURRENCY },
async (jobs) => {
for (const job of jobs) {
const { userId, documentId } = job.data;
if (!userId || !documentId) continue;
try {
await runDocumentIndex(job.data);
} catch (err) {
recordError();
workerLog(
"error",
`[document-index] user=${userId} document=${documentId} failed`,
err,
);
throw err;
}
}
},
);
// v1.25 (W-ENV) — nightly environment fetch. The daily discovery tick (empty
// payload) fans out one per-user job per opted-in account; the queue also
// serves the on-demand backfill payloads from the settings surface. Serial
+4 -1
View File
@@ -108,13 +108,16 @@ const inboundDocument = z
conditionLinks: z.array(conditionLink),
servingClass: z.enum(["inline", "attachment"]),
hasContentIndex: z.boolean(),
contentIndexSource: z
.enum(["vision", "text-ocr", "local-pdf", "local-ocr"])
.nullable(),
createdAt: z.string(),
updatedAt: z.string(),
})
.meta({
id: "InboundDocument",
description:
"A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (drives the re-index affordance). Treat an unknown `kind` value as OTHER when decoding.",
"A stored document. The raw bytes are stored encrypted at rest and never returned; this is the metadata + staging summary. `status` is STORED for a freshly uploaded file (no extraction run). `title` is the user label (plaintext); `documentDate` is the user filing date; `reportDate` is the model-transcribed date (null until extraction runs). `servingClass` says how `/original` delivers the file — render inline (`inline`) or download-only (`attachment`); render/download by it, never by MIME guess. `hasContentIndex` is true when the document has a content-search index (auto-indexed on upload); `contentIndexSource` says how that index was produced — `vision` means an AI provider read the original, the other values are provider-free local extractions, and it is null when `hasContentIndex` is false. Treat an unknown `kind` value as OTHER when decoding.",
});
const inboundDocumentDetail = inboundDocument
+48 -2
View File
@@ -59,6 +59,45 @@ export const INBOUND_DOCUMENT_KINDS = [
] as const;
export type InboundDocumentKindValue = (typeof INBOUND_DOCUMENT_KINDS)[number];
/**
* How a document's content index was produced (mirrors the server `source`
* column). `vision` means an AI provider read the original (the AI-first path);
* every other value is a provider-free local extraction. The UI reads it to
* tell an AI-read document from a locally-indexed one — the former already had
* the richer read, the latter can still be offered "Read with AI".
*/
export const DOCUMENT_CONTENT_INDEX_SOURCES = [
"vision",
"text-ocr",
"local-pdf",
"local-ocr",
] as const;
export type DocumentContentIndexSourceValue =
(typeof DOCUMENT_CONTENT_INDEX_SOURCES)[number];
/** True when the index was produced by an AI provider reading the original. */
export function isAiReadSource(
source: DocumentContentIndexSourceValue | string | null,
): boolean {
return source === "vision";
}
/**
* Narrow the free-String DB `source` column to the DTO union, or `null` for an
* absent / unrecognised value. The column is a free String server-side (adding
* a source needs no migration) so the read path guards it before it hits the
* contract.
*/
export function toContentIndexSource(
source: string | null | undefined,
): DocumentContentIndexSourceValue | null {
return DOCUMENT_CONTENT_INDEX_SOURCES.includes(
source as DocumentContentIndexSourceValue,
)
? (source as DocumentContentIndexSourceValue)
: null;
}
/** The document lifecycle states. STORED is the library default. */
export const INBOUND_DOCUMENT_STATUSES = [
"STORED",
@@ -230,10 +269,17 @@ export interface InboundDocumentDto {
servingClass: "inline" | "attachment";
/**
* Whether the document has a content-search index (encrypted extracted text +
* blind token array). Drives the "index for search" / re-index affordances;
* `false` until the document is indexed.
* blind token array). `false` until the document is indexed (auto-indexed on
* upload; the UI reads this for the searchable status, not as a to-do).
*/
hasContentIndex: boolean;
/**
* How that index was produced — `vision` (an AI provider read the original)
* vs a local extraction (`local-pdf` / `text-ocr` / `local-ocr`), or `null`
* when `hasContentIndex` is false. Lets the UI tell an AI-read document from
* a locally-indexed one and offer a richer "Read with AI" pass on the latter.
*/
contentIndexSource: DocumentContentIndexSourceValue | null;
createdAt: string;
updatedAt: string;
}
@@ -97,6 +97,10 @@ async function routes() {
r: Request,
c: RouteCtx,
) => Promise<Response>,
patchById: byId.PATCH as unknown as (
r: Request,
c: RouteCtx,
) => Promise<Response>,
postIndex: index.POST as unknown as (
r: Request,
c: RouteCtx,
@@ -191,6 +195,52 @@ describe("document vault — content index (text mode)", () => {
});
});
it("threads contentIndexSource through list + detail and preserves it across a metadata PATCH", async () => {
await seedVaultUser("vault-provenance");
const { post, get, getById, patchById, postIndex } = await routes();
const up = await post(uploadRequest(PNG_1X1, "scan.png", "Vorher"));
const { data: doc } = await up.json();
await postIndex(
textIndexRequest(doc.id, "kreatinin natrium kalium werte"),
ctx(doc.id),
);
// List surfaces the real provenance (text-ocr here — no provider).
const listed = await get(
new Request("http://localhost/api/documents/inbound"),
);
const listRow = (await listed.json()).data.documents.find(
(d: { id: string }) => d.id === doc.id,
);
expect(listRow.hasContentIndex).toBe(true);
expect(listRow.contentIndexSource).toBe("text-ocr");
// Detail agrees.
const detail = await getById(
new Request(`http://localhost/api/documents/inbound/${doc.id}`),
ctx(doc.id),
);
expect((await detail.json()).data.contentIndexSource).toBe("text-ocr");
// Regression (the UI wave flagged this): a metadata PATCH must return the
// index provenance from the freshly-read index row, not a stale null.
const patched = await patchById(
new Request(`http://localhost/api/documents/inbound/${doc.id}`, {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify({ title: "Nachher" }),
}),
ctx(doc.id),
);
expect(patched.status).toBe(200);
const patchedBody = (await patched.json()).data;
expect(patchedBody.title).toBe("Nachher");
expect(patchedBody.hasContentIndex).toBe(true);
expect(patchedBody.contentIndexSource).toBe("text-ocr");
});
it("re-indexing the same document is idempotent (upsert in place)", async () => {
await seedVaultUser("vault-reindex");
const { post, postIndex } = await routes();