From 7c5d1a264e8b23e0fe0011d6f9a9aa5688f7b66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 7 Jul 2026 18:08:12 +0200 Subject: [PATCH] feat(documents): add DocumentContentIndex table for blind content search P2-1-T1. Migration 0228 adds the 1:1 document_content_index sibling of inbound_documents: AES-256-GCM ciphertext of the extracted text plus a deduped HMAC-SHA256 token array (search_tokens) with a GIN index for the array-overlap match the list search will union with title/filename ILIKE. Both foreign keys cascade. Nothing readable sits at rest; the index subkey is HKDF-derived at runtime, so tokens are opaque without the server key. --- .../0228_document_content_index/migration.sql | 56 +++++++++++++++++++ prisma/schema.prisma | 51 +++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 prisma/migrations/0228_document_content_index/migration.sql diff --git a/prisma/migrations/0228_document_content_index/migration.sql b/prisma/migrations/0228_document_content_index/migration.sql new file mode 100644 index 000000000..bf9cd1fd2 --- /dev/null +++ b/prisma/migrations/0228_document_content_index/migration.sql @@ -0,0 +1,56 @@ +-- Document vault P2: the blind content-search index. +-- +-- `document_content_index` — a 1:1 sibling of `inbound_documents` that makes +-- a document searchable INSIDE its body without ever storing the body in the +-- clear. Two artefacts per row: +-- - `text_encrypted` — AES-256-GCM ciphertext of the normalised extracted +-- text (the `encrypt()`-string-as-UTF-8 BYTEA shape every other PHI +-- column uses). Recoverable server-side so key rotation can re-tokenise. +-- - `search_tokens` — deduped HMAC-SHA256 (truncated hex) of the normalised +-- tokens, under an HKDF-derived index subkey. One-way / opaque at rest. +-- A GIN index over `search_tokens` accelerates the array-overlap (`&&`) +-- predicate the list search unions with the title/filename ILIKE. Prisma +-- cannot express a GIN index, so it lives here as raw SQL. +-- +-- Both foreign keys cascade: a deleted document (or user) takes its index with +-- it. Additive; applies clean on a 0227 database and on a fresh database. + +-- CreateTable +CREATE TABLE "document_content_index" ( + "id" TEXT NOT NULL, + "document_id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "text_encrypted" BYTEA NOT NULL, + "search_tokens" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + "source" TEXT NOT NULL, + "provider_type" TEXT, + "tokenizer_version" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "document_content_index_pkey" PRIMARY KEY ("id") +); + +-- 1:1 with the document. +CREATE UNIQUE INDEX "document_content_index_document_id_key" + ON "document_content_index" ("document_id"); + +-- Owner-scoped queries (the usage gauge + the backfill discovery). +CREATE INDEX "document_content_index_user_id_idx" + ON "document_content_index" ("user_id"); + +-- Blind-token search: GIN over the token array for the array-overlap (`&&`) +-- match the list search runs. Documented in schema.prisma — Prisma cannot +-- express a GIN index, so it lives here as raw SQL. +CREATE INDEX "document_content_index_search_tokens_gin" + ON "document_content_index" USING GIN ("search_tokens"); + +ALTER TABLE "document_content_index" + ADD CONSTRAINT "document_content_index_document_id_fkey" + FOREIGN KEY ("document_id") REFERENCES "inbound_documents" ("id") + ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE "document_content_index" + ADD CONSTRAINT "document_content_index_user_id_fkey" + FOREIGN KEY ("user_id") REFERENCES "users" ("id") + ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 62555c6c5..1b8b4529b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -784,6 +784,7 @@ model User { inboundDocuments InboundDocument[] extractedFacts ExtractedFact[] documentConditionLinks DocumentConditionLink[] + documentContentIndexes DocumentContentIndex[] // v1.19.0 — full Withings ECG waveform recordings (encrypted at rest). // The paired AFib verdict still lands as an IRREGULAR_RHYTHM_NOTIFICATION // Measurement EVENT row (v1.18.11); this holds the raw signal samples. @@ -5815,6 +5816,9 @@ model InboundDocument { facts ExtractedFact[] conditionLinks DocumentConditionLink[] + /// 1:1 blind content-search index (encrypted extracted text + HMAC token + /// array). Absent until the document is indexed; cascades on delete. + contentIndex DocumentContentIndex? createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") @@ -5900,3 +5904,50 @@ model DocumentConditionLink { @@index([userId]) @@map("document_condition_links") } + +/// v1.27.22 (Document vault P2) — the blind content-search index for one stored +/// document (1:1). Content search matches INSIDE a document without ever +/// storing its body in the clear: the extracted text is kept AES-256-GCM at +/// rest (`textEncrypted`, the same `*Encrypted` Bytes convention as every other +/// PHI column) and, separately, tokenised into deterministic HMAC-SHA256 hashes +/// (`searchTokens`) under an HKDF-derived index subkey. A query is tokenised + +/// hashed the same way and matched with a GIN-accelerated array-overlap — so +/// the corpus is searchable at SQL speed while nothing readable sits at rest. +/// Honest limit: the hash index does whole-token equality only (no substring / +/// prefix / fuzzy); the vault list still ILIKEs the short title/filename for +/// substring and unions the two result sets. Both artefacts cascade with the +/// document. `tokenizerVersion` lets a future tokenizer change be re-indexed +/// unambiguously; the index subkey is HKDF-derived at runtime from the active +/// versioned key (no schema column), and key rotation re-tokenises from the +/// decrypted `textEncrypted`. +model DocumentContentIndex { + id String @id @default(cuid()) + documentId String @unique @map("document_id") + userId String @map("user_id") + /// AES-256-GCM ciphertext of the normalised extracted text (capped, first + /// ~64 KiB), stored as the `encrypt()`-string-as-UTF-8 Bytes shape shared + /// with the Coach columns. The plaintext is recoverable server-side so + /// rotation can re-tokenise; the list query never selects it. + textEncrypted Bytes @map("text_encrypted") + /// Deduped HMAC-SHA256 (truncated hex) of the normalised tokens — the blind + /// index. One-way: opaque without the server-held index subkey. GIN-indexed + /// (raw SQL in the migration; Prisma cannot express a GIN index). + searchTokens String[] @default([]) @map("search_tokens") + /// Provenance of the indexed text: "vision" (provider transcription of the + /// stored original) or "text-ocr" (browser-OCR text posted by the client). + source String + /// Provider type that produced the text (diagnostic only), or null on the + /// text-ocr path. + providerType String? @map("provider_type") + /// Tokeniser algorithm version — bump to force a re-index on a scheme change. + tokenizerVersion String @map("tokenizer_version") + + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + document InboundDocument @relation(fields: [documentId], references: [id], onDelete: Cascade) + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@index([userId]) + @@map("document_content_index") +}