mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 22:02:16 +00:00
feat(documents): add DocumentContentIndex table for blind content search
P2-1-T1. Migration 0228 adds the 1:1 document_content_index sibling of inbound_documents: AES-256-GCM ciphertext of the extracted text plus a deduped HMAC-SHA256 token array (search_tokens) with a GIN index for the array-overlap match the list search will union with title/filename ILIKE. Both foreign keys cascade. Nothing readable sits at rest; the index subkey is HKDF-derived at runtime, so tokens are opaque without the server key.
This commit is contained in:
1 parent
1474c87011
commit
7c5d1a264e
2 files changed
+107
No files matched your search
@@ -0,0 +1,56 @@
|
||||
-- Document vault P2: the blind content-search index.
|
||||
--
|
||||
-- `document_content_index` — a 1:1 sibling of `inbound_documents` that makes
|
||||
-- a document searchable INSIDE its body without ever storing the body in the
|
||||
-- clear. Two artefacts per row:
|
||||
-- - `text_encrypted` — AES-256-GCM ciphertext of the normalised extracted
|
||||
-- text (the `encrypt()`-string-as-UTF-8 BYTEA shape every other PHI
|
||||
-- column uses). Recoverable server-side so key rotation can re-tokenise.
|
||||
-- - `search_tokens` — deduped HMAC-SHA256 (truncated hex) of the normalised
|
||||
-- tokens, under an HKDF-derived index subkey. One-way / opaque at rest.
|
||||
-- A GIN index over `search_tokens` accelerates the array-overlap (`&&`)
|
||||
-- predicate the list search unions with the title/filename ILIKE. Prisma
|
||||
-- cannot express a GIN index, so it lives here as raw SQL.
|
||||
--
|
||||
-- Both foreign keys cascade: a deleted document (or user) takes its index with
|
||||
-- it. Additive; applies clean on a 0227 database and on a fresh database.
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "document_content_index" (
|
||||
"id" TEXT NOT NULL,
|
||||
"document_id" TEXT NOT NULL,
|
||||
"user_id" TEXT NOT NULL,
|
||||
"text_encrypted" BYTEA NOT NULL,
|
||||
"search_tokens" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
|
||||
"source" TEXT NOT NULL,
|
||||
"provider_type" TEXT,
|
||||
"tokenizer_version" TEXT NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "document_content_index_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- 1:1 with the document.
|
||||
CREATE UNIQUE INDEX "document_content_index_document_id_key"
|
||||
ON "document_content_index" ("document_id");
|
||||
|
||||
-- Owner-scoped queries (the usage gauge + the backfill discovery).
|
||||
CREATE INDEX "document_content_index_user_id_idx"
|
||||
ON "document_content_index" ("user_id");
|
||||
|
||||
-- Blind-token search: GIN over the token array for the array-overlap (`&&`)
|
||||
-- match the list search runs. Documented in schema.prisma — Prisma cannot
|
||||
-- express a GIN index, so it lives here as raw SQL.
|
||||
CREATE INDEX "document_content_index_search_tokens_gin"
|
||||
ON "document_content_index" USING GIN ("search_tokens");
|
||||
|
||||
ALTER TABLE "document_content_index"
|
||||
ADD CONSTRAINT "document_content_index_document_id_fkey"
|
||||
FOREIGN KEY ("document_id") REFERENCES "inbound_documents" ("id")
|
||||
ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "document_content_index"
|
||||
ADD CONSTRAINT "document_content_index_user_id_fkey"
|
||||
FOREIGN KEY ("user_id") REFERENCES "users" ("id")
|
||||
ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -784,6 +784,7 @@ model User {
|
||||
inboundDocuments InboundDocument[]
|
||||
extractedFacts ExtractedFact[]
|
||||
documentConditionLinks DocumentConditionLink[]
|
||||
documentContentIndexes DocumentContentIndex[]
|
||||
// v1.19.0 — full Withings ECG waveform recordings (encrypted at rest).
|
||||
// The paired AFib verdict still lands as an IRREGULAR_RHYTHM_NOTIFICATION
|
||||
// Measurement EVENT row (v1.18.11); this holds the raw signal samples.
|
||||
@@ -5815,6 +5816,9 @@ model InboundDocument {
|
||||
|
||||
facts ExtractedFact[]
|
||||
conditionLinks DocumentConditionLink[]
|
||||
/// 1:1 blind content-search index (encrypted extracted text + HMAC token
|
||||
/// array). Absent until the document is indexed; cascades on delete.
|
||||
contentIndex DocumentContentIndex?
|
||||
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
@@ -5900,3 +5904,50 @@ model DocumentConditionLink {
|
||||
@@index([userId])
|
||||
@@map("document_condition_links")
|
||||
}
|
||||
|
||||
/// v1.27.22 (Document vault P2) — the blind content-search index for one stored
|
||||
/// document (1:1). Content search matches INSIDE a document without ever
|
||||
/// storing its body in the clear: the extracted text is kept AES-256-GCM at
|
||||
/// rest (`textEncrypted`, the same `*Encrypted` Bytes convention as every other
|
||||
/// PHI column) and, separately, tokenised into deterministic HMAC-SHA256 hashes
|
||||
/// (`searchTokens`) under an HKDF-derived index subkey. A query is tokenised +
|
||||
/// hashed the same way and matched with a GIN-accelerated array-overlap — so
|
||||
/// the corpus is searchable at SQL speed while nothing readable sits at rest.
|
||||
/// Honest limit: the hash index does whole-token equality only (no substring /
|
||||
/// prefix / fuzzy); the vault list still ILIKEs the short title/filename for
|
||||
/// substring and unions the two result sets. Both artefacts cascade with the
|
||||
/// document. `tokenizerVersion` lets a future tokenizer change be re-indexed
|
||||
/// unambiguously; the index subkey is HKDF-derived at runtime from the active
|
||||
/// versioned key (no schema column), and key rotation re-tokenises from the
|
||||
/// decrypted `textEncrypted`.
|
||||
model DocumentContentIndex {
|
||||
id String @id @default(cuid())
|
||||
documentId String @unique @map("document_id")
|
||||
userId String @map("user_id")
|
||||
/// AES-256-GCM ciphertext of the normalised extracted text (capped, first
|
||||
/// ~64 KiB), stored as the `encrypt()`-string-as-UTF-8 Bytes shape shared
|
||||
/// with the Coach columns. The plaintext is recoverable server-side so
|
||||
/// rotation can re-tokenise; the list query never selects it.
|
||||
textEncrypted Bytes @map("text_encrypted")
|
||||
/// Deduped HMAC-SHA256 (truncated hex) of the normalised tokens — the blind
|
||||
/// index. One-way: opaque without the server-held index subkey. GIN-indexed
|
||||
/// (raw SQL in the migration; Prisma cannot express a GIN index).
|
||||
searchTokens String[] @default([]) @map("search_tokens")
|
||||
/// Provenance of the indexed text: "vision" (provider transcription of the
|
||||
/// stored original) or "text-ocr" (browser-OCR text posted by the client).
|
||||
source String
|
||||
/// Provider type that produced the text (diagnostic only), or null on the
|
||||
/// text-ocr path.
|
||||
providerType String? @map("provider_type")
|
||||
/// Tokeniser algorithm version — bump to force a re-index on a scheme change.
|
||||
tokenizerVersion String @map("tokenizer_version")
|
||||
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
|
||||
document InboundDocument @relation(fields: [documentId], references: [id], onDelete: Cascade)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([userId])
|
||||
@@map("document_content_index")
|
||||
}
|
||||
Reference in new issue
Block a user