mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 07:42:23 +00:00
feat(documents): read documents and lab scans in the background
"Read with AI" and the lab scan used to run their model calls inside the
request. At the default response time a read of three calls outlasted the
100 s a proxy in front of the server commonly allows, and the person saw a
cut connection for a read the server was still doing.
Both now run as a background run (`DocumentAiRun`, migration 0366):
- `POST /api/documents/inbound/{id}/index` with `Prefer: respond-async`
answers 202 with a run id once every quick refusal has passed. Without
the header it answers exactly as before, which the iPhone app relies on.
A second request while a read of the same document is queued or running
attaches to that run.
- `POST /api/labs/ocr/extract` always answers 202. Only the web client
calls it.
- `GET /api/ai-runs/{id}` serves the run: the same body the synchronous
route answers with, or the failure code and status it would have given.
Scoped to the caller; another account's run is a 404.
The worker (`document-ai-run`, a long queue with a one-hour expiry, one
job per run under `lockedPass`) claims the run, asks the capability again
for the record and re-checks the wire for the provider it picks, sets the
run's deadline from the person's AI response time and refuses a read that
cannot fit the job's expiry. Sync route and worker run one body per kind,
so the two paths cannot drift. Budget is reserved at enqueue and settled
once by whoever ends the run.
Input and result are sealed with labelled AES-256-GCM, registered for key
rotation; the input is dropped when the run ends. A five-minute reaper
fails a run no worker took within fifteen minutes (and hands its
reservation back), fails a run past its deadline, and deletes a finished
run after an hour. Runs are outside backups and inside the data wipe.
The web client polls through the query-key factory and says, calmly, that
the read continues in the background, and when no worker has picked it up.
This commit is contained in:
1 parent
556bd6d6cd
commit
0acf3da575
49 files changed
+4533
-870
No files matched your search
@@ -0,0 +1,54 @@
|
||||
-- Document AI as background runs. A read that used to run inside the request
|
||||
-- now runs in the worker; the request answers 202 with this row's id and the
|
||||
-- client polls it. The row is a ticket: the input is cleared when the run
|
||||
-- finishes and the row is deleted an hour later. Idempotent, so a re-run of a
|
||||
-- half-applied migration converges.
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE "document_ai_run_kind" AS ENUM ('DOCUMENT_INDEX', 'LABS_OCR_EXTRACT');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
CREATE TYPE "document_ai_run_status" AS ENUM ('QUEUED', 'RUNNING', 'SUCCEEDED', 'FAILED');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "document_ai_runs" (
|
||||
"id" TEXT NOT NULL,
|
||||
"user_id" TEXT NOT NULL,
|
||||
"kind" "document_ai_run_kind" NOT NULL,
|
||||
"document_id" TEXT,
|
||||
"status" "document_ai_run_status" NOT NULL DEFAULT 'QUEUED',
|
||||
"params_json" JSONB NOT NULL,
|
||||
"input_encrypted" BYTEA,
|
||||
"result_encrypted" BYTEA,
|
||||
"error_code" TEXT,
|
||||
"error_status" INTEGER,
|
||||
"error_message" TEXT,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"started_at" TIMESTAMP(3),
|
||||
"finished_at" TIMESTAMP(3),
|
||||
"expires_at" TIMESTAMP(3) NOT NULL,
|
||||
CONSTRAINT "document_ai_runs_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "document_ai_runs_user_id_created_at_idx"
|
||||
ON "document_ai_runs" ("user_id", "created_at");
|
||||
CREATE INDEX IF NOT EXISTS "document_ai_runs_document_id_kind_status_idx"
|
||||
ON "document_ai_runs" ("document_id", "kind", "status");
|
||||
CREATE INDEX IF NOT EXISTS "document_ai_runs_status_expires_at_idx"
|
||||
ON "document_ai_runs" ("status", "expires_at");
|
||||
|
||||
DO $$ BEGIN
|
||||
ALTER TABLE "document_ai_runs"
|
||||
ADD CONSTRAINT "document_ai_runs_user_id_fkey"
|
||||
FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN
|
||||
ALTER TABLE "document_ai_runs"
|
||||
ADD CONSTRAINT "document_ai_runs_document_id_fkey"
|
||||
FOREIGN KEY ("document_id") REFERENCES "inbound_documents"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
@@ -942,6 +942,7 @@ model User {
|
||||
// v1.39.3 — the person's own connection to a Paperless-ngx or Papra
|
||||
// instance for the document picker. See DocumentSourceConnection.
|
||||
documentSourceConnections DocumentSourceConnection[]
|
||||
documentAiRuns DocumentAiRun[]
|
||||
extractedFacts ExtractedFact[]
|
||||
documentConditionLinks DocumentConditionLink[]
|
||||
documentContentIndexes DocumentContentIndex[]
|
||||
@@ -8190,6 +8191,8 @@ model InboundDocument {
|
||||
shareLinks ClinicianShareLinkDocument[]
|
||||
/// Further import source keys that resolved to this document by content.
|
||||
sourceAliases DocumentSourceAlias[]
|
||||
/// Background AI reads of this document (v1.40). Transient; cascades.
|
||||
aiRuns DocumentAiRun[]
|
||||
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
updatedAt DateTime @updatedAt @map("updated_at")
|
||||
@@ -8987,3 +8990,75 @@ model OnboardingRecord {
|
||||
|
||||
@@map("onboarding_records")
|
||||
}
|
||||
|
||||
/// v1.40 — what a background document AI run is doing.
|
||||
enum DocumentAiRunKind {
|
||||
/// "Read with AI": transcribe the stored original into the content index,
|
||||
/// then continue into lab staging.
|
||||
DOCUMENT_INDEX
|
||||
/// A lab report scan (photo, PDF or browser-read text) into proposed rows
|
||||
/// for the review screen. Nothing is written until the person commits.
|
||||
LABS_OCR_EXTRACT
|
||||
|
||||
@@map("document_ai_run_kind")
|
||||
}
|
||||
|
||||
/// v1.40 — where a background document AI run stands.
|
||||
enum DocumentAiRunStatus {
|
||||
QUEUED
|
||||
RUNNING
|
||||
SUCCEEDED
|
||||
FAILED
|
||||
|
||||
@@map("document_ai_run_status")
|
||||
}
|
||||
|
||||
/// v1.40 — one model read that runs in the background instead of inside the
|
||||
/// request that asked for it, so a proxy that cuts long requests cannot cut
|
||||
/// the read. The request answers 202 with the run id; the client polls
|
||||
/// `GET /api/ai-runs/{id}`. A ticket, not a record: the input is dropped the
|
||||
/// moment the run finishes and the whole row is deleted an hour later
|
||||
/// (`document-ai-run-reaper`). Never in a backup.
|
||||
model DocumentAiRun {
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id")
|
||||
kind DocumentAiRunKind
|
||||
/// The document a document run reads. Null for a lab scan, whose input is
|
||||
/// the upload itself.
|
||||
documentId String? @map("document_id")
|
||||
status DocumentAiRunStatus @default(QUEUED)
|
||||
/// How to run it: the input mode, the upload's sniffed MIME, the budget
|
||||
/// reservation taken at enqueue. Never health data.
|
||||
paramsJson Json @map("params_json")
|
||||
/// AES-256-GCM (`encryptBytes`, labelled) of the input the worker needs and
|
||||
/// cannot read anywhere else: the uploaded scan, or the text the browser
|
||||
/// read off it. Null for a run that reads a stored document, and set to
|
||||
/// null when the run finishes.
|
||||
inputEncrypted Bytes? @map("input_encrypted")
|
||||
/// AES-256-GCM (`encryptBytes`, labelled) of the result as JSON. It holds
|
||||
/// model text and lab values. Served by the poll route until the row goes.
|
||||
resultEncrypted Bytes? @map("result_encrypted")
|
||||
/// The failure code a client branches on, the same code the synchronous
|
||||
/// route would have answered with.
|
||||
errorCode String? @map("error_code")
|
||||
/// The HTTP status the synchronous route would have answered with, so a
|
||||
/// client handles a failed run as it handled the failed request.
|
||||
errorStatus Int? @map("error_status")
|
||||
/// The failure in one sanitised sentence. Never upstream text.
|
||||
errorMessage String? @map("error_message")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
startedAt DateTime? @map("started_at")
|
||||
finishedAt DateTime? @map("finished_at")
|
||||
/// The next deadline the reaper acts on: queued runs fail as
|
||||
/// `aiRuns.workerUnavailable`, running ones as `aiRuns.timedOut`, finished
|
||||
/// ones are deleted.
|
||||
expiresAt DateTime @map("expires_at")
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
document InboundDocument? @relation(fields: [documentId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([userId, createdAt])
|
||||
@@index([documentId, kind, status])
|
||||
@@index([status, expiresAt])
|
||||
@@map("document_ai_runs")
|
||||
}
|
||||
Reference in new issue
Block a user