From 2710dfc04192a92c65540ad2237d9e6634aaf8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 09:03:58 +0200 Subject: [PATCH] feat(insights): add an encrypted table for status notes Status notes (the per-metric text a model writes) move out of audit_logs into InsightStatusCache: one row per user, metric and locale, the note encrypted at rest, upserted in place, with the input and snapshot fingerprints and a separate negative-cache window. Migration 0344 creates the table and deletes every audit_logs row whose action matches insights.%-status.%. The notes are a regenerable cache and the old rows are plaintext, so nothing is copied; the first nightly run after the upgrade writes them again. Take a backup before applying. The table is registered for key rotation, excluded from backups as a regenerable cache, and deleted by Delete All Data. Refs #975 --- .../0344_insight_status_cache/migration.sql | 44 ++++++ prisma/schema.prisma | 52 ++++++++ scripts/rotate-encryption-key.ts | 19 +++ src/lib/crypto/encrypted-columns.ts | 6 + src/lib/data-wipe/wipe-plan.ts | 1 + src/lib/export/backup-plan.ts | 2 + .../insight-status-cache-migration.test.ts | 125 ++++++++++++++++++ 7 files changed, 249 insertions(+) create mode 100644 prisma/migrations/0344_insight_status_cache/migration.sql create mode 100644 tests/integration/insight-status-cache-migration.test.ts diff --git a/prisma/migrations/0344_insight_status_cache/migration.sql b/prisma/migrations/0344_insight_status_cache/migration.sql new file mode 100644 index 000000000..9571572c1 --- /dev/null +++ b/prisma/migrations/0344_insight_status_cache/migration.sql @@ -0,0 +1,44 @@ +-- Status notes move out of the audit log. +-- +-- The per-metric status notes used to be appended to `audit_logs` as plaintext +-- JSON, one row per metric per day, next to the security trail. They now live +-- in their own table, encrypted at rest, one row per (user, metric, locale), +-- upserted in place. +-- +-- 1. The new table. +-- CreateTable +CREATE TABLE "insight_status_caches" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "metric" TEXT NOT NULL, + "locale" TEXT NOT NULL, + "text_encrypted" BYTEA, + "items_encrypted" BYTEA, + "input_hash" TEXT, + "snapshot_hash" TEXT, + "date_key" TEXT NOT NULL, + "generated_at" TIMESTAMP(3), + "retry_at" TIMESTAMP(3), + "negative_reason" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "insight_status_caches_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "insight_status_caches_user_id_metric_locale_key" ON "insight_status_caches"("user_id", "metric", "locale"); + +-- AddForeignKey +ALTER TABLE "insight_status_caches" ADD CONSTRAINT "insight_status_caches_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- 2. The old rows go. DESTRUCTIVE by design: the notes are a regenerable +-- cache, nothing reads them from `audit_logs` any more, and leaving them +-- would keep plaintext health text in a table documented as holding none. +-- Nothing is copied across, because the old rows are plaintext and a +-- migration cannot encrypt; the first nightly run after the upgrade writes +-- every note again. Take a database backup before applying, as for 0339. +-- Matches exactly the cache keys `insights.-status.` (the +-- seven specialised scopes and the `metric:` / `derived-score:` / +-- `biomarker:` generic ones); no security-trail action has this shape. +DELETE FROM "audit_logs" WHERE "action" LIKE 'insights.%-status.%'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 1745fce4b..66075b211 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -860,6 +860,9 @@ model User { // v1.11.0 W3 — one durable period-narrative row per (period, locale). // Delete/regenerate-clean; the generated prose is AES-256-GCM at rest. insightNarratives InsightNarrative[] + // v1.39.0 — the per-metric status notes, one encrypted row per + // (metric, locale). A regenerable cache; it used to live in `audit_logs`. + insightStatusCaches InsightStatusCache[] // v1.31.0 — the data-arrival spine's durable per-day reaction marker. One // row per (kind, local day); the unique index IS the once-per-kind-per-day // claim every reaction surface reads. Retained 14 days. @@ -6722,6 +6725,55 @@ model InsightNarrative { @@map("insight_narratives") } +// v1.39.0 — the per-metric status notes ("Status notes"): what a model wrote +// about one metric for one person in one language, and the fingerprints that +// let the next run skip the model when nothing changed. +// +// One row per (user, metric, locale), upserted in place. Until v1.39.0 these +// notes were appended to `audit_logs` as plaintext JSON, one row per metric per +// day, mixed with the security trail; migration 0344 dropped those rows, so the +// first night after the upgrade regenerates every note once. +// +// A regenerable cache, not a record: excluded from backups, deleted by "Delete +// All Data", purged when the person withdraws AI consent, and never served +// while the `statusText` capability is unavailable. +model InsightStatusCache { + id String @id @default(cuid()) + userId String @map("user_id") + /// The status scope: `weight`, `blood-pressure`, `general`, …, or a prefixed + /// generic scope (`metric:`, `derived-score:`, `biomarker:`). + metric String + /// The UI locale the note was written in. + locale String + /// AES-256-GCM ciphertext of the note. Null while the row only carries a + /// negative-cache window (a generation that stalled before any note existed). + textEncrypted Bytes? @map("text_encrypted") + /// AES-256-GCM ciphertext of per-item notes as JSON, for a card that carries + /// them (medication compliance: one line per medication). Null otherwise. + itemsEncrypted Bytes? @map("items_encrypted") + /// Cheap fingerprint of the salient inputs (count + newest reading). + inputHash String? @map("input_hash") + /// Fingerprint of the full data snapshot the note was written from. + snapshotHash String? @map("snapshot_hash") + /// The person's local YYYY-MM-DD the note counts as current for. + dateKey String @map("date_key") + /// When the note was written, or last re-dated by an unchanged-data run. + generatedAt DateTime? @map("generated_at") + /// Negative cache: while in the future, a read does not enqueue another + /// generation. Cleared when a note is written. + retryAt DateTime? @map("retry_at") + /// Why the last generation served the deterministic line instead of a note: + /// `timeout` | `error` | `screened`. Set together with `retryAt`. + negativeReason String? @map("negative_reason") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, metric, locale]) + @@map("insight_status_caches") +} + // ─── Data-arrival spine (v1.31.0) ───────────────────────────── // // One row per (user, arrival kind, local day). The row is written by the diff --git a/scripts/rotate-encryption-key.ts b/scripts/rotate-encryption-key.ts index ff2367ff7..1546d0e90 100644 --- a/scripts/rotate-encryption-key.ts +++ b/scripts/rotate-encryption-key.ts @@ -454,6 +454,25 @@ async function main() { ), ); + // ───── v1.39.0 InsightStatusCache."textEncrypted" / "itemsEncrypted" ───── + // The per-metric status notes. Both nullable (a row may carry only a + // negative-cache window; only one card stores per-item lines), and + // `rotateBytesColumn` skips a NULL payload. + results.push( + await rotateBytesColumn( + "InsightStatusCache", + "textEncrypted", + prisma.insightStatusCache, + ), + ); + results.push( + await rotateBytesColumn( + "InsightStatusCache", + "itemsEncrypted", + prisma.insightStatusCache, + ), + ); + // ───── v1.31.0 ArrivalReaction."lineEncrypted" (Bytes, nullable) ───── // The data-arrival spine's generated reaction line. Nullable by design — a // provider-less install writes markers with no line at all — and diff --git a/src/lib/crypto/encrypted-columns.ts b/src/lib/crypto/encrypted-columns.ts index 1133fcf8b..c9ce41df9 100644 --- a/src/lib/crypto/encrypted-columns.ts +++ b/src/lib/crypto/encrypted-columns.ts @@ -232,6 +232,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [ // ───── Insight narratives (Bytes column) ───── { model: "InsightNarrative", field: "encryptedContent", kind: "bytes" }, + // ───── v1.39.0 status notes (Bytes columns) ───── + // Both nullable: a row can carry only a negative-cache window, and only the + // medication-compliance card stores per-item lines. + { model: "InsightStatusCache", field: "textEncrypted", kind: "bytes" }, + { model: "InsightStatusCache", field: "itemsEncrypted", kind: "bytes" }, + // ───── v1.31.0 data-arrival spine (Bytes column) ───── // Nullable: the reaction marker is written on every salient arrival, but the // generated line only exists where a provider was reachable and in budget. diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 015bdb295..27209034d 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -157,6 +157,7 @@ export const WIPE_MODELS = [ "CoachReminder", "CoachUsage", "InsightNarrative", + "InsightStatusCache", "ProviderHealth", // ── Sharing ───────────────────────────────────────────────────────────── diff --git a/src/lib/export/backup-plan.ts b/src/lib/export/backup-plan.ts index e650fb705..a9c0c5daa 100644 --- a/src/lib/export/backup-plan.ts +++ b/src/lib/export/backup-plan.ts @@ -582,6 +582,8 @@ export const DERIVED_MODELS: Readonly> = { "A concurrency claim guarding the generator above. Meaningless outside the run that took it.", InsightNarrative: "Generated prose over measurements. Same reasoning as WorkoutInsight — the record restores, the essay about it does not.", + InsightStatusCache: + "Regenerable cache: the per-metric status notes a model wrote over measurements, rewritten by the nightly run. Same reasoning as InsightNarrative.", DocumentContentIndex: "A search index over `InboundDocument` text; rebuilt by the indexer.", DocumentThumbnail: diff --git a/tests/integration/insight-status-cache-migration.test.ts b/tests/integration/insight-status-cache-migration.test.ts new file mode 100644 index 000000000..9bb2e46fd --- /dev/null +++ b/tests/integration/insight-status-cache-migration.test.ts @@ -0,0 +1,125 @@ +/** + * Migration 0344: status notes move out of the audit log. + * + * The container this suite runs on has every migration applied already, so + * the new table is asserted as it landed (columns, the one-row-per-note + * unique key, the cascade from the user), and the destructive data step is + * pulled out of the `.sql` file and run again against rows seeded the way an + * upgrading instance holds them: the old status rows go, and every other + * audit row stays, including the ones whose action merely starts with + * `insights.`. + * + * The step runs twice; the second run must change nothing. + */ +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const MIGRATION = readFileSync( + path.join( + process.cwd(), + "prisma", + "migrations", + "0344_insight_status_cache", + "migration.sql", + ), + "utf8", +); + +/** Every `DELETE … ;` statement in the migration, comments stripped. */ +function dataSteps(): string[] { + const sql = MIGRATION.split("\n") + .filter((line) => !line.trimStart().startsWith("--")) + .join("\n"); + const steps = [...sql.matchAll(/DELETE\s+FROM\s+"[a-z_]+"[\s\S]*?;/gi)].map( + (m) => m[0], + ); + // The data step is the point; an empty match is a failure, not a pass. + expect(steps.length).toBe(1); + return steps; +} + +async function runDataSteps() { + const prisma = getPrismaClient(); + for (const step of dataSteps()) await prisma.$executeRawUnsafe(step); +} + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +describe("the new table", () => { + it("keys one note per (user, metric, locale) and cascades from the user", async () => { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username: "mig-0344", email: "mig-0344@example.test" }, + }); + await prisma.insightStatusCache.create({ + data: { + userId: user.id, + metric: "weight", + locale: "en", + dateKey: "2026-09-20", + textEncrypted: Buffer.from("ciphertext-stand-in", "utf8"), + }, + }); + await expect( + prisma.insightStatusCache.create({ + data: { + userId: user.id, + metric: "weight", + locale: "en", + dateKey: "2026-09-21", + }, + }), + ).rejects.toThrow(); + + await prisma.user.delete({ where: { id: user.id } }); + expect(await prisma.insightStatusCache.count()).toBe(0); + }); +}); + +describe("the audit-log purge", () => { + it("deletes every status note row and nothing else", async () => { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username: "mig-0344-audit", email: "mig-0344-a@example.test" }, + }); + const statusActions = [ + "insights.weight-status.en", + "insights.blood-pressure-status.de", + "insights.metric:RESTING_HEART_RATE-status.fr", + "insights.derived-score:READINESS-status.en", + "insights.biomarker:ldl-status.it", + "insights.medication-compliance-status.pl", + ]; + const kept = [ + "insights.generate", + "insights.coach.fence_drift", + "insights.recommendation.feedback", + "auth.login", + "consent.ai.revoke", + ]; + for (const action of [...statusActions, ...kept]) { + await prisma.auditLog.create({ + data: { + userId: user.id, + action, + details: JSON.stringify({ text: "plaintext" }), + }, + }); + } + + await runDataSteps(); + await runDataSteps(); + + const remaining = await prisma.auditLog.findMany({ + select: { action: true }, + orderBy: { action: "asc" }, + }); + expect(remaining.map((r) => r.action)).toEqual([...kept].sort()); + }); +});