feat(insights): add an encrypted table for status notes

Status notes (the per-metric text a model writes) move out of audit_logs
into InsightStatusCache: one row per user, metric and locale, the note
encrypted at rest, upserted in place, with the input and snapshot
fingerprints and a separate negative-cache window.

Migration 0344 creates the table and deletes every audit_logs row whose
action matches insights.%-status.%. The notes are a regenerable cache and
the old rows are plaintext, so nothing is copied; the first nightly run
after the upgrade writes them again. Take a backup before applying.

The table is registered for key rotation, excluded from backups as a
regenerable cache, and deleted by Delete All Data.

Refs #975
This commit is contained in:
Marc-André Bombeck committed 2026-09-24 09:03:58 +02:00
1 parent f434ba7d29
commit 2710dfc041
7 files changed
+249

No files matched your search

@@ -0,0 +1,44 @@
-- Status notes move out of the audit log.
--
-- The per-metric status notes used to be appended to `audit_logs` as plaintext
-- JSON, one row per metric per day, next to the security trail. They now live
-- in their own table, encrypted at rest, one row per (user, metric, locale),
-- upserted in place.
--
-- 1. The new table.
-- CreateTable
CREATE TABLE "insight_status_caches" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"metric" TEXT NOT NULL,
"locale" TEXT NOT NULL,
"text_encrypted" BYTEA,
"items_encrypted" BYTEA,
"input_hash" TEXT,
"snapshot_hash" TEXT,
"date_key" TEXT NOT NULL,
"generated_at" TIMESTAMP(3),
"retry_at" TIMESTAMP(3),
"negative_reason" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "insight_status_caches_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "insight_status_caches_user_id_metric_locale_key" ON "insight_status_caches"("user_id", "metric", "locale");
-- AddForeignKey
ALTER TABLE "insight_status_caches" ADD CONSTRAINT "insight_status_caches_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- 2. The old rows go. DESTRUCTIVE by design: the notes are a regenerable
-- cache, nothing reads them from `audit_logs` any more, and leaving them
-- would keep plaintext health text in a table documented as holding none.
-- Nothing is copied across, because the old rows are plaintext and a
-- migration cannot encrypt; the first nightly run after the upgrade writes
-- every note again. Take a database backup before applying, as for 0339.
-- Matches exactly the cache keys `insights.<scope>-status.<locale>` (the
-- seven specialised scopes and the `metric:` / `derived-score:` /
-- `biomarker:` generic ones); no security-trail action has this shape.
DELETE FROM "audit_logs" WHERE "action" LIKE 'insights.%-status.%';
+52
View File
@@ -860,6 +860,9 @@ model User {
// v1.11.0 W3 — one durable period-narrative row per (period, locale).
// Delete/regenerate-clean; the generated prose is AES-256-GCM at rest.
insightNarratives InsightNarrative[]
// v1.39.0 — the per-metric status notes, one encrypted row per
// (metric, locale). A regenerable cache; it used to live in `audit_logs`.
insightStatusCaches InsightStatusCache[]
// v1.31.0 — the data-arrival spine's durable per-day reaction marker. One
// row per (kind, local day); the unique index IS the once-per-kind-per-day
// claim every reaction surface reads. Retained 14 days.
@@ -6722,6 +6725,55 @@ model InsightNarrative {
@@map("insight_narratives")
}
// v1.39.0 — the per-metric status notes ("Status notes"): what a model wrote
// about one metric for one person in one language, and the fingerprints that
// let the next run skip the model when nothing changed.
//
// One row per (user, metric, locale), upserted in place. Until v1.39.0 these
// notes were appended to `audit_logs` as plaintext JSON, one row per metric per
// day, mixed with the security trail; migration 0344 dropped those rows, so the
// first night after the upgrade regenerates every note once.
//
// A regenerable cache, not a record: excluded from backups, deleted by "Delete
// All Data", purged when the person withdraws AI consent, and never served
// while the `statusText` capability is unavailable.
model InsightStatusCache {
id String @id @default(cuid())
userId String @map("user_id")
/// The status scope: `weight`, `blood-pressure`, `general`, …, or a prefixed
/// generic scope (`metric:<ID>`, `derived-score:<ID>`, `biomarker:<id>`).
metric String
/// The UI locale the note was written in.
locale String
/// AES-256-GCM ciphertext of the note. Null while the row only carries a
/// negative-cache window (a generation that stalled before any note existed).
textEncrypted Bytes? @map("text_encrypted")
/// AES-256-GCM ciphertext of per-item notes as JSON, for a card that carries
/// them (medication compliance: one line per medication). Null otherwise.
itemsEncrypted Bytes? @map("items_encrypted")
/// Cheap fingerprint of the salient inputs (count + newest reading).
inputHash String? @map("input_hash")
/// Fingerprint of the full data snapshot the note was written from.
snapshotHash String? @map("snapshot_hash")
/// The person's local YYYY-MM-DD the note counts as current for.
dateKey String @map("date_key")
/// When the note was written, or last re-dated by an unchanged-data run.
generatedAt DateTime? @map("generated_at")
/// Negative cache: while in the future, a read does not enqueue another
/// generation. Cleared when a note is written.
retryAt DateTime? @map("retry_at")
/// Why the last generation served the deterministic line instead of a note:
/// `timeout` | `error` | `screened`. Set together with `retryAt`.
negativeReason String? @map("negative_reason")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([userId, metric, locale])
@@map("insight_status_caches")
}
// ─── Data-arrival spine (v1.31.0) ─────────────────────────────
//
// One row per (user, arrival kind, local day). The row is written by the
+19
View File
@@ -454,6 +454,25 @@ async function main() {
),
);
// ───── v1.39.0 InsightStatusCache."textEncrypted" / "itemsEncrypted" ─────
// The per-metric status notes. Both nullable (a row may carry only a
// negative-cache window; only one card stores per-item lines), and
// `rotateBytesColumn` skips a NULL payload.
results.push(
await rotateBytesColumn(
"InsightStatusCache",
"textEncrypted",
prisma.insightStatusCache,
),
);
results.push(
await rotateBytesColumn(
"InsightStatusCache",
"itemsEncrypted",
prisma.insightStatusCache,
),
);
// ───── v1.31.0 ArrivalReaction."lineEncrypted" (Bytes, nullable) ─────
// The data-arrival spine's generated reaction line. Nullable by design — a
// provider-less install writes markers with no line at all — and
+6
View File
@@ -232,6 +232,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
// ───── Insight narratives (Bytes column) ─────
{ model: "InsightNarrative", field: "encryptedContent", kind: "bytes" },
// ───── v1.39.0 status notes (Bytes columns) ─────
// Both nullable: a row can carry only a negative-cache window, and only the
// medication-compliance card stores per-item lines.
{ model: "InsightStatusCache", field: "textEncrypted", kind: "bytes" },
{ model: "InsightStatusCache", field: "itemsEncrypted", kind: "bytes" },
// ───── v1.31.0 data-arrival spine (Bytes column) ─────
// Nullable: the reaction marker is written on every salient arrival, but the
// generated line only exists where a provider was reachable and in budget.
+1
View File
@@ -157,6 +157,7 @@ export const WIPE_MODELS = [
"CoachReminder",
"CoachUsage",
"InsightNarrative",
"InsightStatusCache",
"ProviderHealth",
// ── Sharing ─────────────────────────────────────────────────────────────
+2
View File
@@ -582,6 +582,8 @@ export const DERIVED_MODELS: Readonly<Record<string, string>> = {
"A concurrency claim guarding the generator above. Meaningless outside the run that took it.",
InsightNarrative:
"Generated prose over measurements. Same reasoning as WorkoutInsight — the record restores, the essay about it does not.",
InsightStatusCache:
"Regenerable cache: the per-metric status notes a model wrote over measurements, rewritten by the nightly run. Same reasoning as InsightNarrative.",
DocumentContentIndex:
"A search index over `InboundDocument` text; rebuilt by the indexer.",
DocumentThumbnail:
@@ -0,0 +1,125 @@
/**
* Migration 0344: status notes move out of the audit log.
*
* The container this suite runs on has every migration applied already, so
* the new table is asserted as it landed (columns, the one-row-per-note
* unique key, the cascade from the user), and the destructive data step is
* pulled out of the `.sql` file and run again against rows seeded the way an
* upgrading instance holds them: the old status rows go, and every other
* audit row stays, including the ones whose action merely starts with
* `insights.`.
*
* The step runs twice; the second run must change nothing.
*/
import { readFileSync } from "node:fs";
import path from "node:path";
import { beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
const MIGRATION = readFileSync(
path.join(
process.cwd(),
"prisma",
"migrations",
"0344_insight_status_cache",
"migration.sql",
),
"utf8",
);
/** Every `DELETE … ;` statement in the migration, comments stripped. */
function dataSteps(): string[] {
const sql = MIGRATION.split("\n")
.filter((line) => !line.trimStart().startsWith("--"))
.join("\n");
const steps = [...sql.matchAll(/DELETE\s+FROM\s+"[a-z_]+"[\s\S]*?;/gi)].map(
(m) => m[0],
);
// The data step is the point; an empty match is a failure, not a pass.
expect(steps.length).toBe(1);
return steps;
}
async function runDataSteps() {
const prisma = getPrismaClient();
for (const step of dataSteps()) await prisma.$executeRawUnsafe(step);
}
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
});
describe("the new table", () => {
it("keys one note per (user, metric, locale) and cascades from the user", async () => {
const prisma = getPrismaClient();
const user = await prisma.user.create({
data: { username: "mig-0344", email: "mig-0344@example.test" },
});
await prisma.insightStatusCache.create({
data: {
userId: user.id,
metric: "weight",
locale: "en",
dateKey: "2026-09-20",
textEncrypted: Buffer.from("ciphertext-stand-in", "utf8"),
},
});
await expect(
prisma.insightStatusCache.create({
data: {
userId: user.id,
metric: "weight",
locale: "en",
dateKey: "2026-09-21",
},
}),
).rejects.toThrow();
await prisma.user.delete({ where: { id: user.id } });
expect(await prisma.insightStatusCache.count()).toBe(0);
});
});
describe("the audit-log purge", () => {
it("deletes every status note row and nothing else", async () => {
const prisma = getPrismaClient();
const user = await prisma.user.create({
data: { username: "mig-0344-audit", email: "mig-0344-a@example.test" },
});
const statusActions = [
"insights.weight-status.en",
"insights.blood-pressure-status.de",
"insights.metric:RESTING_HEART_RATE-status.fr",
"insights.derived-score:READINESS-status.en",
"insights.biomarker:ldl-status.it",
"insights.medication-compliance-status.pl",
];
const kept = [
"insights.generate",
"insights.coach.fence_drift",
"insights.recommendation.feedback",
"auth.login",
"consent.ai.revoke",
];
for (const action of [...statusActions, ...kept]) {
await prisma.auditLog.create({
data: {
userId: user.id,
action,
details: JSON.stringify({ text: "plaintext" }),
},
});
}
await runDataSteps();
await runDataSteps();
const remaining = await prisma.auditLog.findMany({
select: { action: true },
orderBy: { action: "asc" },
});
expect(remaining.map((r) => r.action)).toEqual([...kept].sort());
});
});