Merge branch 'fix/encrypt-free-text-and-local-ai' into release/v1.39.3

# Conflicts:
#	src/app/api/import/csv/route.ts
This commit is contained in:
Marc-André Bombeck committed 2026-09-26 21:50:05 +02:00
commit 66b1aef40c
69 files changed
+3625 -367

No files matched your search

+43 -8
View File
@@ -112,6 +112,8 @@ const COACH_PLAN_CUE = "if the evening reading is over 140";
const COACH_PLAN_ACTION = "then walk twenty minutes before dinner tomorrow";
const COACH_REMINDER_NOTE = "ask how the evening walks are going";
const COACH_SUMMARY = "earlier turns: weight trend and evening walks";
const METRIC_LEGACY_NOTE = "left hand, after the long ride";
const METRIC_SEALED_NOTE = "right hand, rested";
const COACH_USER_TURN = "my readings look higher this week, is that real?";
const COACH_ASSISTANT_TURN =
"the last seven mornings average 4 mmHg above the fortnight before";
@@ -723,13 +725,26 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise<void> {
userId: OWNER_ID,
name: "Grip strength",
unit: "kg",
// v1.39.3 — one reading whose note is still readable (a row the
// free-text backfill has not reached) and one sealed the current way.
// Both must come back sealed, and neither readable.
entries: {
create: {
userId: OWNER_ID,
value: 44,
unit: "kg",
measuredAt: AT("2026-07-01T18:00:00.000Z"),
},
create: [
{
userId: OWNER_ID,
value: 44,
unit: "kg",
measuredAt: AT("2026-07-01T18:00:00.000Z"),
note: METRIC_LEGACY_NOTE,
},
{
userId: OWNER_ID,
value: 45,
unit: "kg",
measuredAt: AT("2026-07-02T18:00:00.000Z"),
noteEncrypted: encryptToBytes(METRIC_SEALED_NOTE),
},
],
},
},
});
@@ -923,7 +938,9 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise<void> {
await prisma.coachConversation.create({
data: {
userId: OWNER_ID,
title: "About my June labs",
// v1.39.3 — sealed the current way; the thread above keeps a readable
// legacy title, the shape of a row the backfill has not reached.
titleEncrypted: encryptToBytes("About my June labs"),
documentScoped: true,
messages: {
create: [
@@ -1524,6 +1541,20 @@ describe("every model the plan claims two-ended survives a real restore", () =>
sourceId: "4711",
aiReadDeferred: true,
});
const metricEntries = await prisma.customMetricEntry.findMany({
where: { userId: OWNER_ID },
orderBy: { measuredAt: "asc" },
});
expect(
metricEntries.map((entry) => ({
note: readNote(entry.noteEncrypted, entry.note),
readable: entry.note,
})),
"both reading notes come back, sealed, with nothing readable",
).toEqual([
{ note: METRIC_LEGACY_NOTE, readable: null },
{ note: METRIC_SEALED_NOTE, readable: null },
]);
const threads = await prisma.coachConversation.findMany({
where: { userId: OWNER_ID },
orderBy: { createdAt: "asc" },
@@ -1534,7 +1565,9 @@ describe("every model the plan claims two-ended survives a real restore", () =>
});
expect(
threads.map((thread) => ({
title: thread.title,
title: readNote(thread.titleEncrypted, thread.title),
// A restore seals every title and never writes the readable column.
readableTitle: thread.title,
documentScoped: thread.documentScoped,
summary: thread.summaryEncrypted
? decryptFromBytes(thread.summaryEncrypted)
@@ -1552,6 +1585,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
).toEqual([
{
title: "How is my blood pressure trending?",
readableTitle: null,
documentScoped: false,
summary: COACH_SUMMARY,
summaryTurnCount: 4,
@@ -1573,6 +1607,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
},
{
title: "About my June labs",
readableTitle: null,
documentScoped: true,
summary: null,
summaryTurnCount: 0,
@@ -0,0 +1,148 @@
/**
* v1.39.3 — importing the same large CSV twice.
*
* A re-upload of a 9 000-row export with an externalId column ended in a 500:
* every row already existed, so the route ran one update per row, serially,
* inside a single interactive transaction, and that outlived Prisma's 5 s
* timeout. Only a real database shows it. This imports the same file twice and
* then a lightly edited copy, and reads the table back after each pass.
*/
import { NextRequest } from "next/server";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { cookieJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
const USER = "user-csv-reimport";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => {
cookieJar.set(name, value);
},
delete: (name: string) => {
cookieJar.delete(name);
},
})),
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
const ROWS = 9000;
/** 8 000 externalId rows and 1 000 without one, a minute apart. */
function fileLines(changed: ReadonlySet<number> = new Set()): string[] {
return Array.from({ length: ROWS }, (_, i) => {
const at = new Date(Date.UTC(2025, 0, 1) + i * 60_000).toISOString();
const value = (70 + (i % 50) / 10 + (changed.has(i) ? 1 : 0)).toFixed(1);
const note = i % 100 === 0 ? "after the long ride" : "";
const externalId = i < 8000 ? `export-${i}` : "";
return `WEIGHT,${value},kg,${at},,${note},${externalId}`;
});
}
function csvRequest(lines: string[]): NextRequest {
return new NextRequest("http://localhost/api/import/csv", {
method: "POST",
headers: { "content-type": "text/csv" },
body: [HEADER, ...lines].join("\n"),
});
}
interface CsvEnvelope {
data: {
inserted: number;
updated: number;
skipped: number;
total: number;
} | null;
}
async function signIn() {
await getPrismaClient().user.create({
data: {
id: USER,
username: "csv-reimport",
email: "csv-reimport@example.test",
timezone: "Europe/Berlin",
},
});
const session = await getPrismaClient().session.create({
data: { userId: USER, expiresAt: new Date(Date.now() + 60 * 60 * 1000) },
});
cookieJar.set("healthlog_session", session.id);
}
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
cookieJar.clear();
});
describe("POST /api/import/csv — re-importing a 9 000-row file (real Postgres)", () => {
it("imports once, writes nothing the second time, and updates only what changed", async () => {
await signIn();
const { POST } = await import("@/app/api/import/csv/route");
const prisma = getPrismaClient();
const first = await POST(csvRequest(fileLines()));
expect(first.status).toBe(200);
expect(((await first.json()) as CsvEnvelope).data).toMatchObject({
inserted: ROWS,
updated: 0,
skipped: 0,
});
expect(await prisma.measurement.count({ where: { userId: USER } })).toBe(
ROWS,
);
const before = await prisma.measurement.findMany({
where: { userId: USER },
select: { id: true, value: true, notesEncrypted: true },
orderBy: { measuredAt: "asc" },
});
const second = await POST(csvRequest(fileLines()));
expect(second.status).toBe(200);
expect(((await second.json()) as CsvEnvelope).data).toMatchObject({
inserted: 0,
updated: 0,
skipped: ROWS,
});
const after = await prisma.measurement.findMany({
where: { userId: USER },
select: { id: true, value: true, notesEncrypted: true },
orderBy: { measuredAt: "asc" },
});
// Same rows, untouched: not even the note ciphertext was re-sealed.
expect(after).toEqual(before);
const changed = new Set([0, 1, 4999, 7999]);
const third = await POST(csvRequest(fileLines(changed)));
expect(third.status).toBe(200);
expect(((await third.json()) as CsvEnvelope).data).toMatchObject({
inserted: 0,
updated: changed.size,
skipped: ROWS - changed.size,
});
expect(await prisma.measurement.count({ where: { userId: USER } })).toBe(
ROWS,
);
const edited = await prisma.measurement.findFirst({
where: { userId: USER, externalId: "export-4999" },
select: { value: true },
});
expect(edited?.value).toBeCloseTo(70 + (4999 % 50) / 10 + 1, 5);
}, 120_000);
});
@@ -0,0 +1,181 @@
/**
* v1.39.3 — the Coach conversation title and the custom-metric reading note
* move to AES-256-GCM at rest. Migration 0357 only adds the ciphertext
* columns; the existing rows are sealed by the boot-time backfill. This runs
* the real discovery and the real per-user pass against Postgres, with the
* migrated schema:
*
* - discovery finds exactly the accounts still holding a readable value;
* - the pass seals every such row in its own transaction, nulls the readable
* column, and reads back the same text;
* - a second pass and a second discovery find nothing (it converges);
* - a row written the current way is left as it was.
*/
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { setGlobalBoss } from "@/lib/jobs/boss-instance";
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { readNote } from "@/lib/crypto/note-cipher";
import {
enqueueBootTimeFreeTextEncryptionBackfill,
runFreeTextEncryptionBackfillForUser,
} from "@/lib/jobs/free-text-encryption-backfill";
import { getPrismaClient, truncateAllTables } from "./setup";
function makeCapturingBoss() {
const sent: { queue: string; userId: string; startAfter?: number }[] = [];
const boss = {
send: async (
queue: string,
payload: { userId?: string },
opts: { startAfter?: number },
) => {
if (payload.userId) sent.push({ queue, userId: payload.userId, ...opts });
return `job-${sent.length}`;
},
};
return { boss, sent };
}
async function seedUser(id: string) {
await getPrismaClient().user.create({
data: {
id,
username: id,
email: `${id}@example.test`,
timezone: "Europe/Berlin",
},
});
}
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
});
afterEach(() => {
setGlobalBoss(null as never);
});
describe("free-text encryption backfill (real Postgres)", () => {
it("discovers, seals, and converges", async () => {
const prisma = getPrismaClient();
await Promise.all([seedUser("ft-a"), seedUser("ft-b"), seedUser("ft-c")]);
// ft-a: a legacy readable title and a sealed one.
await prisma.coachConversation.create({
data: { id: "conv-legacy", userId: "ft-a", title: "Legacy title" },
});
await prisma.coachConversation.create({
data: {
id: "conv-sealed",
userId: "ft-a",
titleEncrypted: encryptToBytes("Sealed title"),
},
});
// ft-b: a legacy readable note on a custom-metric reading.
const metric = await prisma.customMetric.create({
data: { userId: "ft-b", name: "Grip", unit: "kg" },
});
await prisma.customMetricEntry.create({
data: {
id: "entry-legacy",
userId: "ft-b",
customMetricId: metric.id,
value: 40,
unit: "kg",
measuredAt: new Date("2026-09-01T08:00:00.000Z"),
note: "legacy note",
},
});
await prisma.customMetricEntry.create({
data: {
id: "entry-none",
userId: "ft-b",
customMetricId: metric.id,
value: 41,
unit: "kg",
measuredAt: new Date("2026-09-02T08:00:00.000Z"),
},
});
// ft-c: nothing readable, only a sealed title.
await prisma.coachConversation.create({
data: { userId: "ft-c", titleEncrypted: encryptToBytes("Only sealed") },
});
const first = makeCapturingBoss();
setGlobalBoss(first.boss as never);
const discovered = await enqueueBootTimeFreeTextEncryptionBackfill(180);
expect(discovered.error).toBeNull();
expect(first.sent.map((s) => s.userId).sort()).toEqual(["ft-a", "ft-b"]);
// The boot call is deferred past the startup storm.
expect(first.sent.every((s) => s.startAfter === 180)).toBe(true);
expect(await runFreeTextEncryptionBackfillForUser("ft-a")).toEqual({
conversationTitlesMigrated: 1,
metricNotesMigrated: 0,
});
expect(await runFreeTextEncryptionBackfillForUser("ft-b")).toEqual({
conversationTitlesMigrated: 0,
metricNotesMigrated: 1,
});
const conversations = await prisma.coachConversation.findMany({
where: { userId: "ft-a" },
orderBy: { id: "asc" },
});
expect(
conversations.map((c) => ({
id: c.id,
readable: c.title,
text: readNote(c.titleEncrypted, c.title),
})),
).toEqual([
{ id: "conv-legacy", readable: null, text: "Legacy title" },
{ id: "conv-sealed", readable: null, text: "Sealed title" },
]);
const entries = await prisma.customMetricEntry.findMany({
where: { userId: "ft-b" },
orderBy: { id: "asc" },
});
expect(
entries.map((e) => ({
id: e.id,
readable: e.note,
text: readNote(e.noteEncrypted, e.note),
})),
).toEqual([
{ id: "entry-legacy", readable: null, text: "legacy note" },
{ id: "entry-none", readable: null, text: null },
]);
// Converged: nothing left to discover, nothing left to seal.
const second = makeCapturingBoss();
setGlobalBoss(second.boss as never);
expect(
(await enqueueBootTimeFreeTextEncryptionBackfill()).error,
).toBeNull();
expect(second.sent).toEqual([]);
expect(await runFreeTextEncryptionBackfillForUser("ft-a")).toEqual({
conversationTitlesMigrated: 0,
metricNotesMigrated: 0,
});
});
it("uses the partial indexes migration 0357 adds", async () => {
const indexes = await getPrismaClient().$queryRaw<
{ indexname: string; indexdef: string }[]
>`
SELECT indexname, indexdef FROM pg_indexes
WHERE indexname IN (
'coach_conversations_title_backfill_idx',
'custom_metric_entries_note_backfill_idx'
)
ORDER BY indexname`;
expect(indexes.map((i) => i.indexname)).toEqual([
"coach_conversations_title_backfill_idx",
"custom_metric_entries_note_backfill_idx",
]);
expect(indexes[0].indexdef).toContain("WHERE (title IS NOT NULL)");
expect(indexes[1].indexdef).toContain("WHERE (note IS NOT NULL)");
});
});