mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
Merge branch 'fix/encrypt-free-text-and-local-ai' into release/v1.39.3
# Conflicts: # src/app/api/import/csv/route.ts
This commit is contained in:
commit
66b1aef40c
69 files changed
+3625
-367
No files matched your search
@@ -112,6 +112,8 @@ const COACH_PLAN_CUE = "if the evening reading is over 140";
|
||||
const COACH_PLAN_ACTION = "then walk twenty minutes before dinner tomorrow";
|
||||
const COACH_REMINDER_NOTE = "ask how the evening walks are going";
|
||||
const COACH_SUMMARY = "earlier turns: weight trend and evening walks";
|
||||
const METRIC_LEGACY_NOTE = "left hand, after the long ride";
|
||||
const METRIC_SEALED_NOTE = "right hand, rested";
|
||||
const COACH_USER_TURN = "my readings look higher this week, is that real?";
|
||||
const COACH_ASSISTANT_TURN =
|
||||
"the last seven mornings average 4 mmHg above the fortnight before";
|
||||
@@ -723,13 +725,26 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise<void> {
|
||||
userId: OWNER_ID,
|
||||
name: "Grip strength",
|
||||
unit: "kg",
|
||||
// v1.39.3 — one reading whose note is still readable (a row the
|
||||
// free-text backfill has not reached) and one sealed the current way.
|
||||
// Both must come back sealed, and neither readable.
|
||||
entries: {
|
||||
create: {
|
||||
userId: OWNER_ID,
|
||||
value: 44,
|
||||
unit: "kg",
|
||||
measuredAt: AT("2026-07-01T18:00:00.000Z"),
|
||||
},
|
||||
create: [
|
||||
{
|
||||
userId: OWNER_ID,
|
||||
value: 44,
|
||||
unit: "kg",
|
||||
measuredAt: AT("2026-07-01T18:00:00.000Z"),
|
||||
note: METRIC_LEGACY_NOTE,
|
||||
},
|
||||
{
|
||||
userId: OWNER_ID,
|
||||
value: 45,
|
||||
unit: "kg",
|
||||
measuredAt: AT("2026-07-02T18:00:00.000Z"),
|
||||
noteEncrypted: encryptToBytes(METRIC_SEALED_NOTE),
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -923,7 +938,9 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise<void> {
|
||||
await prisma.coachConversation.create({
|
||||
data: {
|
||||
userId: OWNER_ID,
|
||||
title: "About my June labs",
|
||||
// v1.39.3 — sealed the current way; the thread above keeps a readable
|
||||
// legacy title, the shape of a row the backfill has not reached.
|
||||
titleEncrypted: encryptToBytes("About my June labs"),
|
||||
documentScoped: true,
|
||||
messages: {
|
||||
create: [
|
||||
@@ -1524,6 +1541,20 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
sourceId: "4711",
|
||||
aiReadDeferred: true,
|
||||
});
|
||||
const metricEntries = await prisma.customMetricEntry.findMany({
|
||||
where: { userId: OWNER_ID },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
});
|
||||
expect(
|
||||
metricEntries.map((entry) => ({
|
||||
note: readNote(entry.noteEncrypted, entry.note),
|
||||
readable: entry.note,
|
||||
})),
|
||||
"both reading notes come back, sealed, with nothing readable",
|
||||
).toEqual([
|
||||
{ note: METRIC_LEGACY_NOTE, readable: null },
|
||||
{ note: METRIC_SEALED_NOTE, readable: null },
|
||||
]);
|
||||
const threads = await prisma.coachConversation.findMany({
|
||||
where: { userId: OWNER_ID },
|
||||
orderBy: { createdAt: "asc" },
|
||||
@@ -1534,7 +1565,9 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
});
|
||||
expect(
|
||||
threads.map((thread) => ({
|
||||
title: thread.title,
|
||||
title: readNote(thread.titleEncrypted, thread.title),
|
||||
// A restore seals every title and never writes the readable column.
|
||||
readableTitle: thread.title,
|
||||
documentScoped: thread.documentScoped,
|
||||
summary: thread.summaryEncrypted
|
||||
? decryptFromBytes(thread.summaryEncrypted)
|
||||
@@ -1552,6 +1585,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
).toEqual([
|
||||
{
|
||||
title: "How is my blood pressure trending?",
|
||||
readableTitle: null,
|
||||
documentScoped: false,
|
||||
summary: COACH_SUMMARY,
|
||||
summaryTurnCount: 4,
|
||||
@@ -1573,6 +1607,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
},
|
||||
{
|
||||
title: "About my June labs",
|
||||
readableTitle: null,
|
||||
documentScoped: true,
|
||||
summary: null,
|
||||
summaryTurnCount: 0,
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* v1.39.3 — importing the same large CSV twice.
|
||||
*
|
||||
* A re-upload of a 9 000-row export with an externalId column ended in a 500:
|
||||
* every row already existed, so the route ran one update per row, serially,
|
||||
* inside a single interactive transaction, and that outlived Prisma's 5 s
|
||||
* timeout. Only a real database shows it. This imports the same file twice and
|
||||
* then a lightly edited copy, and reads the table back after each pass.
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { cookieJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
const USER = "user-csv-reimport";
|
||||
|
||||
vi.mock("next/headers", async () => {
|
||||
const { cookieJar, headerJar } = await import("./mock-next-headers");
|
||||
return {
|
||||
headers: vi.fn(async () => ({
|
||||
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
|
||||
})),
|
||||
cookies: vi.fn(async () => ({
|
||||
get: (name: string) => {
|
||||
const value = cookieJar.get(name);
|
||||
return value ? { name, value } : undefined;
|
||||
},
|
||||
set: (name: string, value: string) => {
|
||||
cookieJar.set(name, value);
|
||||
},
|
||||
delete: (name: string) => {
|
||||
cookieJar.delete(name);
|
||||
},
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const HEADER = "type,value,unit,measuredAt,glucoseContext,notes,externalId";
|
||||
const ROWS = 9000;
|
||||
|
||||
/** 8 000 externalId rows and 1 000 without one, a minute apart. */
|
||||
function fileLines(changed: ReadonlySet<number> = new Set()): string[] {
|
||||
return Array.from({ length: ROWS }, (_, i) => {
|
||||
const at = new Date(Date.UTC(2025, 0, 1) + i * 60_000).toISOString();
|
||||
const value = (70 + (i % 50) / 10 + (changed.has(i) ? 1 : 0)).toFixed(1);
|
||||
const note = i % 100 === 0 ? "after the long ride" : "";
|
||||
const externalId = i < 8000 ? `export-${i}` : "";
|
||||
return `WEIGHT,${value},kg,${at},,${note},${externalId}`;
|
||||
});
|
||||
}
|
||||
|
||||
function csvRequest(lines: string[]): NextRequest {
|
||||
return new NextRequest("http://localhost/api/import/csv", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "text/csv" },
|
||||
body: [HEADER, ...lines].join("\n"),
|
||||
});
|
||||
}
|
||||
|
||||
interface CsvEnvelope {
|
||||
data: {
|
||||
inserted: number;
|
||||
updated: number;
|
||||
skipped: number;
|
||||
total: number;
|
||||
} | null;
|
||||
}
|
||||
|
||||
async function signIn() {
|
||||
await getPrismaClient().user.create({
|
||||
data: {
|
||||
id: USER,
|
||||
username: "csv-reimport",
|
||||
email: "csv-reimport@example.test",
|
||||
timezone: "Europe/Berlin",
|
||||
},
|
||||
});
|
||||
const session = await getPrismaClient().session.create({
|
||||
data: { userId: USER, expiresAt: new Date(Date.now() + 60 * 60 * 1000) },
|
||||
});
|
||||
cookieJar.set("healthlog_session", session.id);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
cookieJar.clear();
|
||||
});
|
||||
|
||||
describe("POST /api/import/csv — re-importing a 9 000-row file (real Postgres)", () => {
|
||||
it("imports once, writes nothing the second time, and updates only what changed", async () => {
|
||||
await signIn();
|
||||
const { POST } = await import("@/app/api/import/csv/route");
|
||||
const prisma = getPrismaClient();
|
||||
|
||||
const first = await POST(csvRequest(fileLines()));
|
||||
expect(first.status).toBe(200);
|
||||
expect(((await first.json()) as CsvEnvelope).data).toMatchObject({
|
||||
inserted: ROWS,
|
||||
updated: 0,
|
||||
skipped: 0,
|
||||
});
|
||||
expect(await prisma.measurement.count({ where: { userId: USER } })).toBe(
|
||||
ROWS,
|
||||
);
|
||||
const before = await prisma.measurement.findMany({
|
||||
where: { userId: USER },
|
||||
select: { id: true, value: true, notesEncrypted: true },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
});
|
||||
|
||||
const second = await POST(csvRequest(fileLines()));
|
||||
expect(second.status).toBe(200);
|
||||
expect(((await second.json()) as CsvEnvelope).data).toMatchObject({
|
||||
inserted: 0,
|
||||
updated: 0,
|
||||
skipped: ROWS,
|
||||
});
|
||||
const after = await prisma.measurement.findMany({
|
||||
where: { userId: USER },
|
||||
select: { id: true, value: true, notesEncrypted: true },
|
||||
orderBy: { measuredAt: "asc" },
|
||||
});
|
||||
// Same rows, untouched: not even the note ciphertext was re-sealed.
|
||||
expect(after).toEqual(before);
|
||||
|
||||
const changed = new Set([0, 1, 4999, 7999]);
|
||||
const third = await POST(csvRequest(fileLines(changed)));
|
||||
expect(third.status).toBe(200);
|
||||
expect(((await third.json()) as CsvEnvelope).data).toMatchObject({
|
||||
inserted: 0,
|
||||
updated: changed.size,
|
||||
skipped: ROWS - changed.size,
|
||||
});
|
||||
expect(await prisma.measurement.count({ where: { userId: USER } })).toBe(
|
||||
ROWS,
|
||||
);
|
||||
const edited = await prisma.measurement.findFirst({
|
||||
where: { userId: USER, externalId: "export-4999" },
|
||||
select: { value: true },
|
||||
});
|
||||
expect(edited?.value).toBeCloseTo(70 + (4999 % 50) / 10 + 1, 5);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* v1.39.3 — the Coach conversation title and the custom-metric reading note
|
||||
* move to AES-256-GCM at rest. Migration 0357 only adds the ciphertext
|
||||
* columns; the existing rows are sealed by the boot-time backfill. This runs
|
||||
* the real discovery and the real per-user pass against Postgres, with the
|
||||
* migrated schema:
|
||||
*
|
||||
* - discovery finds exactly the accounts still holding a readable value;
|
||||
* - the pass seals every such row in its own transaction, nulls the readable
|
||||
* column, and reads back the same text;
|
||||
* - a second pass and a second discovery find nothing (it converges);
|
||||
* - a row written the current way is left as it was.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { setGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
|
||||
import { readNote } from "@/lib/crypto/note-cipher";
|
||||
import {
|
||||
enqueueBootTimeFreeTextEncryptionBackfill,
|
||||
runFreeTextEncryptionBackfillForUser,
|
||||
} from "@/lib/jobs/free-text-encryption-backfill";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
function makeCapturingBoss() {
|
||||
const sent: { queue: string; userId: string; startAfter?: number }[] = [];
|
||||
const boss = {
|
||||
send: async (
|
||||
queue: string,
|
||||
payload: { userId?: string },
|
||||
opts: { startAfter?: number },
|
||||
) => {
|
||||
if (payload.userId) sent.push({ queue, userId: payload.userId, ...opts });
|
||||
return `job-${sent.length}`;
|
||||
},
|
||||
};
|
||||
return { boss, sent };
|
||||
}
|
||||
|
||||
async function seedUser(id: string) {
|
||||
await getPrismaClient().user.create({
|
||||
data: {
|
||||
id,
|
||||
username: id,
|
||||
email: `${id}@example.test`,
|
||||
timezone: "Europe/Berlin",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setGlobalBoss(null as never);
|
||||
});
|
||||
|
||||
describe("free-text encryption backfill (real Postgres)", () => {
|
||||
it("discovers, seals, and converges", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await Promise.all([seedUser("ft-a"), seedUser("ft-b"), seedUser("ft-c")]);
|
||||
|
||||
// ft-a: a legacy readable title and a sealed one.
|
||||
await prisma.coachConversation.create({
|
||||
data: { id: "conv-legacy", userId: "ft-a", title: "Legacy title" },
|
||||
});
|
||||
await prisma.coachConversation.create({
|
||||
data: {
|
||||
id: "conv-sealed",
|
||||
userId: "ft-a",
|
||||
titleEncrypted: encryptToBytes("Sealed title"),
|
||||
},
|
||||
});
|
||||
// ft-b: a legacy readable note on a custom-metric reading.
|
||||
const metric = await prisma.customMetric.create({
|
||||
data: { userId: "ft-b", name: "Grip", unit: "kg" },
|
||||
});
|
||||
await prisma.customMetricEntry.create({
|
||||
data: {
|
||||
id: "entry-legacy",
|
||||
userId: "ft-b",
|
||||
customMetricId: metric.id,
|
||||
value: 40,
|
||||
unit: "kg",
|
||||
measuredAt: new Date("2026-09-01T08:00:00.000Z"),
|
||||
note: "legacy note",
|
||||
},
|
||||
});
|
||||
await prisma.customMetricEntry.create({
|
||||
data: {
|
||||
id: "entry-none",
|
||||
userId: "ft-b",
|
||||
customMetricId: metric.id,
|
||||
value: 41,
|
||||
unit: "kg",
|
||||
measuredAt: new Date("2026-09-02T08:00:00.000Z"),
|
||||
},
|
||||
});
|
||||
// ft-c: nothing readable, only a sealed title.
|
||||
await prisma.coachConversation.create({
|
||||
data: { userId: "ft-c", titleEncrypted: encryptToBytes("Only sealed") },
|
||||
});
|
||||
|
||||
const first = makeCapturingBoss();
|
||||
setGlobalBoss(first.boss as never);
|
||||
const discovered = await enqueueBootTimeFreeTextEncryptionBackfill(180);
|
||||
expect(discovered.error).toBeNull();
|
||||
expect(first.sent.map((s) => s.userId).sort()).toEqual(["ft-a", "ft-b"]);
|
||||
// The boot call is deferred past the startup storm.
|
||||
expect(first.sent.every((s) => s.startAfter === 180)).toBe(true);
|
||||
|
||||
expect(await runFreeTextEncryptionBackfillForUser("ft-a")).toEqual({
|
||||
conversationTitlesMigrated: 1,
|
||||
metricNotesMigrated: 0,
|
||||
});
|
||||
expect(await runFreeTextEncryptionBackfillForUser("ft-b")).toEqual({
|
||||
conversationTitlesMigrated: 0,
|
||||
metricNotesMigrated: 1,
|
||||
});
|
||||
|
||||
const conversations = await prisma.coachConversation.findMany({
|
||||
where: { userId: "ft-a" },
|
||||
orderBy: { id: "asc" },
|
||||
});
|
||||
expect(
|
||||
conversations.map((c) => ({
|
||||
id: c.id,
|
||||
readable: c.title,
|
||||
text: readNote(c.titleEncrypted, c.title),
|
||||
})),
|
||||
).toEqual([
|
||||
{ id: "conv-legacy", readable: null, text: "Legacy title" },
|
||||
{ id: "conv-sealed", readable: null, text: "Sealed title" },
|
||||
]);
|
||||
const entries = await prisma.customMetricEntry.findMany({
|
||||
where: { userId: "ft-b" },
|
||||
orderBy: { id: "asc" },
|
||||
});
|
||||
expect(
|
||||
entries.map((e) => ({
|
||||
id: e.id,
|
||||
readable: e.note,
|
||||
text: readNote(e.noteEncrypted, e.note),
|
||||
})),
|
||||
).toEqual([
|
||||
{ id: "entry-legacy", readable: null, text: "legacy note" },
|
||||
{ id: "entry-none", readable: null, text: null },
|
||||
]);
|
||||
|
||||
// Converged: nothing left to discover, nothing left to seal.
|
||||
const second = makeCapturingBoss();
|
||||
setGlobalBoss(second.boss as never);
|
||||
expect(
|
||||
(await enqueueBootTimeFreeTextEncryptionBackfill()).error,
|
||||
).toBeNull();
|
||||
expect(second.sent).toEqual([]);
|
||||
expect(await runFreeTextEncryptionBackfillForUser("ft-a")).toEqual({
|
||||
conversationTitlesMigrated: 0,
|
||||
metricNotesMigrated: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the partial indexes migration 0357 adds", async () => {
|
||||
const indexes = await getPrismaClient().$queryRaw<
|
||||
{ indexname: string; indexdef: string }[]
|
||||
>`
|
||||
SELECT indexname, indexdef FROM pg_indexes
|
||||
WHERE indexname IN (
|
||||
'coach_conversations_title_backfill_idx',
|
||||
'custom_metric_entries_note_backfill_idx'
|
||||
)
|
||||
ORDER BY indexname`;
|
||||
expect(indexes.map((i) => i.indexname)).toEqual([
|
||||
"coach_conversations_title_backfill_idx",
|
||||
"custom_metric_entries_note_backfill_idx",
|
||||
]);
|
||||
expect(indexes[0].indexdef).toContain("WHERE (title IS NOT NULL)");
|
||||
expect(indexes[1].indexdef).toContain("WHERE (note IS NOT NULL)");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user