Merge branch 'fix/encrypt-free-text-and-local-ai' into release/v1.39.3

# Conflicts:
#	src/app/api/import/csv/route.ts

src/app/api/admin/ai-settings/__tests__/route.test.ts

@@ -6,7 +6,7 @@
  * HTTPS + hostname allowlist on the base URL, and the masked
  * `keyPreview` in every response.
  */
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 import { NextRequest } from "next/server";
 
 vi.mock("@/lib/db", () => ({
@@ -15,6 +15,7 @@ vi.mock("@/lib/db", () => ({
       findUnique: vi.fn(),
       upsert: vi.fn(),
     },
+    user: { findMany: vi.fn().mockResolvedValue([]) },
   },
 }));
 
@@ -100,6 +101,7 @@ describe("GET /api/admin/ai-settings", () => {
       keyPreview: null,
       model: "gpt-4o",
       baseUrl: "https://api.openai.com/v1",
+      legacyPrivateHostGrant: null,
     });
   });
 
@@ -216,3 +218,52 @@ describe("PUT /api/admin/ai-settings", () => {
     });
   });
 });
+
+/**
+ * v1.39.3 — while the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is set,
+ * the page names it and lists the origins it is standing in for, so the
+ * operator can copy them into `AI_PRIVATE_ORIGINS`.
+ */
+describe("GET /api/admin/ai-settings — deprecated private-host grant", () => {
+  beforeEach(async () => {
+    const { _resetAiGrantsForTests } =
+      await import("@/lib/ai/local-host-allowlist");
+    _resetAiGrantsForTests();
+    vi.mocked(requireAdmin).mockResolvedValue(ADMIN_CTX as never);
+    vi.mocked(prisma.appSettings.findUnique).mockResolvedValue({
+      adminAiKeyEncrypted: null,
+      adminAiModel: null,
+      adminAiBaseUrl: "http://gateway.lan:4000/v1",
+    } as never);
+    vi.mocked(prisma.user.findMany).mockResolvedValue([
+      { aiBaseUrl: "http://10.0.0.5:11434/v1", aiCompatBaseUrl: null },
+      {
+        aiBaseUrl: "http://10.0.0.5:11434/v1/",
+        aiCompatBaseUrl: "https://openrouter.ai/api/v1",
+      },
+      { aiBaseUrl: "http://169.254.169.254/latest", aiCompatBaseUrl: null },
+    ] as never);
+    vi.stubEnv("AI_PRIVATE_ORIGINS", "");
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
+  it("lists the private origins in use, once each, when `true` is set", async () => {
+    vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
+    const res = await GET();
+    const body = await res.json();
+    expect(body.data.legacyPrivateHostGrant).toEqual({
+      originsInUse: ["http://10.0.0.5:11434", "http://gateway.lan:4000"],
+    });
+  });
+
+  it("is null when `true` is not set", async () => {
+    vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "ollama.lan");
+    const res = await GET();
+    expect((await res.json()).data.legacyPrivateHostGrant).toBeNull();
+    expect(prisma.user.findMany).not.toHaveBeenCalled();
+  });
+});

src/app/api/admin/ai-settings/route.ts

@@ -7,6 +7,7 @@ import { apiSuccess, getClientIp, safeJson } from "@/lib/api-response";
 import { encrypt, decrypt } from "@/lib/crypto";
 import { checkRateLimit } from "@/lib/rate-limit";
 import { annotate } from "@/lib/logging/context";
+import { legacyPrivateHostReport } from "@/lib/ai/legacy-private-host-report";
 
 export const dynamic = "force-dynamic";
 
@@ -29,6 +30,11 @@ export const GET = apiHandler(async () => {
       : null,
     model,
     baseUrl,
+    // v1.39.3 — present while the deprecated ALLOW_LOCAL_AI_PRIVATE_HOSTS=true
+    // is set, naming the origins to move into AI_PRIVATE_ORIGINS.
+    legacyPrivateHostGrant: await legacyPrivateHostReport(
+      settings?.adminAiBaseUrl,
+    ),
   });
 });
 

src/app/api/custom-metrics/[id]/entries/[entryId]/route.ts

@@ -12,6 +12,7 @@ import {
 import { auditLog } from "@/lib/auth/audit";
 import { invalidateUserCorrelationPatterns } from "@/lib/cache/invalidate";
 import { serialiseCustomMetricEntry } from "@/lib/custom-metrics/custom-metric-store";
+import { encryptNote } from "@/lib/crypto/note-cipher";
 import { prisma } from "@/lib/db";
 import { annotate } from "@/lib/logging/context";
 import { updateCustomMetricEntrySchema } from "@/lib/validations/custom-metrics";
@@ -83,7 +84,13 @@ export const PATCH = apiHandler(
     const data: Record<string, unknown> = {};
     if (d.value !== undefined) data.value = d.value;
     if (d.measuredAt !== undefined) data.measuredAt = d.measuredAt;
-    if (d.note !== undefined) data.note = d.note;
+    // v1.39.3 — the note is stored encrypted. Any note write (a new value or
+    // an explicit clear) also nulls the legacy readable column, so an edited
+    // row never keeps its old text beside the new ciphertext.
+    if (d.note !== undefined) {
+      data.noteEncrypted = encryptNote(d.note);
+      data.note = null;
+    }
 
     const updated = await prisma.customMetricEntry.update({
       where: { id: entryId },

src/app/api/custom-metrics/[id]/entries/__tests__/note-encryption.test.ts

@@ -0,0 +1,238 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { NextRequest } from "next/server";
+
+vi.mock("@/lib/rate-limit", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("@/lib/rate-limit")>()),
+  // The shared single-record write ceiling. Allowed here: this file is about
+  // what the handler does with a body it accepted, not about the bucket.
+  checkRecordWriteRateLimit: vi.fn(async () => ({
+    allowed: true,
+    limit: 300,
+    remaining: 299,
+    resetAt: Date.now() + 60_000,
+  })),
+}));
+vi.mock("@/lib/cache/invalidate", () => ({
+  invalidateUserCorrelationPatterns: vi.fn(),
+}));
+vi.mock("@/lib/db", () => ({
+  prisma: {
+    customMetric: { findFirst: vi.fn() },
+    customMetricEntry: {
+      findMany: vi.fn(),
+      count: vi.fn(),
+      create: vi.fn(),
+      findFirst: vi.fn(),
+      update: vi.fn(),
+    },
+    auditLog: { create: vi.fn().mockResolvedValue({}) },
+  },
+}));
+
+vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
+
+vi.mock("@/lib/auth/audit", () => ({
+  auditLog: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() }));
+
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("@/lib/idempotency", () => ({
+  withIdempotency: (fn: unknown) => fn,
+}));
+
+vi.mock("next/headers", () => ({
+  headers: vi.fn(async () => ({ get: () => null })),
+  cookies: vi.fn(async () => ({
+    get: () => undefined,
+    set: () => {},
+    delete: () => {},
+  })),
+}));
+
+import { GET, POST } from "../route";
+import { PATCH } from "../[entryId]/route";
+import { getSession } from "@/lib/auth/session";
+import { prisma } from "@/lib/db";
+import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec";
+
+/**
+ * v1.39.3 — the note on a custom-metric reading is encrypted at rest. A write
+ * stores ciphertext and leaves the readable column empty; a read opens the
+ * ciphertext and falls back to the readable column only for a row the backfill
+ * has not reached.
+ */
+
+process.env.ENCRYPTION_KEY ??=
+  "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+
+const SESSION_OK = {
+  session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
+  user: { id: "user-1", username: "u", role: "USER" as const, locale: "en" },
+};
+
+const ENTRY = {
+  id: "e-1",
+  userId: "user-1",
+  customMetricId: "cm-1",
+  value: 42,
+  unit: "kg",
+  measuredAt: new Date("2026-06-10T00:00:00.000Z"),
+  note: null as string | null,
+  noteEncrypted: null as Uint8Array | null,
+  createdAt: new Date("2026-06-10T00:00:00.000Z"),
+  deletedAt: null,
+};
+
+function jsonRequest(url: string, method: string, body: unknown) {
+  return new NextRequest(url, {
+    method,
+    body: JSON.stringify(body),
+    headers: { "content-type": "application/json" },
+  });
+}
+
+beforeEach(() => {
+  vi.resetAllMocks();
+  vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never);
+  vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
+});
+
+describe("custom-metric reading note at rest", () => {
+  it("POST stores the note as ciphertext and answers with the readable note", async () => {
+    vi.mocked(prisma.customMetric.findFirst).mockResolvedValue({
+      id: "cm-1",
+      unit: "kg",
+    } as never);
+    vi.mocked(prisma.customMetricEntry.create).mockImplementation(
+      (async (args: { data: Record<string, unknown> }) => ({
+        ...ENTRY,
+        ...args.data,
+      })) as never,
+    );
+
+    const res = await POST(
+      jsonRequest("http://localhost/api/custom-metrics/cm-1/entries", "POST", {
+        value: 42,
+        measuredAt: "2026-06-10T00:00:00.000Z",
+        note: "after the long ride",
+      }),
+      { params: Promise.resolve({ id: "cm-1" }) },
+    );
+    expect(res.status).toBe(201);
+    const data = vi.mocked(prisma.customMetricEntry.create).mock.calls[0][0]
+      .data as Record<string, unknown>;
+    expect(data).not.toHaveProperty("note");
+    expect(decryptFromBytes(data.noteEncrypted as Uint8Array)).toBe(
+      "after the long ride",
+    );
+    const body = await res.json();
+    expect(body.data.note).toBe("after the long ride");
+    expect(body.data).not.toHaveProperty("noteEncrypted");
+  });
+
+  it("GET opens sealed notes and still shows a legacy readable one", async () => {
+    vi.mocked(prisma.customMetric.findFirst).mockResolvedValue({
+      id: "cm-1",
+    } as never);
+    vi.mocked(prisma.customMetricEntry.findMany).mockResolvedValue([
+      { ...ENTRY, id: "e-1", noteEncrypted: encryptToBytes("sealed note") },
+      { ...ENTRY, id: "e-2", note: "legacy note" },
+      { ...ENTRY, id: "e-3" },
+    ] as never);
+    vi.mocked(prisma.customMetricEntry.count).mockResolvedValue(3 as never);
+
+    const res = await GET(
+      new NextRequest("http://localhost/api/custom-metrics/cm-1/entries"),
+      { params: Promise.resolve({ id: "cm-1" }) },
+    );
+    expect(res.status).toBe(200);
+    const body = await res.json();
+    expect(
+      body.data.entries.map((e: { note: string | null }) => e.note),
+    ).toEqual(["sealed note", "legacy note", null]);
+    for (const entry of body.data.entries) {
+      expect(entry).not.toHaveProperty("noteEncrypted");
+    }
+  });
+
+  it("PATCH seals a new note and clears the legacy readable column", async () => {
+    vi.mocked(prisma.customMetricEntry.findFirst).mockResolvedValue({
+      ...ENTRY,
+      note: "old readable note",
+    } as never);
+    vi.mocked(prisma.customMetricEntry.update).mockImplementation(
+      (async (args: { data: Record<string, unknown> }) => ({
+        ...ENTRY,
+        ...args.data,
+      })) as never,
+    );
+
+    const res = await PATCH(
+      jsonRequest(
+        "http://localhost/api/custom-metrics/cm-1/entries/e-1",
+        "PATCH",
+        { note: "edited" },
+      ),
+      { params: Promise.resolve({ id: "cm-1", entryId: "e-1" }) },
+    );
+    expect(res.status).toBe(200);
+    const data = vi.mocked(prisma.customMetricEntry.update).mock.calls[0][0]
+      .data as Record<string, unknown>;
+    expect(data.note).toBeNull();
+    expect(decryptFromBytes(data.noteEncrypted as Uint8Array)).toBe("edited");
+    expect((await res.json()).data.note).toBe("edited");
+  });
+
+  it("PATCH with an explicit null clears both columns", async () => {
+    vi.mocked(prisma.customMetricEntry.findFirst).mockResolvedValue({
+      ...ENTRY,
+      noteEncrypted: encryptToBytes("to be cleared"),
+    } as never);
+    vi.mocked(prisma.customMetricEntry.update).mockImplementation(
+      (async (args: { data: Record<string, unknown> }) => ({
+        ...ENTRY,
+        ...args.data,
+      })) as never,
+    );
+
+    const res = await PATCH(
+      jsonRequest(
+        "http://localhost/api/custom-metrics/cm-1/entries/e-1",
+        "PATCH",
+        { note: null },
+      ),
+      { params: Promise.resolve({ id: "cm-1", entryId: "e-1" }) },
+    );
+    expect(res.status).toBe(200);
+    const data = vi.mocked(prisma.customMetricEntry.update).mock.calls[0][0]
+      .data as Record<string, unknown>;
+    expect(data).toMatchObject({ note: null, noteEncrypted: null });
+    expect((await res.json()).data.note).toBeNull();
+  });
+
+  it("PATCH without a note leaves both note columns untouched", async () => {
+    vi.mocked(prisma.customMetricEntry.findFirst).mockResolvedValue(
+      ENTRY as never,
+    );
+    vi.mocked(prisma.customMetricEntry.update).mockResolvedValue(
+      ENTRY as never,
+    );
+    await PATCH(
+      jsonRequest(
+        "http://localhost/api/custom-metrics/cm-1/entries/e-1",
+        "PATCH",
+        { value: 43 },
+      ),
+      { params: Promise.resolve({ id: "cm-1", entryId: "e-1" }) },
+    );
+    const data = vi.mocked(prisma.customMetricEntry.update).mock.calls[0][0]
+      .data as Record<string, unknown>;
+    expect(data).not.toHaveProperty("note");
+    expect(data).not.toHaveProperty("noteEncrypted");
+  });
+});

src/app/api/custom-metrics/[id]/entries/route.ts

@@ -13,6 +13,7 @@ import { checkRecordWriteRateLimit } from "@/lib/rate-limit";
 import { auditLog } from "@/lib/auth/audit";
 import { invalidateUserCorrelationPatterns } from "@/lib/cache/invalidate";
 import { serialiseCustomMetricEntry } from "@/lib/custom-metrics/custom-metric-store";
+import { encryptNote } from "@/lib/crypto/note-cipher";
 import { prisma } from "@/lib/db";
 import { withIdempotency } from "@/lib/idempotency";
 import { annotate } from "@/lib/logging/context";
@@ -161,7 +162,8 @@ async function postCustomMetricEntry(
       value,
       unit: metric.unit,
       measuredAt,
-      note: note ?? null,
+      // v1.39.3 — the note is stored encrypted; the readable column stays NULL.
+      noteEncrypted: encryptNote(note),
     },
   });
 

src/app/api/import/csv/__tests__/route.test.ts

@@ -17,10 +17,13 @@ vi.mock("@/lib/db", () => {
   return {
     prisma: {
       measurement,
-      // Run the batched write callback against the same measurement mock so
-      // createManyAndReturn / updateMany calls are observable.
-      $transaction: vi.fn(async (fn: (tx: unknown) => unknown) =>
-        fn({ measurement }),
+      // Both transaction forms against the same measurement mock so
+      // createManyAndReturn / updateMany calls are observable: a callback
+      // gets the mock as `tx`, a batch array is simply awaited.
+      $transaction: vi.fn(async (arg: unknown) =>
+        typeof arg === "function"
+          ? (arg as (tx: unknown) => unknown)({ measurement })
+          : Promise.all(arg as Promise<unknown>[]),
       ),
     },
   };
@@ -37,6 +40,10 @@ vi.mock("@/lib/crypto/note-cipher", () => ({
     s === null || s === undefined || s.length === 0
       ? null
       : new Uint8Array(Buffer.from(`enc:${s}`, "utf8")),
+  readNote: (c: Uint8Array | null | undefined, p: string | null | undefined) =>
+    c && c.byteLength > 0
+      ? Buffer.from(c).toString("utf8").slice(4)
+      : (p ?? null),
 }));
 
 vi.mock("@/lib/auth/audit", () => ({
@@ -115,8 +122,10 @@ beforeEach(() => {
     prisma.$transaction as unknown as {
       mockImplementation: (f: unknown) => void;
     }
-  ).mockImplementation(async (fn: (tx: unknown) => unknown) =>
-    fn({ measurement }),
+  ).mockImplementation(async (arg: unknown) =>
+    typeof arg === "function"
+      ? (arg as (tx: unknown) => unknown)({ measurement })
+      : Promise.all(arg as Promise<unknown>[]),
   );
 });
 
@@ -239,7 +248,19 @@ describe("POST /api/import/csv — batched write + per-row envelope", () => {
     // row matches like a live one; the update must carry the resurrection
     // (IMPORT rows are re-importable by design).
     mMeasurement().findMany.mockResolvedValue([
-      { type: "WEIGHT", externalId: "ext-tomb" },
+      {
+        type: "WEIGHT",
+        externalId: "ext-tomb",
+        // Identical values: only the tombstone differs, and that alone
+        // makes the line an update rather than a duplicate.
+        value: 81.0,
+        unit: "kg",
+        measuredAt: new Date("2026-05-01T08:00:00Z"),
+        glucoseContext: null,
+        notes: null,
+        notesEncrypted: null,
+        deletedAt: new Date("2026-05-02T08:00:00Z"),
+      },
     ]);
 
     const res = await POST(
@@ -424,3 +445,105 @@ describe("POST /api/import/csv — contextless blood glucose", () => {
     });
   });
 });
+
+/**
+ * v1.39.3 — re-importing the same file. A 9 000-row re-upload with an
+ * externalId column used to run one update per row inside a single
+ * interactive transaction and ended in a 500 when that outlived Prisma's 5 s
+ * timeout. An unchanged row is now a duplicate with no write, a changed one
+ * still updates, and lookups and writes run in bounded chunks.
+ */
+describe("POST /api/import/csv — re-import", () => {
+  beforeEach(() => {
+    vi.mocked(checkRateLimit).mockResolvedValue({
+      allowed: true,
+      remaining: 4,
+      resetAt: Date.now() + 3_600_000,
+    } as never);
+    mMeasurement().findMany.mockReset();
+    mMeasurement().createManyAndReturn.mockReset();
+    mMeasurement().createManyAndReturn.mockResolvedValue([]);
+    mMeasurement().updateMany.mockReset();
+    mMeasurement().updateMany.mockResolvedValue({ count: 1 });
+    vi.mocked(prisma.$transaction).mockClear();
+  });
+
+  function stored(externalId: string, overrides: Record<string, unknown> = {}) {
+    return {
+      type: "WEIGHT",
+      externalId,
+      value: 80.5,
+      unit: "kg",
+      measuredAt: new Date("2026-05-01T08:00:00Z"),
+      glucoseContext: null,
+      notes: null,
+      notesEncrypted: new Uint8Array(Buffer.from("enc:after run", "utf8")),
+      deletedAt: null,
+      ...overrides,
+    };
+  }
+
+  it("writes nothing for a row that is already stored exactly so", async () => {
+    mMeasurement().findMany.mockResolvedValue([stored("ext-1")]);
+    const res = await POST(
+      csvRequest(
+        [HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,after run,ext-1"].join(
+          "\n",
+        ),
+      ),
+    );
+    const body = (await res.json()) as CsvEnvelope;
+    expect(body.data).toMatchObject({ inserted: 0, updated: 0, skipped: 1 });
+    expect(body.data?.rows).toEqual([
+      { line: 2, status: "skipped", reason: "duplicate" },
+    ]);
+    expect(mMeasurement().updateMany).not.toHaveBeenCalled();
+    expect(mMeasurement().createManyAndReturn).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    ["value", "WEIGHT,81,kg,2026-05-01T08:00:00Z,,after run,ext-1"],
+    ["note", "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,before run,ext-1"],
+    ["instant", "WEIGHT,80.5,kg,2026-05-01T09:00:00Z,,after run,ext-1"],
+  ])("still updates a row whose %s changed", async (_what, line) => {
+    mMeasurement().findMany.mockResolvedValue([stored("ext-1")]);
+    const res = await POST(csvRequest([HEADER, line].join("\n")));
+    const body = (await res.json()) as CsvEnvelope;
+    expect(body.data).toMatchObject({ updated: 1, skipped: 0 });
+    expect(mMeasurement().updateMany).toHaveBeenCalledTimes(1);
+  });
+
+  it("probes per type in bounded chunks and writes updates in bounded batches", async () => {
+    const lines = Array.from({ length: 2500 }, (_, i) => {
+      const at = new Date(Date.UTC(2026, 0, 1) + i * 60_000).toISOString();
+      return `WEIGHT,${70 + (i % 10)},kg,${at},,,ext-${i}`;
+    });
+    // Every key exists, with a different value, so every line updates.
+    mMeasurement().findMany.mockImplementation((async (args: {
+      where: { externalId: { in: string[] } };
+    }) =>
+      args.where.externalId.in.map((externalId) =>
+        stored(externalId, { value: 1 }),
+      )) as never);
+
+    const res = await POST(csvRequest([HEADER, ...lines].join("\n")));
+    const body = (await res.json()) as CsvEnvelope;
+    expect(body.data?.updated).toBe(2500);
+
+    const probes = mMeasurement().findMany.mock.calls.map(
+      (c) => (c[0] as { where: { externalId: { in: string[] } } }).where,
+    );
+    expect(probes).toHaveLength(3);
+    expect(probes.every((w) => w.externalId.in.length <= 1000)).toBe(true);
+    // 2 500 updates in batches of at most 200, each its own transaction:
+    // no single transaction carries the whole file.
+    const batches = vi
+      .mocked(prisma.$transaction)
+      .mock.calls.map((c) => c[0] as unknown as unknown[]);
+    expect(batches).toHaveLength(13);
+    expect(batches.every((b) => Array.isArray(b) && b.length <= 200)).toBe(
+      true,
+    );
+    expect(mMeasurement().updateMany).toHaveBeenCalledTimes(2500);
+  });
+});

src/app/api/import/csv/route.ts

@@ -8,7 +8,7 @@ import {
   getClientIp,
   readBodyText,
 } from "@/lib/api-response";
-import { encryptNote } from "@/lib/crypto/note-cipher";
+import { encryptNote, readNote } from "@/lib/crypto/note-cipher";
 import { checkRateLimit } from "@/lib/rate-limit";
 import { NextRequest } from "next/server";
 
@@ -26,6 +26,63 @@ import type { NormalisedMeasurementRow } from "@/lib/import/csv-measurements";
 import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
 import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger";
 
+/** Keys per dedup probe: well under Postgres's bind-parameter cap. */
+const LOOKUP_CHUNK = 1000;
+/** Rows per create call / per update batch transaction. */
+const WRITE_CHUNK = 200;
+
+function chunked<T>(items: readonly T[], size: number): T[][] {
+  const out: T[][] = [];
+  for (let i = 0; i < items.length; i += size) {
+    out.push(items.slice(i, i + size));
+  }
+  return out;
+}
+
+/** Group rows' probe keys by measurement type, de-duplicated. */
+function groupByType<K>(
+  rows: ReadonlyArray<{ row: NormalisedMeasurementRow }>,
+  keyOf: (r: { row: NormalisedMeasurementRow }) => K,
+): Map<MeasurementType, K[]> {
+  const byType = new Map<MeasurementType, Map<string, K>>();
+  for (const r of rows) {
+    const type = r.row.type as MeasurementType;
+    const key = keyOf(r);
+    const bucket = byType.get(type) ?? new Map<string, K>();
+    bucket.set(key instanceof Date ? String(key.getTime()) : String(key), key);
+    byType.set(type, bucket);
+  }
+  return new Map([...byType].map(([type, keys]) => [type, [...keys.values()]]));
+}
+
+/**
+ * True when a stored IMPORT row already holds exactly what this line would
+ * write: same value, unit, instant, glucose context and note, and live. Such
+ * a line is a duplicate, not an update.
+ */
+function sameStoredRow(
+  stored: {
+    value: number;
+    unit: string;
+    measuredAt: Date;
+    glucoseContext: GlucoseContext | null;
+    notes: string | null;
+    notesEncrypted: Uint8Array | null;
+    deletedAt: Date | null;
+  },
+  m: NormalisedMeasurementRow,
+): boolean {
+  return (
+    stored.deletedAt === null &&
+    stored.value === m.value &&
+    stored.unit === m.unit &&
+    stored.measuredAt.getTime() === m.measuredAt.getTime() &&
+    stored.glucoseContext ===
+      ((m.glucoseContext as GlucoseContext | undefined) ?? null) &&
+    readNote(stored.notesEncrypted, stored.notes) === (m.notes || null)
+  );
+}
+
 /**
  * CSV measurement import (v1.17.1).
  *
@@ -141,38 +198,72 @@ export const POST = apiHandler(async (request: NextRequest) => {
     const plainRows = okRows.filter((r) => !r.row.externalId);
 
     // --- externalId rows: idempotent upsert on
-    // (userId, type, source=IMPORT, externalId). One probe fetches every
-    // pre-existing key; a hit means the row updates in place.
-    const extExisting =
-      extRows.length > 0
-        ? await prisma.measurement.findMany({
-            where: {
-              userId,
-              source: "IMPORT",
-              OR: extRows.map((r) => ({
-                type: r.row.type as MeasurementType,
-                externalId: r.row.externalId as string,
-              })),
-            },
-            select: { type: true, externalId: true },
-          })
-        : [];
-    const extExistingSet = new Set(
-      extExisting.map((e) => `${e.type}::${e.externalId}`),
-    );
+    // (userId, type, source=IMPORT, externalId). One probe per type and chunk
+    // fetches every pre-existing key with the columns an update would write,
+    // so a row whose stored values already match is left alone.
+    const extExisting = new Map<
+      string,
+      {
+        value: number;
+        unit: string;
+        measuredAt: Date;
+        glucoseContext: GlucoseContext | null;
+        notes: string | null;
+        notesEncrypted: Uint8Array | null;
+        deletedAt: Date | null;
+      }
+    >();
+    for (const [type, ids] of groupByType(
+      extRows,
+      (r) => r.row.externalId as string,
+    )) {
+      for (const chunk of chunked(ids, LOOKUP_CHUNK)) {
+        const found = await prisma.measurement.findMany({
+          where: {
+            userId,
+            source: "IMPORT",
+            type,
+            externalId: { in: chunk },
+          },
+          select: {
+            externalId: true,
+            value: true,
+            unit: true,
+            measuredAt: true,
+            glucoseContext: true,
+            notes: true,
+            notesEncrypted: true,
+            deletedAt: true,
+          },
+        });
+        for (const row of found) {
+          extExisting.set(`${type}::${row.externalId}`, row);
+        }
+      }
+    }
 
     // Partition in file order, honouring in-file repeats: the first sighting
     // of a brand-new key inserts, every later sighting (or any sighting of a
     // key already in the DB) updates in place. The last value for a key wins,
     // matching the old sequential-upsert overwrite. Insert/update maps are
     // keyed by (type, externalId) so a repeated key collapses to one write.
+    // A key already in the DB whose stored row equals this line is a
+    // duplicate: no write, reported like the natural-key duplicates below.
+    // That is what makes re-importing the same file cheap — a second upload
+    // of an unchanged 9 000-row export writes nothing.
     const seenExt = new Set<string>();
     const extInserts = new Map<string, NormalisedMeasurementRow>();
     const extUpdates = new Map<string, NormalisedMeasurementRow>();
     for (const result of extRows) {
       const m = result.row;
       const key = `${m.type}::${m.externalId}`;
-      if (extExistingSet.has(key)) {
+      const stored = extExisting.get(key);
+      if (stored && !extUpdates.has(key) && sameStoredRow(stored, m)) {
+        skipped++;
+        writeOutcome.set(result.line, "duplicate");
+        continue;
+      }
+      if (stored) {
         extUpdates.set(key, m);
         updated++;
         writeOutcome.set(result.line, "updated");
@@ -198,24 +289,28 @@ export const POST = apiHandler(async (request: NextRequest) => {
     // --- externalId-less rows: create-only, deduped on the natural unique key
     // (userId, type, measuredAt, source=IMPORT, sleepStage=null). A collision
     // — against the DB or an earlier row in this same file — is a duplicate,
-    // never an overwrite (each sample is a canonical reading).
-    const plainExisting =
-      plainRows.length > 0
-        ? await prisma.measurement.findMany({
-            where: {
-              userId,
-              source: "IMPORT",
-              OR: plainRows.map((r) => ({
-                type: r.row.type as MeasurementType,
-                measuredAt: r.row.measuredAt,
-              })),
-            },
-            select: { type: true, measuredAt: true },
-          })
-        : [];
-    const plainExistingSet = new Set(
-      plainExisting.map((e) => `${e.type}::${e.measuredAt.getTime()}`),
-    );
+    // never an overwrite (each sample is a canonical reading). Probed per type
+    // and chunk, like the externalId rows.
+    const plainExistingSet = new Set<string>();
+    for (const [type, instants] of groupByType(
+      plainRows,
+      (r) => r.row.measuredAt,
+    )) {
+      for (const chunk of chunked(instants, LOOKUP_CHUNK)) {
+        const found = await prisma.measurement.findMany({
+          where: {
+            userId,
+            source: "IMPORT",
+            type,
+            measuredAt: { in: chunk },
+          },
+          select: { measuredAt: true },
+        });
+        for (const row of found) {
+          plainExistingSet.add(`${type}::${row.measuredAt.getTime()}`);
+        }
+      }
+    }
     const seenPlain = new Set<string>();
     const plainInserts: NormalisedMeasurementRow[] = [];
     for (const result of plainRows) {
@@ -236,55 +331,65 @@ export const POST = apiHandler(async (request: NextRequest) => {
       }
     }
 
-    // Bulk write: createManyAndReturn the survivors (chunked under the PG
-    // parameter cap, skipDuplicates to absorb a concurrent double-submit
-    // race), then grouped updateMany for pre-existing externalId rows.
+    // Bulk write in bounded chunks, each its own short transaction.
+    //
+    // This used to be ONE interactive transaction around every create chunk
+    // AND one `updateMany` per re-imported row. Re-importing a 9 000-row file
+    // with an externalId column meant 9 000 serial updates inside it, which
+    // outlived Prisma's 5 s interactive-transaction timeout and ended in a
+    // 500. A chunk now commits on its own: creates go through
+    // `createManyAndReturn` (skipDuplicates absorbs a concurrent
+    // double-submit), updates through one batch transaction per chunk. A
+    // failure part-way leaves the earlier chunks written, and that is safe to
+    // retry: a re-upload of the same file skips what landed (unchanged
+    // externalId rows and natural-key duplicates are not written again).
     const toCreate: Prisma.MeasurementCreateManyInput[] = [
       ...[...extInserts.values()].map(buildCreateData),
       ...plainInserts.map(buildCreateData),
     ];
-    const CREATE_CHUNK = 200;
     let createdCount = 0;
     const createdRows: Array<{
       id: string;
       type: MeasurementType;
       measuredAt: Date;
     }> = [];
-    await prisma.$transaction(async (tx) => {
-      for (let i = 0; i < toCreate.length; i += CREATE_CHUNK) {
-        const rows = await tx.measurement.createManyAndReturn({
-          data: toCreate.slice(i, i + CREATE_CHUNK),
-          skipDuplicates: true,
-          select: { id: true, type: true, measuredAt: true },
-        });
-        createdRows.push(...rows);
-        createdCount += rows.length;
-      }
-      for (const m of extUpdates.values()) {
-        await tx.measurement.updateMany({
-          where: {
-            userId,
-            source: "IMPORT",
-            type: m.type as MeasurementType,
-            externalId: m.externalId as string,
-          },
-          data: {
-            value: m.value,
-            unit: m.unit,
-            measuredAt: m.measuredAt,
-            notes: null,
-            notesEncrypted: encryptNote(m.notes ?? null),
-            glucoseContext:
-              (m.glucoseContext as GlucoseContext | undefined) ?? null,
-            // No-op on a live row, a deliberate RESURRECTION on a
-            // tombstoned one — IMPORT rows are re-importable by design, so
-            // a re-imported externalId brings the row back (mirrors the
-            // source-owned sync resurrect rule).
-            deletedAt: null,
-          },
-        });
-      }
-    });
+    for (const chunk of chunked(toCreate, WRITE_CHUNK)) {
+      const rows = await prisma.measurement.createManyAndReturn({
+        data: chunk,
+        skipDuplicates: true,
+        select: { id: true, type: true, measuredAt: true },
+      });
+      createdRows.push(...rows);
+      createdCount += rows.length;
+    }
+    for (const chunk of chunked([...extUpdates.values()], WRITE_CHUNK)) {
+      await prisma.$transaction(
+        chunk.map((m) =>
+          prisma.measurement.updateMany({
+            where: {
+              userId,
+              source: "IMPORT",
+              type: m.type as MeasurementType,
+              externalId: m.externalId as string,
+            },
+            data: {
+              value: m.value,
+              unit: m.unit,
+              measuredAt: m.measuredAt,
+              notes: null,
+              notesEncrypted: encryptNote(m.notes ?? null),
+              glucoseContext:
+                (m.glucoseContext as GlucoseContext | undefined) ?? null,
+              // No-op on a live row, a deliberate RESURRECTION on a
+              // tombstoned one — IMPORT rows are re-importable by design, so
+              // a re-imported externalId brings the row back (mirrors the
+              // source-owned sync resurrect rule).
+              deletedAt: null,
+            },
+          }),
+        ),
+      );
+    }
 
     void emitInsertedMeasurementArrivals(
       userId,

src/app/api/labs/ocr/extract/__tests__/route.test.ts

@@ -66,6 +66,7 @@ import { rasterizePdf } from "@/lib/documents/rasterize-pdf";
 import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget";
 import { checkRateLimit, refundRateLimit } from "@/lib/rate-limit";
 import { OcrExtractError, runOcrExtraction } from "@/lib/labs/ocr-extract";
+import { annotate } from "@/lib/logging/context";
 
 const SESSION_OK = {
   session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
@@ -164,6 +165,29 @@ describe("POST /api/labs/ocr/extract — text mode budget", () => {
     expect(res.status).toBe(502);
     expect(body.error).toContain("configured AI provider");
   });
+
+  it("never passes the upstream body on, to the caller or to the event", async () => {
+    const upstreamBody = "ami-id instance-id iam/security-credentials/admin";
+    vi.mocked(runOcrExtraction).mockRejectedValue(
+      Object.assign(new Error("Local AI request failed (403)"), {
+        httpStatus: 403,
+        model: "llama3",
+        bodyExcerpt: upstreamBody,
+      }),
+    );
+
+    const res = await POST(textReq());
+    expect(res.status).toBe(502);
+    expect(await res.text()).not.toContain("iam/security-credentials");
+    const events = JSON.stringify(vi.mocked(annotate).mock.calls);
+    expect(events).not.toContain("iam/security-credentials");
+    // The status and the model stay, so an operator can still tell why.
+    expect(annotate).toHaveBeenCalledWith(
+      expect.objectContaining({
+        meta: expect.objectContaining({ upstreamStatus: 403, model: "llama3" }),
+      }),
+    );
+  });
 });
 
 describe("POST /api/labs/ocr/extract — vision PDF rasterization", () => {

src/app/api/labs/ocr/extract/route.ts

@@ -72,11 +72,16 @@ export const dynamic = "force-dynamic";
 /** OCR'd text is bounded in the schema; cap the JSON body proportionally. */
 const TEXT_BODY_MAX_BYTES = 512 * 1024;
 
+/**
+ * The wide-event facts of a failed provider call: the status and the model,
+ * never the upstream body. The base URL can be one a user typed, and what an
+ * arbitrary endpoint answers has no business in the event stream (v1.39.3);
+ * the response to the caller carries neither.
+ */
 function providerFailureMeta(error: unknown, mode?: "text") {
   const err = error as {
     httpStatus?: unknown;
     model?: unknown;
-    bodyExcerpt?: unknown;
   };
   return {
     reason: "provider_error",
@@ -85,9 +90,6 @@ function providerFailureMeta(error: unknown, mode?: "text") {
       ? { upstreamStatus: err.httpStatus }
       : {}),
     ...(typeof err.model === "string" ? { model: err.model } : {}),
-    ...(typeof err.bodyExcerpt === "string"
-      ? { upstreamError: err.bodyExcerpt }
-      : {}),
   };
 }
 

src/app/api/user/ai-provider/__tests__/legacy-private-host.test.ts

@@ -0,0 +1,126 @@
+/**
+ * v1.39.3 — `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is deprecated and narrowed,
+ * not removed. Saving a private base URL under it is still accepted for the
+ * operator's own configuration (an admin account) and refused for any other
+ * account, which needs an exact origin in `AI_PRIVATE_ORIGINS`. Metadata and
+ * link-local are refused for everyone. The real policy and the real public-host
+ * check run here; only persistence is stubbed.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@/lib/api-handler", () => ({
+  apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
+  requireAuth: vi.fn(async () => ({
+    user: { id: "u-1" },
+    session: { id: "s-1" },
+  })),
+  HttpError: class HttpError extends Error {
+    status: number;
+    constructor(status: number, message: string) {
+      super(message);
+      this.status = status;
+    }
+  },
+}));
+vi.mock("@/lib/db", () => ({
+  prisma: { user: { findUnique: vi.fn(), update: vi.fn() } },
+}));
+vi.mock("@/lib/crypto", () => ({
+  decrypt: vi.fn((v: string) => `dec:${v}`),
+  encrypt: vi.fn((v: string) => `enc:${v}`),
+}));
+vi.mock("@/lib/ai/provider", () => ({
+  resolveProviderAvailability: vi.fn(),
+}));
+vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+vi.mock("@/lib/ai/server-provider-health", () => ({
+  readServerProviderHealth: vi.fn(),
+}));
+vi.mock("@/lib/feature-flags", async () =>
+  (
+    await import("@/__tests__/helpers/assistant-switches-mock")
+  ).mockAssistantSwitches(vi.fn()),
+);
+vi.mock("@/lib/sharing/provider-work-authority", () => ({
+  providerWorkAuthorityForRecord: vi.fn(() => ({ origin: "owner" })),
+  providerCredentialPolicy: vi.fn(() => "personal"),
+}));
+vi.mock("@/lib/ai/consent-guard", () => ({
+  hasActiveConsentForSurface: vi.fn(),
+}));
+
+import { PATCH } from "../route";
+import { prisma } from "@/lib/db";
+import { _resetAiGrantsForTests } from "@/lib/ai/local-host-allowlist";
+
+const patch = PATCH as (req: Request) => Promise<Response>;
+
+function save(body: Record<string, string>) {
+  return patch(
+    new Request("http://localhost/api/user/ai-provider", {
+      method: "PATCH",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+    }),
+  );
+}
+
+function asRole(role: "ADMIN" | "USER") {
+  vi.mocked(prisma.user.findUnique).mockResolvedValue({ role } as never);
+}
+
+beforeEach(() => {
+  vi.mocked(prisma.user.findUnique).mockReset();
+  vi.mocked(prisma.user.update).mockReset();
+  vi.mocked(prisma.user.update).mockResolvedValue({} as never);
+  _resetAiGrantsForTests();
+  vi.stubEnv("AI_PRIVATE_ORIGINS", "");
+  vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
+  vi.spyOn(console, "warn").mockImplementation(() => {});
+});
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+  vi.restoreAllMocks();
+});
+
+describe("PATCH /api/user/ai-provider under ALLOW_LOCAL_AI_PRIVATE_HOSTS=true", () => {
+  it.each([
+    ["baseUrl", "http://10.0.0.5:11434/v1"],
+    ["compatBaseUrl", "http://192.168.1.20:4000/v1"],
+  ])("refuses a non-admin account's private %s", async (field, url) => {
+    asRole("USER");
+    const res = await save({ [field]: url });
+    expect(res.status).toBe(422);
+    expect((await res.json()).error).toContain("AI_PRIVATE_ORIGINS");
+    expect(prisma.user.update).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    ["baseUrl", "http://10.0.0.5:11434/v1"],
+    ["compatBaseUrl", "http://192.168.1.20:4000/v1"],
+  ])("accepts an admin account's private %s", async (field, url) => {
+    asRole("ADMIN");
+    const res = await save({ [field]: url });
+    expect(res.status).toBe(200);
+    expect(prisma.user.update).toHaveBeenCalledTimes(1);
+  });
+
+  it.each([
+    "http://169.254.169.254/latest/meta-data/",
+    "http://[fe80::1]:11434/v1",
+    "http://0.0.0.0:11434/v1",
+  ])("refuses %s even for an admin account", async (url) => {
+    asRole("ADMIN");
+    const res = await save({ baseUrl: url });
+    expect(res.status).toBe(422);
+    expect(prisma.user.update).not.toHaveBeenCalled();
+  });
+
+  it("accepts a non-admin account's private URL once its exact origin is granted", async () => {
+    vi.stubEnv("AI_PRIVATE_ORIGINS", "http://10.0.0.5:11434");
+    asRole("USER");
+    const res = await save({ baseUrl: "http://10.0.0.5:11434/v1" });
+    expect(res.status).toBe(200);
+  });
+});

src/app/api/user/ai-provider/route.ts

@@ -179,6 +179,26 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
   }
   const body = parsed.data;
 
+  // v1.39.3 — whose configuration this is. Settings saved on an admin account
+  // are the operator's own and are still covered by the deprecated
+  // `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true`; any other account needs an exact
+  // origin in `AI_PRIVATE_ORIGINS`. Read from the saved row, the same rule
+  // the provider resolution applies when it later dials the URL.
+  const needsHostPolicy =
+    (typeof body.baseUrl === "string" && body.baseUrl !== "") ||
+    (typeof body.compatBaseUrl === "string" && body.compatBaseUrl !== "");
+  const owner = needsHostPolicy
+    ? {
+        operatorTrusted:
+          (
+            await prisma.user.findUnique({
+              where: { id: user.id },
+              select: { role: true },
+            })
+          )?.role === "ADMIN",
+      }
+    : {};
+
   const updates: Record<string, unknown> = {};
 
   if (body.provider !== undefined) {
@@ -197,14 +217,15 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
       const trimmed = body.baseUrl.trim();
       // SSRF guard: by default reject private/internal hostnames so a
       // compromised user account cannot point the server at the cloud
-      // metadata endpoint or internal admin panels. v1.18.7 (SECURITY LOW) —
-      // ops opt in via `ALLOW_LOCAL_AI_PRIVATE_HOSTS`, now a host allowlist:
-      // `true` permits any private host (legacy), a comma-separated host list
-      // permits only those exact hostnames.
-      const allowPrivate = isLocalAiHostAllowed(trimmed);
+      // metadata endpoint or internal admin panels. The operator grants a
+      // private endpoint by exact origin (`AI_PRIVATE_ORIGINS`, or the legacy
+      // host list); the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` covers
+      // an admin account's own settings only (`owner` above), and no grant
+      // can open metadata or link-local.
+      const allowPrivate = isLocalAiHostAllowed(trimmed, owner);
       if (!allowPrivate && !isPublicUrl(trimmed)) {
         return apiError(
-          "Base URL points to an internal/private host. Ops must allow it on this instance via ALLOW_LOCAL_AI_PRIVATE_HOSTS — set it to the exact host (e.g. ollama.lan) or to true for any private host (intended for self-hosted Ollama / LM Studio).",
+          "Base URL points to an internal/private host. The operator must allow its exact origin on this instance via AI_PRIVATE_ORIGINS (e.g. http://ollama.lan:11434) for a self-hosted Ollama / LM Studio.",
           422,
         );
       }
@@ -226,9 +247,9 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
       // private one only when the operator allowlisted it. Gateways on a LAN
       // are the normal case for LiteLLM / vLLM, so the escape hatch matters
       // here as much as it does for Ollama.
-      if (!isLocalAiHostAllowed(trimmed) && !isPublicUrl(trimmed)) {
+      if (!isLocalAiHostAllowed(trimmed, owner) && !isPublicUrl(trimmed)) {
         return apiError(
-          "Base URL points to an internal/private host. Ops must allow it on this instance via ALLOW_LOCAL_AI_PRIVATE_HOSTS — set it to the exact host (e.g. litellm.lan) or to true for any private host (intended for a self-hosted gateway).",
+          "Base URL points to an internal/private host. The operator must allow its exact origin on this instance via AI_PRIVATE_ORIGINS (e.g. http://litellm.lan:4000) for a self-hosted gateway.",
           422,
         );
       }