Merge branch 'fix/encrypt-free-text-and-local-ai' into release/v1.39.3

# Conflicts:
#	src/app/api/import/csv/route.ts
This commit is contained in:
Marc-André Bombeck committed 2026-09-26 21:50:05 +02:00
commit 66b1aef40c
69 files changed
+3625 -367

No files matched your search

@@ -23,10 +23,11 @@
*
* ## Why the matchers are what they are
*
* A grep for `process.env.X` alone is not the read set. Four modules resolve a
* name through a helper and index `process.env` dynamically, so their reads
* are invisible to the direct matcher — `ALLOW_LOCAL_AI_PRIVATE_HOSTS` and
* every `OIDC_*` among them. Each is registered below with its own call-shape
* A grep for `process.env.X` alone is not the read set. Three modules resolve
* a name through a helper and index `process.env` dynamically, so their reads
* are invisible to the direct matcher — every `OIDC_*` among them. (The AI
* private-host grants were a fourth until v1.39.3, when they became direct
* `process.env` reads.) Each is registered below with its own call-shape
* matcher AND its own non-zero assertion, so a refactor that renames the
* helper fails this file loudly instead of quietly shrinking the read set.
*
@@ -79,12 +80,6 @@ const INDIRECT_READERS: ReadonlyArray<{
minimum: 1,
why: "the retention windows are parsed through a local intEnv() helper",
},
{
file: "lib/ai/local-host-allowlist.ts",
pattern: /ENV_VAR\s*=\s*["']([A-Z][A-Z0-9_]*)["']/g,
minimum: 1,
why: "the name is hoisted into an ENV_VAR constant, then indexed",
},
{
file: "lib/boot/readiness-summary.ts",
pattern: /\benv\.([A-Z][A-Z0-9_]*)/g,
@@ -0,0 +1,71 @@
/**
* The native OIDC leg ends at `healthlog://oidc-callback?error=<reason>`, and
* the app maps each reason to its own message. The only written contract for
* that vocabulary is the login route's 302 description in the OpenAPI
* registry (the callback itself is provider-driven and has no entry).
*
* An app build once matched against hyphenated shorthand from a coordination
* note (`identity-conflict`) while the server sent `oidc_identity_conflict`,
* so every reason fell through to the generic message. This binds the two
* ends that can be read exactly: every reason either route can send on a
* redirect a native client may receive is named in that description, and
* nothing is named there that no route sends.
*
* Its limit: it reads string literals passed to the redirect helpers, so a
* reason assembled at runtime would slip past it. None is today.
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
const ROOT = join(__dirname, "..");
const read = (rel: string) => readFileSync(join(ROOT, rel), "utf8");
/** Literal reasons passed to one of the named redirect helpers. */
function reasons(source: string, helpers: string[]): Set<string> {
const found = new Set<string>();
const pattern = new RegExp(
`\\b(?:${helpers.join("|")})\\(\\s*"(oidc_[a-z_]+)"\\s*\\)`,
"g",
);
for (const m of source.matchAll(pattern)) found.add(m[1]);
return found;
}
describe("native OIDC error reasons are documented", () => {
// The login route answers both legs through `loginError`; the callback's
// native-aware helper is `failRedirect`. `errorRedirect` in the callback is
// the browser-only branch taken before the state (and so the native flag)
// could be read, and never reaches the app.
const sent = new Set([
...reasons(read("app/api/auth/oidc/login/route.ts"), ["loginError"]),
...reasons(read("app/api/auth/oidc/callback/route.ts"), ["failRedirect"]),
]);
// The login route's 302 description is one string literal on one line.
const description =
read("lib/openapi/routes/auth.ts")
.split("\n")
.find(
(line) =>
line.includes("healthlog://oidc-callback?error=<reason>") &&
line.includes("oidc_invalid_request"),
) ?? "";
const documented = new Set(
[...description.matchAll(/`(oidc_[a-z_]+)`/g)].map((m) => m[1]),
);
it("finds the reasons on both ends", () => {
expect(sent.size).toBeGreaterThanOrEqual(8);
expect(documented.size).toBeGreaterThanOrEqual(8);
});
it("documents every reason a route sends", () => {
expect([...sent].filter((r) => !documented.has(r))).toEqual([]);
});
it("documents no reason that no route sends", () => {
expect([...documented].filter((r) => !sent.has(r))).toEqual([]);
});
});
@@ -6,7 +6,7 @@
* HTTPS + hostname allowlist on the base URL, and the masked
* `keyPreview` in every response.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { NextRequest } from "next/server";
vi.mock("@/lib/db", () => ({
@@ -15,6 +15,7 @@ vi.mock("@/lib/db", () => ({
findUnique: vi.fn(),
upsert: vi.fn(),
},
user: { findMany: vi.fn().mockResolvedValue([]) },
},
}));
@@ -100,6 +101,7 @@ describe("GET /api/admin/ai-settings", () => {
keyPreview: null,
model: "gpt-4o",
baseUrl: "https://api.openai.com/v1",
legacyPrivateHostGrant: null,
});
});
@@ -216,3 +218,52 @@ describe("PUT /api/admin/ai-settings", () => {
});
});
});
/**
* v1.39.3 — while the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is set,
* the page names it and lists the origins it is standing in for, so the
* operator can copy them into `AI_PRIVATE_ORIGINS`.
*/
describe("GET /api/admin/ai-settings — deprecated private-host grant", () => {
beforeEach(async () => {
const { _resetAiGrantsForTests } =
await import("@/lib/ai/local-host-allowlist");
_resetAiGrantsForTests();
vi.mocked(requireAdmin).mockResolvedValue(ADMIN_CTX as never);
vi.mocked(prisma.appSettings.findUnique).mockResolvedValue({
adminAiKeyEncrypted: null,
adminAiModel: null,
adminAiBaseUrl: "http://gateway.lan:4000/v1",
} as never);
vi.mocked(prisma.user.findMany).mockResolvedValue([
{ aiBaseUrl: "http://10.0.0.5:11434/v1", aiCompatBaseUrl: null },
{
aiBaseUrl: "http://10.0.0.5:11434/v1/",
aiCompatBaseUrl: "https://openrouter.ai/api/v1",
},
{ aiBaseUrl: "http://169.254.169.254/latest", aiCompatBaseUrl: null },
] as never);
vi.stubEnv("AI_PRIVATE_ORIGINS", "");
vi.spyOn(console, "warn").mockImplementation(() => {});
});
afterEach(() => {
vi.unstubAllEnvs();
});
it("lists the private origins in use, once each, when `true` is set", async () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
const res = await GET();
const body = await res.json();
expect(body.data.legacyPrivateHostGrant).toEqual({
originsInUse: ["http://10.0.0.5:11434", "http://gateway.lan:4000"],
});
});
it("is null when `true` is not set", async () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "ollama.lan");
const res = await GET();
expect((await res.json()).data.legacyPrivateHostGrant).toBeNull();
expect(prisma.user.findMany).not.toHaveBeenCalled();
});
});
+6
View File
@@ -7,6 +7,7 @@ import { apiSuccess, getClientIp, safeJson } from "@/lib/api-response";
import { encrypt, decrypt } from "@/lib/crypto";
import { checkRateLimit } from "@/lib/rate-limit";
import { annotate } from "@/lib/logging/context";
import { legacyPrivateHostReport } from "@/lib/ai/legacy-private-host-report";
export const dynamic = "force-dynamic";
@@ -29,6 +30,11 @@ export const GET = apiHandler(async () => {
: null,
model,
baseUrl,
// v1.39.3 — present while the deprecated ALLOW_LOCAL_AI_PRIVATE_HOSTS=true
// is set, naming the origins to move into AI_PRIVATE_ORIGINS.
legacyPrivateHostGrant: await legacyPrivateHostReport(
settings?.adminAiBaseUrl,
),
});
});
@@ -12,6 +12,7 @@ import {
import { auditLog } from "@/lib/auth/audit";
import { invalidateUserCorrelationPatterns } from "@/lib/cache/invalidate";
import { serialiseCustomMetricEntry } from "@/lib/custom-metrics/custom-metric-store";
import { encryptNote } from "@/lib/crypto/note-cipher";
import { prisma } from "@/lib/db";
import { annotate } from "@/lib/logging/context";
import { updateCustomMetricEntrySchema } from "@/lib/validations/custom-metrics";
@@ -83,7 +84,13 @@ export const PATCH = apiHandler(
const data: Record<string, unknown> = {};
if (d.value !== undefined) data.value = d.value;
if (d.measuredAt !== undefined) data.measuredAt = d.measuredAt;
if (d.note !== undefined) data.note = d.note;
// v1.39.3 — the note is stored encrypted. Any note write (a new value or
// an explicit clear) also nulls the legacy readable column, so an edited
// row never keeps its old text beside the new ciphertext.
if (d.note !== undefined) {
data.noteEncrypted = encryptNote(d.note);
data.note = null;
}
const updated = await prisma.customMetricEntry.update({
where: { id: entryId },
@@ -0,0 +1,238 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
vi.mock("@/lib/rate-limit", async (importOriginal) => ({
...(await importOriginal<typeof import("@/lib/rate-limit")>()),
// The shared single-record write ceiling. Allowed here: this file is about
// what the handler does with a body it accepted, not about the bucket.
checkRecordWriteRateLimit: vi.fn(async () => ({
allowed: true,
limit: 300,
remaining: 299,
resetAt: Date.now() + 60_000,
})),
}));
vi.mock("@/lib/cache/invalidate", () => ({
invalidateUserCorrelationPatterns: vi.fn(),
}));
vi.mock("@/lib/db", () => ({
prisma: {
customMetric: { findFirst: vi.fn() },
customMetricEntry: {
findMany: vi.fn(),
count: vi.fn(),
create: vi.fn(),
findFirst: vi.fn(),
update: vi.fn(),
},
auditLog: { create: vi.fn().mockResolvedValue({}) },
},
}));
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
vi.mock("@/lib/auth/audit", () => ({
auditLog: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() }));
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/idempotency", () => ({
withIdempotency: (fn: unknown) => fn,
}));
vi.mock("next/headers", () => ({
headers: vi.fn(async () => ({ get: () => null })),
cookies: vi.fn(async () => ({
get: () => undefined,
set: () => {},
delete: () => {},
})),
}));
import { GET, POST } from "../route";
import { PATCH } from "../[entryId]/route";
import { getSession } from "@/lib/auth/session";
import { prisma } from "@/lib/db";
import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec";
/**
* v1.39.3 — the note on a custom-metric reading is encrypted at rest. A write
* stores ciphertext and leaves the readable column empty; a read opens the
* ciphertext and falls back to the readable column only for a row the backfill
* has not reached.
*/
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
const SESSION_OK = {
session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
user: { id: "user-1", username: "u", role: "USER" as const, locale: "en" },
};
const ENTRY = {
id: "e-1",
userId: "user-1",
customMetricId: "cm-1",
value: 42,
unit: "kg",
measuredAt: new Date("2026-06-10T00:00:00.000Z"),
note: null as string | null,
noteEncrypted: null as Uint8Array | null,
createdAt: new Date("2026-06-10T00:00:00.000Z"),
deletedAt: null,
};
function jsonRequest(url: string, method: string, body: unknown) {
return new NextRequest(url, {
method,
body: JSON.stringify(body),
headers: { "content-type": "application/json" },
});
}
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never);
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
});
describe("custom-metric reading note at rest", () => {
it("POST stores the note as ciphertext and answers with the readable note", async () => {
vi.mocked(prisma.customMetric.findFirst).mockResolvedValue({
id: "cm-1",
unit: "kg",
} as never);
vi.mocked(prisma.customMetricEntry.create).mockImplementation(
(async (args: { data: Record<string, unknown> }) => ({
...ENTRY,
...args.data,
})) as never,
);
const res = await POST(
jsonRequest("http://localhost/api/custom-metrics/cm-1/entries", "POST", {
value: 42,
measuredAt: "2026-06-10T00:00:00.000Z",
note: "after the long ride",
}),
{ params: Promise.resolve({ id: "cm-1" }) },
);
expect(res.status).toBe(201);
const data = vi.mocked(prisma.customMetricEntry.create).mock.calls[0][0]
.data as Record<string, unknown>;
expect(data).not.toHaveProperty("note");
expect(decryptFromBytes(data.noteEncrypted as Uint8Array)).toBe(
"after the long ride",
);
const body = await res.json();
expect(body.data.note).toBe("after the long ride");
expect(body.data).not.toHaveProperty("noteEncrypted");
});
it("GET opens sealed notes and still shows a legacy readable one", async () => {
vi.mocked(prisma.customMetric.findFirst).mockResolvedValue({
id: "cm-1",
} as never);
vi.mocked(prisma.customMetricEntry.findMany).mockResolvedValue([
{ ...ENTRY, id: "e-1", noteEncrypted: encryptToBytes("sealed note") },
{ ...ENTRY, id: "e-2", note: "legacy note" },
{ ...ENTRY, id: "e-3" },
] as never);
vi.mocked(prisma.customMetricEntry.count).mockResolvedValue(3 as never);
const res = await GET(
new NextRequest("http://localhost/api/custom-metrics/cm-1/entries"),
{ params: Promise.resolve({ id: "cm-1" }) },
);
expect(res.status).toBe(200);
const body = await res.json();
expect(
body.data.entries.map((e: { note: string | null }) => e.note),
).toEqual(["sealed note", "legacy note", null]);
for (const entry of body.data.entries) {
expect(entry).not.toHaveProperty("noteEncrypted");
}
});
it("PATCH seals a new note and clears the legacy readable column", async () => {
vi.mocked(prisma.customMetricEntry.findFirst).mockResolvedValue({
...ENTRY,
note: "old readable note",
} as never);
vi.mocked(prisma.customMetricEntry.update).mockImplementation(
(async (args: { data: Record<string, unknown> }) => ({
...ENTRY,
...args.data,
})) as never,
);
const res = await PATCH(
jsonRequest(
"http://localhost/api/custom-metrics/cm-1/entries/e-1",
"PATCH",
{ note: "edited" },
),
{ params: Promise.resolve({ id: "cm-1", entryId: "e-1" }) },
);
expect(res.status).toBe(200);
const data = vi.mocked(prisma.customMetricEntry.update).mock.calls[0][0]
.data as Record<string, unknown>;
expect(data.note).toBeNull();
expect(decryptFromBytes(data.noteEncrypted as Uint8Array)).toBe("edited");
expect((await res.json()).data.note).toBe("edited");
});
it("PATCH with an explicit null clears both columns", async () => {
vi.mocked(prisma.customMetricEntry.findFirst).mockResolvedValue({
...ENTRY,
noteEncrypted: encryptToBytes("to be cleared"),
} as never);
vi.mocked(prisma.customMetricEntry.update).mockImplementation(
(async (args: { data: Record<string, unknown> }) => ({
...ENTRY,
...args.data,
})) as never,
);
const res = await PATCH(
jsonRequest(
"http://localhost/api/custom-metrics/cm-1/entries/e-1",
"PATCH",
{ note: null },
),
{ params: Promise.resolve({ id: "cm-1", entryId: "e-1" }) },
);
expect(res.status).toBe(200);
const data = vi.mocked(prisma.customMetricEntry.update).mock.calls[0][0]
.data as Record<string, unknown>;
expect(data).toMatchObject({ note: null, noteEncrypted: null });
expect((await res.json()).data.note).toBeNull();
});
it("PATCH without a note leaves both note columns untouched", async () => {
vi.mocked(prisma.customMetricEntry.findFirst).mockResolvedValue(
ENTRY as never,
);
vi.mocked(prisma.customMetricEntry.update).mockResolvedValue(
ENTRY as never,
);
await PATCH(
jsonRequest(
"http://localhost/api/custom-metrics/cm-1/entries/e-1",
"PATCH",
{ value: 43 },
),
{ params: Promise.resolve({ id: "cm-1", entryId: "e-1" }) },
);
const data = vi.mocked(prisma.customMetricEntry.update).mock.calls[0][0]
.data as Record<string, unknown>;
expect(data).not.toHaveProperty("note");
expect(data).not.toHaveProperty("noteEncrypted");
});
});
@@ -13,6 +13,7 @@ import { checkRecordWriteRateLimit } from "@/lib/rate-limit";
import { auditLog } from "@/lib/auth/audit";
import { invalidateUserCorrelationPatterns } from "@/lib/cache/invalidate";
import { serialiseCustomMetricEntry } from "@/lib/custom-metrics/custom-metric-store";
import { encryptNote } from "@/lib/crypto/note-cipher";
import { prisma } from "@/lib/db";
import { withIdempotency } from "@/lib/idempotency";
import { annotate } from "@/lib/logging/context";
@@ -161,7 +162,8 @@ async function postCustomMetricEntry(
value,
unit: metric.unit,
measuredAt,
note: note ?? null,
// v1.39.3 — the note is stored encrypted; the readable column stays NULL.
noteEncrypted: encryptNote(note),
},
});
+130 -7
View File
@@ -17,10 +17,13 @@ vi.mock("@/lib/db", () => {
return {
prisma: {
measurement,
// Run the batched write callback against the same measurement mock so
// createManyAndReturn / updateMany calls are observable.
$transaction: vi.fn(async (fn: (tx: unknown) => unknown) =>
fn({ measurement }),
// Both transaction forms against the same measurement mock so
// createManyAndReturn / updateMany calls are observable: a callback
// gets the mock as `tx`, a batch array is simply awaited.
$transaction: vi.fn(async (arg: unknown) =>
typeof arg === "function"
? (arg as (tx: unknown) => unknown)({ measurement })
: Promise.all(arg as Promise<unknown>[]),
),
},
};
@@ -37,6 +40,10 @@ vi.mock("@/lib/crypto/note-cipher", () => ({
s === null || s === undefined || s.length === 0
? null
: new Uint8Array(Buffer.from(`enc:${s}`, "utf8")),
readNote: (c: Uint8Array | null | undefined, p: string | null | undefined) =>
c && c.byteLength > 0
? Buffer.from(c).toString("utf8").slice(4)
: (p ?? null),
}));
vi.mock("@/lib/auth/audit", () => ({
@@ -115,8 +122,10 @@ beforeEach(() => {
prisma.$transaction as unknown as {
mockImplementation: (f: unknown) => void;
}
).mockImplementation(async (fn: (tx: unknown) => unknown) =>
fn({ measurement }),
).mockImplementation(async (arg: unknown) =>
typeof arg === "function"
? (arg as (tx: unknown) => unknown)({ measurement })
: Promise.all(arg as Promise<unknown>[]),
);
});
@@ -239,7 +248,19 @@ describe("POST /api/import/csv — batched write + per-row envelope", () => {
// row matches like a live one; the update must carry the resurrection
// (IMPORT rows are re-importable by design).
mMeasurement().findMany.mockResolvedValue([
{ type: "WEIGHT", externalId: "ext-tomb" },
{
type: "WEIGHT",
externalId: "ext-tomb",
// Identical values: only the tombstone differs, and that alone
// makes the line an update rather than a duplicate.
value: 81.0,
unit: "kg",
measuredAt: new Date("2026-05-01T08:00:00Z"),
glucoseContext: null,
notes: null,
notesEncrypted: null,
deletedAt: new Date("2026-05-02T08:00:00Z"),
},
]);
const res = await POST(
@@ -424,3 +445,105 @@ describe("POST /api/import/csv — contextless blood glucose", () => {
});
});
});
/**
* v1.39.3 — re-importing the same file. A 9 000-row re-upload with an
* externalId column used to run one update per row inside a single
* interactive transaction and ended in a 500 when that outlived Prisma's 5 s
* timeout. An unchanged row is now a duplicate with no write, a changed one
* still updates, and lookups and writes run in bounded chunks.
*/
describe("POST /api/import/csv — re-import", () => {
beforeEach(() => {
vi.mocked(checkRateLimit).mockResolvedValue({
allowed: true,
remaining: 4,
resetAt: Date.now() + 3_600_000,
} as never);
mMeasurement().findMany.mockReset();
mMeasurement().createManyAndReturn.mockReset();
mMeasurement().createManyAndReturn.mockResolvedValue([]);
mMeasurement().updateMany.mockReset();
mMeasurement().updateMany.mockResolvedValue({ count: 1 });
vi.mocked(prisma.$transaction).mockClear();
});
function stored(externalId: string, overrides: Record<string, unknown> = {}) {
return {
type: "WEIGHT",
externalId,
value: 80.5,
unit: "kg",
measuredAt: new Date("2026-05-01T08:00:00Z"),
glucoseContext: null,
notes: null,
notesEncrypted: new Uint8Array(Buffer.from("enc:after run", "utf8")),
deletedAt: null,
...overrides,
};
}
it("writes nothing for a row that is already stored exactly so", async () => {
mMeasurement().findMany.mockResolvedValue([stored("ext-1")]);
const res = await POST(
csvRequest(
[HEADER, "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,after run,ext-1"].join(
"\n",
),
),
);
const body = (await res.json()) as CsvEnvelope;
expect(body.data).toMatchObject({ inserted: 0, updated: 0, skipped: 1 });
expect(body.data?.rows).toEqual([
{ line: 2, status: "skipped", reason: "duplicate" },
]);
expect(mMeasurement().updateMany).not.toHaveBeenCalled();
expect(mMeasurement().createManyAndReturn).not.toHaveBeenCalled();
});
it.each([
["value", "WEIGHT,81,kg,2026-05-01T08:00:00Z,,after run,ext-1"],
["note", "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,before run,ext-1"],
["instant", "WEIGHT,80.5,kg,2026-05-01T09:00:00Z,,after run,ext-1"],
])("still updates a row whose %s changed", async (_what, line) => {
mMeasurement().findMany.mockResolvedValue([stored("ext-1")]);
const res = await POST(csvRequest([HEADER, line].join("\n")));
const body = (await res.json()) as CsvEnvelope;
expect(body.data).toMatchObject({ updated: 1, skipped: 0 });
expect(mMeasurement().updateMany).toHaveBeenCalledTimes(1);
});
it("probes per type in bounded chunks and writes updates in bounded batches", async () => {
const lines = Array.from({ length: 2500 }, (_, i) => {
const at = new Date(Date.UTC(2026, 0, 1) + i * 60_000).toISOString();
return `WEIGHT,${70 + (i % 10)},kg,${at},,,ext-${i}`;
});
// Every key exists, with a different value, so every line updates.
mMeasurement().findMany.mockImplementation((async (args: {
where: { externalId: { in: string[] } };
}) =>
args.where.externalId.in.map((externalId) =>
stored(externalId, { value: 1 }),
)) as never);
const res = await POST(csvRequest([HEADER, ...lines].join("\n")));
const body = (await res.json()) as CsvEnvelope;
expect(body.data?.updated).toBe(2500);
const probes = mMeasurement().findMany.mock.calls.map(
(c) => (c[0] as { where: { externalId: { in: string[] } } }).where,
);
expect(probes).toHaveLength(3);
expect(probes.every((w) => w.externalId.in.length <= 1000)).toBe(true);
// 2 500 updates in batches of at most 200, each its own transaction:
// no single transaction carries the whole file.
const batches = vi
.mocked(prisma.$transaction)
.mock.calls.map((c) => c[0] as unknown as unknown[]);
expect(batches).toHaveLength(13);
expect(batches.every((b) => Array.isArray(b) && b.length <= 200)).toBe(
true,
);
expect(mMeasurement().updateMany).toHaveBeenCalledTimes(2500);
});
});
+183 -78
View File
@@ -8,7 +8,7 @@ import {
getClientIp,
readBodyText,
} from "@/lib/api-response";
import { encryptNote } from "@/lib/crypto/note-cipher";
import { encryptNote, readNote } from "@/lib/crypto/note-cipher";
import { checkRateLimit } from "@/lib/rate-limit";
import { NextRequest } from "next/server";
@@ -26,6 +26,63 @@ import type { NormalisedMeasurementRow } from "@/lib/import/csv-measurements";
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger";
/** Keys per dedup probe: well under Postgres's bind-parameter cap. */
const LOOKUP_CHUNK = 1000;
/** Rows per create call / per update batch transaction. */
const WRITE_CHUNK = 200;
function chunked<T>(items: readonly T[], size: number): T[][] {
const out: T[][] = [];
for (let i = 0; i < items.length; i += size) {
out.push(items.slice(i, i + size));
}
return out;
}
/** Group rows' probe keys by measurement type, de-duplicated. */
function groupByType<K>(
rows: ReadonlyArray<{ row: NormalisedMeasurementRow }>,
keyOf: (r: { row: NormalisedMeasurementRow }) => K,
): Map<MeasurementType, K[]> {
const byType = new Map<MeasurementType, Map<string, K>>();
for (const r of rows) {
const type = r.row.type as MeasurementType;
const key = keyOf(r);
const bucket = byType.get(type) ?? new Map<string, K>();
bucket.set(key instanceof Date ? String(key.getTime()) : String(key), key);
byType.set(type, bucket);
}
return new Map([...byType].map(([type, keys]) => [type, [...keys.values()]]));
}
/**
* True when a stored IMPORT row already holds exactly what this line would
* write: same value, unit, instant, glucose context and note, and live. Such
* a line is a duplicate, not an update.
*/
function sameStoredRow(
stored: {
value: number;
unit: string;
measuredAt: Date;
glucoseContext: GlucoseContext | null;
notes: string | null;
notesEncrypted: Uint8Array | null;
deletedAt: Date | null;
},
m: NormalisedMeasurementRow,
): boolean {
return (
stored.deletedAt === null &&
stored.value === m.value &&
stored.unit === m.unit &&
stored.measuredAt.getTime() === m.measuredAt.getTime() &&
stored.glucoseContext ===
((m.glucoseContext as GlucoseContext | undefined) ?? null) &&
readNote(stored.notesEncrypted, stored.notes) === (m.notes || null)
);
}
/**
* CSV measurement import (v1.17.1).
*
@@ -141,38 +198,72 @@ export const POST = apiHandler(async (request: NextRequest) => {
const plainRows = okRows.filter((r) => !r.row.externalId);
// --- externalId rows: idempotent upsert on
// (userId, type, source=IMPORT, externalId). One probe fetches every
// pre-existing key; a hit means the row updates in place.
const extExisting =
extRows.length > 0
? await prisma.measurement.findMany({
where: {
userId,
source: "IMPORT",
OR: extRows.map((r) => ({
type: r.row.type as MeasurementType,
externalId: r.row.externalId as string,
})),
},
select: { type: true, externalId: true },
})
: [];
const extExistingSet = new Set(
extExisting.map((e) => `${e.type}::${e.externalId}`),
);
// (userId, type, source=IMPORT, externalId). One probe per type and chunk
// fetches every pre-existing key with the columns an update would write,
// so a row whose stored values already match is left alone.
const extExisting = new Map<
string,
{
value: number;
unit: string;
measuredAt: Date;
glucoseContext: GlucoseContext | null;
notes: string | null;
notesEncrypted: Uint8Array | null;
deletedAt: Date | null;
}
>();
for (const [type, ids] of groupByType(
extRows,
(r) => r.row.externalId as string,
)) {
for (const chunk of chunked(ids, LOOKUP_CHUNK)) {
const found = await prisma.measurement.findMany({
where: {
userId,
source: "IMPORT",
type,
externalId: { in: chunk },
},
select: {
externalId: true,
value: true,
unit: true,
measuredAt: true,
glucoseContext: true,
notes: true,
notesEncrypted: true,
deletedAt: true,
},
});
for (const row of found) {
extExisting.set(`${type}::${row.externalId}`, row);
}
}
}
// Partition in file order, honouring in-file repeats: the first sighting
// of a brand-new key inserts, every later sighting (or any sighting of a
// key already in the DB) updates in place. The last value for a key wins,
// matching the old sequential-upsert overwrite. Insert/update maps are
// keyed by (type, externalId) so a repeated key collapses to one write.
// A key already in the DB whose stored row equals this line is a
// duplicate: no write, reported like the natural-key duplicates below.
// That is what makes re-importing the same file cheap — a second upload
// of an unchanged 9 000-row export writes nothing.
const seenExt = new Set<string>();
const extInserts = new Map<string, NormalisedMeasurementRow>();
const extUpdates = new Map<string, NormalisedMeasurementRow>();
for (const result of extRows) {
const m = result.row;
const key = `${m.type}::${m.externalId}`;
if (extExistingSet.has(key)) {
const stored = extExisting.get(key);
if (stored && !extUpdates.has(key) && sameStoredRow(stored, m)) {
skipped++;
writeOutcome.set(result.line, "duplicate");
continue;
}
if (stored) {
extUpdates.set(key, m);
updated++;
writeOutcome.set(result.line, "updated");
@@ -198,24 +289,28 @@ export const POST = apiHandler(async (request: NextRequest) => {
// --- externalId-less rows: create-only, deduped on the natural unique key
// (userId, type, measuredAt, source=IMPORT, sleepStage=null). A collision
// — against the DB or an earlier row in this same file — is a duplicate,
// never an overwrite (each sample is a canonical reading).
const plainExisting =
plainRows.length > 0
? await prisma.measurement.findMany({
where: {
userId,
source: "IMPORT",
OR: plainRows.map((r) => ({
type: r.row.type as MeasurementType,
measuredAt: r.row.measuredAt,
})),
},
select: { type: true, measuredAt: true },
})
: [];
const plainExistingSet = new Set(
plainExisting.map((e) => `${e.type}::${e.measuredAt.getTime()}`),
);
// never an overwrite (each sample is a canonical reading). Probed per type
// and chunk, like the externalId rows.
const plainExistingSet = new Set<string>();
for (const [type, instants] of groupByType(
plainRows,
(r) => r.row.measuredAt,
)) {
for (const chunk of chunked(instants, LOOKUP_CHUNK)) {
const found = await prisma.measurement.findMany({
where: {
userId,
source: "IMPORT",
type,
measuredAt: { in: chunk },
},
select: { measuredAt: true },
});
for (const row of found) {
plainExistingSet.add(`${type}::${row.measuredAt.getTime()}`);
}
}
}
const seenPlain = new Set<string>();
const plainInserts: NormalisedMeasurementRow[] = [];
for (const result of plainRows) {
@@ -236,55 +331,65 @@ export const POST = apiHandler(async (request: NextRequest) => {
}
}
// Bulk write: createManyAndReturn the survivors (chunked under the PG
// parameter cap, skipDuplicates to absorb a concurrent double-submit
// race), then grouped updateMany for pre-existing externalId rows.
// Bulk write in bounded chunks, each its own short transaction.
//
// This used to be ONE interactive transaction around every create chunk
// AND one `updateMany` per re-imported row. Re-importing a 9 000-row file
// with an externalId column meant 9 000 serial updates inside it, which
// outlived Prisma's 5 s interactive-transaction timeout and ended in a
// 500. A chunk now commits on its own: creates go through
// `createManyAndReturn` (skipDuplicates absorbs a concurrent
// double-submit), updates through one batch transaction per chunk. A
// failure part-way leaves the earlier chunks written, and that is safe to
// retry: a re-upload of the same file skips what landed (unchanged
// externalId rows and natural-key duplicates are not written again).
const toCreate: Prisma.MeasurementCreateManyInput[] = [
...[...extInserts.values()].map(buildCreateData),
...plainInserts.map(buildCreateData),
];
const CREATE_CHUNK = 200;
let createdCount = 0;
const createdRows: Array<{
id: string;
type: MeasurementType;
measuredAt: Date;
}> = [];
await prisma.$transaction(async (tx) => {
for (let i = 0; i < toCreate.length; i += CREATE_CHUNK) {
const rows = await tx.measurement.createManyAndReturn({
data: toCreate.slice(i, i + CREATE_CHUNK),
skipDuplicates: true,
select: { id: true, type: true, measuredAt: true },
});
createdRows.push(...rows);
createdCount += rows.length;
}
for (const m of extUpdates.values()) {
await tx.measurement.updateMany({
where: {
userId,
source: "IMPORT",
type: m.type as MeasurementType,
externalId: m.externalId as string,
},
data: {
value: m.value,
unit: m.unit,
measuredAt: m.measuredAt,
notes: null,
notesEncrypted: encryptNote(m.notes ?? null),
glucoseContext:
(m.glucoseContext as GlucoseContext | undefined) ?? null,
// No-op on a live row, a deliberate RESURRECTION on a
// tombstoned one — IMPORT rows are re-importable by design, so
// a re-imported externalId brings the row back (mirrors the
// source-owned sync resurrect rule).
deletedAt: null,
},
});
}
});
for (const chunk of chunked(toCreate, WRITE_CHUNK)) {
const rows = await prisma.measurement.createManyAndReturn({
data: chunk,
skipDuplicates: true,
select: { id: true, type: true, measuredAt: true },
});
createdRows.push(...rows);
createdCount += rows.length;
}
for (const chunk of chunked([...extUpdates.values()], WRITE_CHUNK)) {
await prisma.$transaction(
chunk.map((m) =>
prisma.measurement.updateMany({
where: {
userId,
source: "IMPORT",
type: m.type as MeasurementType,
externalId: m.externalId as string,
},
data: {
value: m.value,
unit: m.unit,
measuredAt: m.measuredAt,
notes: null,
notesEncrypted: encryptNote(m.notes ?? null),
glucoseContext:
(m.glucoseContext as GlucoseContext | undefined) ?? null,
// No-op on a live row, a deliberate RESURRECTION on a
// tombstoned one — IMPORT rows are re-importable by design, so
// a re-imported externalId brings the row back (mirrors the
// source-owned sync resurrect rule).
deletedAt: null,
},
}),
),
);
}
void emitInsertedMeasurementArrivals(
userId,
@@ -66,6 +66,7 @@ import { rasterizePdf } from "@/lib/documents/rasterize-pdf";
import { reserveBudget, reconcileSpend } from "@/lib/ai/coach/budget";
import { checkRateLimit, refundRateLimit } from "@/lib/rate-limit";
import { OcrExtractError, runOcrExtraction } from "@/lib/labs/ocr-extract";
import { annotate } from "@/lib/logging/context";
const SESSION_OK = {
session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
@@ -164,6 +165,29 @@ describe("POST /api/labs/ocr/extract — text mode budget", () => {
expect(res.status).toBe(502);
expect(body.error).toContain("configured AI provider");
});
it("never passes the upstream body on, to the caller or to the event", async () => {
const upstreamBody = "ami-id instance-id iam/security-credentials/admin";
vi.mocked(runOcrExtraction).mockRejectedValue(
Object.assign(new Error("Local AI request failed (403)"), {
httpStatus: 403,
model: "llama3",
bodyExcerpt: upstreamBody,
}),
);
const res = await POST(textReq());
expect(res.status).toBe(502);
expect(await res.text()).not.toContain("iam/security-credentials");
const events = JSON.stringify(vi.mocked(annotate).mock.calls);
expect(events).not.toContain("iam/security-credentials");
// The status and the model stay, so an operator can still tell why.
expect(annotate).toHaveBeenCalledWith(
expect.objectContaining({
meta: expect.objectContaining({ upstreamStatus: 403, model: "llama3" }),
}),
);
});
});
describe("POST /api/labs/ocr/extract — vision PDF rasterization", () => {
+6 -4
View File
@@ -72,11 +72,16 @@ export const dynamic = "force-dynamic";
/** OCR'd text is bounded in the schema; cap the JSON body proportionally. */
const TEXT_BODY_MAX_BYTES = 512 * 1024;
/**
* The wide-event facts of a failed provider call: the status and the model,
* never the upstream body. The base URL can be one a user typed, and what an
* arbitrary endpoint answers has no business in the event stream (v1.39.3);
* the response to the caller carries neither.
*/
function providerFailureMeta(error: unknown, mode?: "text") {
const err = error as {
httpStatus?: unknown;
model?: unknown;
bodyExcerpt?: unknown;
};
return {
reason: "provider_error",
@@ -85,9 +90,6 @@ function providerFailureMeta(error: unknown, mode?: "text") {
? { upstreamStatus: err.httpStatus }
: {}),
...(typeof err.model === "string" ? { model: err.model } : {}),
...(typeof err.bodyExcerpt === "string"
? { upstreamError: err.bodyExcerpt }
: {}),
};
}
@@ -0,0 +1,126 @@
/**
* v1.39.3 — `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is deprecated and narrowed,
* not removed. Saving a private base URL under it is still accepted for the
* operator's own configuration (an admin account) and refused for any other
* account, which needs an exact origin in `AI_PRIVATE_ORIGINS`. Metadata and
* link-local are refused for everyone. The real policy and the real public-host
* check run here; only persistence is stubbed.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@/lib/api-handler", () => ({
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
requireAuth: vi.fn(async () => ({
user: { id: "u-1" },
session: { id: "s-1" },
})),
HttpError: class HttpError extends Error {
status: number;
constructor(status: number, message: string) {
super(message);
this.status = status;
}
},
}));
vi.mock("@/lib/db", () => ({
prisma: { user: { findUnique: vi.fn(), update: vi.fn() } },
}));
vi.mock("@/lib/crypto", () => ({
decrypt: vi.fn((v: string) => `dec:${v}`),
encrypt: vi.fn((v: string) => `enc:${v}`),
}));
vi.mock("@/lib/ai/provider", () => ({
resolveProviderAvailability: vi.fn(),
}));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/ai/server-provider-health", () => ({
readServerProviderHealth: vi.fn(),
}));
vi.mock("@/lib/feature-flags", async () =>
(
await import("@/__tests__/helpers/assistant-switches-mock")
).mockAssistantSwitches(vi.fn()),
);
vi.mock("@/lib/sharing/provider-work-authority", () => ({
providerWorkAuthorityForRecord: vi.fn(() => ({ origin: "owner" })),
providerCredentialPolicy: vi.fn(() => "personal"),
}));
vi.mock("@/lib/ai/consent-guard", () => ({
hasActiveConsentForSurface: vi.fn(),
}));
import { PATCH } from "../route";
import { prisma } from "@/lib/db";
import { _resetAiGrantsForTests } from "@/lib/ai/local-host-allowlist";
const patch = PATCH as (req: Request) => Promise<Response>;
function save(body: Record<string, string>) {
return patch(
new Request("http://localhost/api/user/ai-provider", {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
}),
);
}
function asRole(role: "ADMIN" | "USER") {
vi.mocked(prisma.user.findUnique).mockResolvedValue({ role } as never);
}
beforeEach(() => {
vi.mocked(prisma.user.findUnique).mockReset();
vi.mocked(prisma.user.update).mockReset();
vi.mocked(prisma.user.update).mockResolvedValue({} as never);
_resetAiGrantsForTests();
vi.stubEnv("AI_PRIVATE_ORIGINS", "");
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
vi.spyOn(console, "warn").mockImplementation(() => {});
});
afterEach(() => {
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
describe("PATCH /api/user/ai-provider under ALLOW_LOCAL_AI_PRIVATE_HOSTS=true", () => {
it.each([
["baseUrl", "http://10.0.0.5:11434/v1"],
["compatBaseUrl", "http://192.168.1.20:4000/v1"],
])("refuses a non-admin account's private %s", async (field, url) => {
asRole("USER");
const res = await save({ [field]: url });
expect(res.status).toBe(422);
expect((await res.json()).error).toContain("AI_PRIVATE_ORIGINS");
expect(prisma.user.update).not.toHaveBeenCalled();
});
it.each([
["baseUrl", "http://10.0.0.5:11434/v1"],
["compatBaseUrl", "http://192.168.1.20:4000/v1"],
])("accepts an admin account's private %s", async (field, url) => {
asRole("ADMIN");
const res = await save({ [field]: url });
expect(res.status).toBe(200);
expect(prisma.user.update).toHaveBeenCalledTimes(1);
});
it.each([
"http://169.254.169.254/latest/meta-data/",
"http://[fe80::1]:11434/v1",
"http://0.0.0.0:11434/v1",
])("refuses %s even for an admin account", async (url) => {
asRole("ADMIN");
const res = await save({ baseUrl: url });
expect(res.status).toBe(422);
expect(prisma.user.update).not.toHaveBeenCalled();
});
it("accepts a non-admin account's private URL once its exact origin is granted", async () => {
vi.stubEnv("AI_PRIVATE_ORIGINS", "http://10.0.0.5:11434");
asRole("USER");
const res = await save({ baseUrl: "http://10.0.0.5:11434/v1" });
expect(res.status).toBe(200);
});
});
+29 -8
View File
@@ -179,6 +179,26 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
}
const body = parsed.data;
// v1.39.3 — whose configuration this is. Settings saved on an admin account
// are the operator's own and are still covered by the deprecated
// `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true`; any other account needs an exact
// origin in `AI_PRIVATE_ORIGINS`. Read from the saved row, the same rule
// the provider resolution applies when it later dials the URL.
const needsHostPolicy =
(typeof body.baseUrl === "string" && body.baseUrl !== "") ||
(typeof body.compatBaseUrl === "string" && body.compatBaseUrl !== "");
const owner = needsHostPolicy
? {
operatorTrusted:
(
await prisma.user.findUnique({
where: { id: user.id },
select: { role: true },
})
)?.role === "ADMIN",
}
: {};
const updates: Record<string, unknown> = {};
if (body.provider !== undefined) {
@@ -197,14 +217,15 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
const trimmed = body.baseUrl.trim();
// SSRF guard: by default reject private/internal hostnames so a
// compromised user account cannot point the server at the cloud
// metadata endpoint or internal admin panels. v1.18.7 (SECURITY LOW) —
// ops opt in via `ALLOW_LOCAL_AI_PRIVATE_HOSTS`, now a host allowlist:
// `true` permits any private host (legacy), a comma-separated host list
// permits only those exact hostnames.
const allowPrivate = isLocalAiHostAllowed(trimmed);
// metadata endpoint or internal admin panels. The operator grants a
// private endpoint by exact origin (`AI_PRIVATE_ORIGINS`, or the legacy
// host list); the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` covers
// an admin account's own settings only (`owner` above), and no grant
// can open metadata or link-local.
const allowPrivate = isLocalAiHostAllowed(trimmed, owner);
if (!allowPrivate && !isPublicUrl(trimmed)) {
return apiError(
"Base URL points to an internal/private host. Ops must allow it on this instance via ALLOW_LOCAL_AI_PRIVATE_HOSTS — set it to the exact host (e.g. ollama.lan) or to true for any private host (intended for self-hosted Ollama / LM Studio).",
"Base URL points to an internal/private host. The operator must allow its exact origin on this instance via AI_PRIVATE_ORIGINS (e.g. http://ollama.lan:11434) for a self-hosted Ollama / LM Studio.",
422,
);
}
@@ -226,9 +247,9 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
// private one only when the operator allowlisted it. Gateways on a LAN
// are the normal case for LiteLLM / vLLM, so the escape hatch matters
// here as much as it does for Ollama.
if (!isLocalAiHostAllowed(trimmed) && !isPublicUrl(trimmed)) {
if (!isLocalAiHostAllowed(trimmed, owner) && !isPublicUrl(trimmed)) {
return apiError(
"Base URL points to an internal/private host. Ops must allow it on this instance via ALLOW_LOCAL_AI_PRIVATE_HOSTS — set it to the exact host (e.g. litellm.lan) or to true for any private host (intended for a self-hosted gateway).",
"Base URL points to an internal/private host. The operator must allow its exact origin on this instance via AI_PRIVATE_ORIGINS (e.g. http://litellm.lan:4000) for a self-hosted gateway.",
422,
);
}
@@ -0,0 +1,64 @@
import { describe, it, expect, vi } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
import { I18nProvider } from "@/lib/i18n/context";
/**
* v1.39.3 — the admin AI card names the deprecated
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` and gives the exact
* `AI_PRIVATE_ORIGINS` line to set instead, built from the origins the server
* reports in use. Absent when the server reports no such grant.
*/
const state = vi.hoisted(() => ({ data: null as null | object }));
vi.mock("@tanstack/react-query", () => ({
useQuery: () => ({ data: state.data }),
useMutation: () => ({ mutate: vi.fn(), isPending: false }),
useQueryClient: () => ({ invalidateQueries: vi.fn() }),
}));
import { AiServerKeySection } from "../ai-server-key-section";
function render(): string {
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<AiServerKeySection />
</I18nProvider>,
);
}
const base = {
hasKey: false,
keyPreview: null,
model: "gpt-4o",
baseUrl: "https://api.openai.com/v1",
};
describe("<AiServerKeySection> deprecated private-host notice", () => {
it("shows the exact line to set, with every origin in use", () => {
state.data = {
...base,
legacyPrivateHostGrant: {
originsInUse: ["http://10.0.0.5:11434", "http://gateway.lan:4000"],
},
};
const html = render();
expect(html).toContain('data-slot="admin-ai-legacy-private-hosts"');
expect(html).toContain("is deprecated");
expect(html).toContain(
"AI_PRIVATE_ORIGINS=&quot;http://10.0.0.5:11434,http://gateway.lan:4000&quot;",
);
});
it("says the setting can go when no saved URL needs it", () => {
state.data = { ...base, legacyPrivateHostGrant: { originsInUse: [] } };
const html = render();
expect(html).toContain("you can remove ALLOW_LOCAL_AI_PRIVATE_HOSTS now");
expect(html).not.toContain("admin-ai-legacy-private-hosts-line");
});
it("is absent when the server reports no deprecated grant", () => {
state.data = { ...base, legacyPrivateHostGrant: null };
expect(render()).not.toContain("admin-ai-legacy-private-hosts");
});
});
+52 -1
View File
@@ -2,7 +2,7 @@
import { useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { KeyRound, Loader2 } from "lucide-react";
import { AlertTriangle, KeyRound, Loader2 } from "lucide-react";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
@@ -49,6 +49,12 @@ interface AiServerKeyResponse {
keyPreview: string | null;
model: string;
baseUrl: string;
/**
* v1.39.3 — present while the deprecated
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is set: the origins it is standing in
* for, to copy into `AI_PRIVATE_ORIGINS`.
*/
legacyPrivateHostGrant?: { originsInUse: string[] } | null;
}
export function AiServerKeySection() {
@@ -120,6 +126,11 @@ export function AiServerKeySection() {
/>
<div className="space-y-4">
{settings?.legacyPrivateHostGrant ? (
<LegacyPrivateHostNotice
originsInUse={settings.legacyPrivateHostGrant.originsInUse}
/>
) : null}
<div className="space-y-2 text-sm">
<p>{t("admin.aiServerKey.byokHint")}</p>
<p>{t("admin.aiServerKey.consentHint")}</p>
@@ -201,3 +212,43 @@ export function AiServerKeySection() {
</SettingsCard>
);
}
/**
* v1.39.3 — the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true`, with the exact
* line to set instead. The origins come from the saved base URLs on this
* instance; the operator copies the line into the server environment.
*/
function LegacyPrivateHostNotice({ originsInUse }: { originsInUse: string[] }) {
const { t } = useTranslations();
const line = `AI_PRIVATE_ORIGINS="${originsInUse.join(",")}"`;
return (
<div
role="status"
data-slot="admin-ai-legacy-private-hosts"
className="border-warning/40 bg-warning/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<AlertTriangle className="mt-0.5 size-4 shrink-0" aria-hidden="true" />
<div className="min-w-0 space-y-1">
<p className="font-medium">
{t("admin.aiServerKey.legacyGrantTitle")}
</p>
<p>{t("admin.aiServerKey.legacyGrantBody")}</p>
{originsInUse.length > 0 ? (
<>
<p>{t("admin.aiServerKey.legacyGrantOrigins")}</p>
<code
data-slot="admin-ai-legacy-private-hosts-line"
className="bg-muted block overflow-x-auto rounded px-2 py-1 font-mono text-xs break-all select-all"
>
{line}
</code>
</>
) : (
<p>{t("admin.aiServerKey.legacyGrantNoOrigins")}</p>
)}
</div>
</div>
</div>
);
}
+23 -19
View File
@@ -9,14 +9,13 @@
* instead of shipping silently.
*
* The LOCAL AI client and (since v1.37.30) the openai-client's gateway and
* admin-key tags are the deliberate exceptions: they stay CONDITIONAL
* (`requirePublicHost: !allowPrivate`) so an operator can opt into LAN hosts
* via `ALLOW_LOCAL_AI_PRIVATE_HOSTS`. v1.18.7 (SECURITY LOW) — that flag is
* now a host ALLOWLIST (`true` = any private host; a comma-separated list =
* only those), resolved by `isLocalAiHostAllowed`. We assert the client keeps
* the conditional shape AND derives `allowPrivate` from the allowlist helper
* (not a raw `=== "true"` binary), never an unconditional `true` (which would
* break LAN local models) and never absent (which would re-open the default).
* admin-key tags are the deliberate exceptions: an operator can grant a LAN
* endpoint. Since v1.39.3 the grant is an exact origin (`AI_PRIVATE_ORIGINS`,
* or the legacy host list) and the call is dialled through the pinned
* operator-approved dispatcher, never unpinned; the whole policy comes from
* `aiEgressPolicyFor`. We assert every AI dial site spreads that policy, and
* that neither an unconditional `false` (no pin) nor a local
* `isLocalAiHostAllowed` decision (a grant with no pin) reappears there.
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
@@ -35,8 +34,9 @@ describe("SSRF requirePublicHost pin inventory", () => {
it("openai-client routes person-typed base URLs through the allowlist with a pinned floor", () => {
const src = read("ai/openai-client.ts");
expect(src).toMatch(
/requirePublicHost:\s*this\.isGateway \|\| this\.type === "admin-key"\s*\? requirePublicHostFor\(this\.config\.baseUrl\)\s*:\s*true/,
/this\.isGateway \|\| this\.type === "admin-key"\s*\?\s*aiEgressPolicyFor\(url,\s*\{\s*operatorTrusted:\s*this\.config\.operatorTrusted,?\s*\}\)\s*:\s*\{\s*requirePublicHost:\s*true as const\s*\}/,
);
expect(src).not.toMatch(/requirePublicHost:\s*false/);
});
it("anthropic-client pins the BYO base-URL outbound unconditionally", () => {
@@ -83,16 +83,20 @@ describe("SSRF requirePublicHost pin inventory", () => {
},
);
it("local AI client keeps the pin CONDITIONAL (LAN escape hatch)", () => {
it("local AI client dials every call through the shared egress policy", () => {
const src = read("ai/local-client.ts");
// Must carry the conditional form …
expect(src).toMatch(/requirePublicHost:\s*!?\s*allowPrivate/);
// … and must NOT be hardened to an unconditional true (that would break
// a deliberately-private LAN local model).
expect(src).not.toMatch(/requirePublicHost:\s*true/);
// … and `allowPrivate` must come from the host-allowlist helper, not a
// raw binary `=== "true"` (v1.18.7 — the flag became an allowlist).
expect(src).toMatch(/allowPrivate\s*=\s*isLocalAiHostAllowed\(/);
expect(src).not.toMatch(/ALLOW_LOCAL_AI_PRIVATE_HOSTS\s*===\s*"true"/);
// Both the buffered and the streaming call derive the policy …
expect([
...src.matchAll(
/const egress = aiEgressPolicyFor\(url,\s*\{\s*operatorTrusted:\s*this\.config\.operatorTrusted,?\s*\}\)/g,
),
]).toHaveLength(2);
// … and spread it into safeFetch …
expect([...src.matchAll(/\.\.\.egress,/g)]).toHaveLength(2);
// … with no local decision beside it: no hard-coded pin value (true
// would break a granted LAN model, false would drop the pin) and no
// grant check that could skip the operator-approved dispatcher.
expect(src).not.toMatch(/requirePublicHost:\s*(?:true|false|!)/);
expect(src).not.toMatch(/isLocalAiHostAllowed\(/);
});
});
@@ -0,0 +1,100 @@
/**
* v1.39.3 — the destination policy an AI call is dialled with.
*
* A granted private origin goes to `safeFetch` as an exact
* `operatorApprovedPrivateOrigin`, which routes the connect through the pinned
* operator-approved dispatcher: redirects forbidden, metadata, link-local and
* the unspecified address dropped at dial time even for a granted name. Before
* this release a granted host was dialled with no pin at all. Everything else
* is pinned to public hosts.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { safeFetch } = vi.hoisted(() => ({ safeFetch: vi.fn() }));
vi.mock("@/lib/safe-fetch", async (importOriginal) => ({
...(await importOriginal<typeof import("@/lib/safe-fetch")>()),
safeFetch,
}));
import { LocalOpenAICompatibleClient } from "../local-client";
import { _resetAiGrantsForTests } from "../local-host-allowlist";
import { singleUserTurn } from "../types";
function client(baseUrl: string) {
return new LocalOpenAICompatibleClient({
apiKey: null,
model: "llama3:8b",
baseUrl,
});
}
beforeEach(() => {
_resetAiGrantsForTests();
safeFetch.mockReset();
safeFetch.mockResolvedValue(
new Response(
JSON.stringify({ choices: [{ message: { content: "ok" } }] }),
{ status: 200, headers: { "content-type": "application/json" } },
),
);
vi.stubEnv("AI_PRIVATE_ORIGINS", "");
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "");
vi.spyOn(console, "warn").mockImplementation(() => {});
});
afterEach(() => {
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
const call = singleUserTurn({ system: "sys", user: "hi" });
describe("Local AI egress policy", () => {
it("dials a granted origin through the operator-approved pin", async () => {
vi.stubEnv("AI_PRIVATE_ORIGINS", "http://ollama.lan:11434");
await client("http://ollama.lan:11434/v1").generateCompletion(call);
const opts = safeFetch.mock.calls[0][2];
expect(opts).toMatchObject({
requirePublicHost: false,
operatorApprovedPrivateOrigin: "http://ollama.lan:11434",
});
});
it("pins an ungranted private URL to public hosts, so safeFetch refuses it", async () => {
await client("http://10.0.0.5:11434/v1").generateCompletion(call);
const opts = safeFetch.mock.calls[0][2];
expect(opts.requirePublicHost).toBe(true);
expect(opts).not.toHaveProperty("operatorApprovedPrivateOrigin");
});
it("the deprecated `=true` no longer covers a non-admin account's endpoint", async () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
await client("http://10.0.0.5:11434/v1").generateCompletion(call);
expect(safeFetch.mock.calls[0][2].requirePublicHost).toBe(true);
});
it("the deprecated `=true` still covers the operator's own endpoint, pinned", async () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
await new LocalOpenAICompatibleClient({
apiKey: null,
model: "llama3:8b",
baseUrl: "http://10.0.0.5:11434/v1",
operatorTrusted: true,
}).generateCompletion(call);
expect(safeFetch.mock.calls[0][2]).toMatchObject({
requirePublicHost: false,
operatorApprovedPrivateOrigin: "http://10.0.0.5:11434",
});
});
it("streams through the same policy", async () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "ollama.lan");
await client("http://ollama.lan:11434/v1")
.generateCompletionStream(call, () => {})
.catch(() => undefined);
expect(safeFetch.mock.calls[0][2]).toMatchObject({
requirePublicHost: false,
operatorApprovedPrivateOrigin: "http://ollama.lan:11434",
});
});
});
+7 -4
View File
@@ -1,4 +1,11 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
// The client's destination policy has its own suite
// (`local-host-allowlist.test.ts`, `local-client-egress.test.ts`). Here the
// grant is stubbed to "no policy" so the request shape can be asserted
// against the stubbed global fetch; the pinned dispatchers drive undici's own
// fetch, which a global stub cannot see.
vi.mock("../local-host-allowlist", () => ({ aiEgressPolicyFor: () => ({}) }));
import {
LocalOpenAICompatibleClient,
resetLocalJsonDialectCache,
@@ -8,10 +15,6 @@ import { singleUserTurn } from "../types";
describe("LocalOpenAICompatibleClient", () => {
beforeEach(() => {
vi.restoreAllMocks();
// These tests cover the operator-opt-in path (LM Studio / Ollama on
// an RFC1918 host). The safeFetch wrapper enforces the SSRF guard
// unless the operator explicitly accepts a private host.
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
// v1.28.28 (#470) — the JSON dialect is cached per baseUrl for the
// process; clear it so each test starts from the default dialect.
resetLocalJsonDialectCache();
+155 -29
View File
@@ -1,57 +1,183 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
_resetAiGrantsForTests,
aiEgressPolicyFor,
grantedAiOrigin,
isLocalAiHostAllowed,
requirePublicHostFor,
legacyAnyHostConfigured,
LEGACY_ANY_HOST_WARNING,
originNeedingAiGrant,
} from "../local-host-allowlist";
/**
* v1.39.3 — AI base URLs on a private network are granted by exact origin
* (`AI_PRIVATE_ORIGINS`), the grammar the notification and Nightscout grants
* use, and dialled through the pinned operator-approved dispatcher. The legacy
* host list keeps working; its deprecated `=true` form covers only the
* operator's own configurations.
*/
let warn: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
_resetAiGrantsForTests();
vi.stubEnv("AI_PRIVATE_ORIGINS", "");
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "");
warn = vi.spyOn(console, "warn").mockImplementation(() => {});
});
afterEach(() => {
vi.unstubAllEnvs();
warn.mockRestore();
});
describe("isLocalAiHostAllowed", () => {
it("denies every private host when unset (secure default)", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "");
describe("AI_PRIVATE_ORIGINS", () => {
it("grants nothing when unset", () => {
expect(isLocalAiHostAllowed("http://10.0.0.5:11434/v1")).toBe(false);
expect(isLocalAiHostAllowed("http://ollama.lan/v1")).toBe(false);
});
it('treats "false" as denying every private host', () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "false");
expect(isLocalAiHostAllowed("http://10.0.0.5/v1")).toBe(false);
it("grants exactly the listed origin, not its siblings", () => {
vi.stubEnv(
"AI_PRIVATE_ORIGINS",
"http://ollama.lan:11434, http://10.0.0.5:4000",
);
expect(grantedAiOrigin("http://ollama.lan:11434/v1/chat")).toBe(
"http://ollama.lan:11434",
);
expect(isLocalAiHostAllowed("http://10.0.0.5:4000/v1")).toBe(true);
// Another port, another scheme, a subdomain: all separate origins.
expect(isLocalAiHostAllowed("http://ollama.lan:8080/v1")).toBe(false);
expect(isLocalAiHostAllowed("https://ollama.lan:11434/v1")).toBe(false);
expect(isLocalAiHostAllowed("http://evil.ollama.lan:11434/v1")).toBe(false);
expect(isLocalAiHostAllowed("http://10.0.0.6:4000/v1")).toBe(false);
});
it('keeps the legacy "true" = any private host', () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
expect(isLocalAiHostAllowed("http://10.0.0.5:11434/v1")).toBe(true);
expect(isLocalAiHostAllowed("http://anything.lan/v1")).toBe(true);
});
it("permits ONLY the listed hosts when given a comma-separated allowlist", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "ollama.lan, 10.0.0.5");
expect(isLocalAiHostAllowed("http://ollama.lan:11434/v1")).toBe(true);
expect(isLocalAiHostAllowed("http://10.0.0.5:11434/v1")).toBe(true);
// A different private host is still rejected — the metadata endpoint and
// internal panels no longer ride the binary flag.
it("never grants metadata, link-local or the unspecified address, even when listed", () => {
vi.stubEnv(
"AI_PRIVATE_ORIGINS",
"http://169.254.169.254,http://[fe80::1]:80,http://0.0.0.0:11434",
);
expect(isLocalAiHostAllowed("http://169.254.169.254/latest/")).toBe(false);
expect(isLocalAiHostAllowed("http://[fe80::1]/v1")).toBe(false);
expect(isLocalAiHostAllowed("http://0.0.0.0:11434/v1")).toBe(false);
expect(warn).toHaveBeenCalled();
});
});
describe("ALLOW_LOCAL_AI_PRIVATE_HOSTS (legacy)", () => {
it("still grants a listed host on any port", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", " Ollama.LAN , 10.0.0.5 ");
expect(isLocalAiHostAllowed("http://ollama.lan:11434/v1")).toBe(true);
expect(isLocalAiHostAllowed("http://10.0.0.5:8080/v1")).toBe(true);
expect(isLocalAiHostAllowed("http://10.0.0.6/v1")).toBe(false);
});
it("matches hostnames case-insensitively and ignores blank entries", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", " , Ollama.LAN , ");
expect(isLocalAiHostAllowed("http://ollama.lan/v1")).toBe(true);
it("`true` still covers an operator-owned configuration, pinned, and says it is deprecated", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
expect(
aiEgressPolicyFor("http://10.0.0.5:11434/v1", { operatorTrusted: true }),
).toEqual({
requirePublicHost: false,
operatorApprovedPrivateOrigin: "http://10.0.0.5:11434",
});
expect(warn).toHaveBeenCalledWith(LEGACY_ANY_HOST_WARNING);
expect(LEGACY_ANY_HOST_WARNING).toContain("deprecated");
expect(LEGACY_ANY_HOST_WARNING).toContain("AI_PRIVATE_ORIGINS=");
expect(legacyAnyHostConfigured()).toBe(true);
});
it("never permits an unparseable URL", () => {
it("`true` no longer covers a non-admin account's own base URL", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
expect(isLocalAiHostAllowed("not a url")).toBe(true); // host-agnostic "any"
expect(isLocalAiHostAllowed("http://10.0.0.5:11434/v1")).toBe(false);
expect(
isLocalAiHostAllowed("http://10.0.0.5:11434/v1", {
operatorTrusted: false,
}),
).toBe(false);
expect(aiEgressPolicyFor("http://10.0.0.5:11434/v1")).toEqual({
requirePublicHost: true,
});
});
it("`true` never opens metadata, link-local or the unspecified address, even for the operator", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
for (const url of [
"http://169.254.169.254/latest/meta-data/",
"http://[fe80::1]/v1",
"http://0.0.0.0:11434/v1",
"http://[::ffff:169.254.169.254]/v1",
]) {
expect(isLocalAiHostAllowed(url, { operatorTrusted: true }), url).toBe(
false,
);
}
});
it("an exact origin grant covers every account, with or without `true`", () => {
vi.stubEnv("AI_PRIVATE_ORIGINS", "http://ollama.lan:11434");
expect(isLocalAiHostAllowed("http://ollama.lan:11434/v1")).toBe(true);
});
it("refuses a metadata literal and a host:port entry instead of granting a guess", () => {
vi.stubEnv(
"ALLOW_LOCAL_AI_PRIVATE_HOSTS",
"169.254.169.254,ollama.lan:11434",
);
expect(isLocalAiHostAllowed("http://169.254.169.254/latest/")).toBe(false);
expect(isLocalAiHostAllowed("http://ollama.lan:11434/v1")).toBe(false);
expect(warn).toHaveBeenCalledTimes(2);
});
it("an unparseable URL is never granted", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "ollama.lan");
expect(isLocalAiHostAllowed("not a url")).toBe(false);
});
});
it("requirePublicHostFor is the inverse of the allowlist", () => {
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "ollama.lan");
expect(requirePublicHostFor("http://ollama.lan/v1")).toBe(false);
expect(requirePublicHostFor("http://10.0.0.6/v1")).toBe(true);
describe("aiEgressPolicyFor", () => {
it("pins an ungranted URL to the public dispatcher", () => {
expect(aiEgressPolicyFor("http://10.0.0.5/v1")).toEqual({
requirePublicHost: true,
});
expect(aiEgressPolicyFor("https://api.example.com/v1")).toEqual({
requirePublicHost: true,
});
});
it("hands a granted URL to the operator-approved dispatcher for exactly its origin", () => {
vi.stubEnv("AI_PRIVATE_ORIGINS", "http://ollama.lan:11434");
expect(aiEgressPolicyFor("http://ollama.lan:11434/v1/chat")).toEqual({
requirePublicHost: false,
operatorApprovedPrivateOrigin: "http://ollama.lan:11434",
});
});
});
describe("originNeedingAiGrant", () => {
it("names the origin of a LAN endpoint no grant covers yet", () => {
for (const [url, origin] of [
["http://10.0.0.5:11434/v1", "http://10.0.0.5:11434"],
["http://ollama.lan:11434/v1", "http://ollama.lan:11434"],
["http://ollama:11434/v1", "http://ollama:11434"],
["http://nas.local:1234/v1", "http://nas.local:1234"],
["http://localhost:11434/v1", "http://localhost:11434"],
]) {
expect(originNeedingAiGrant(url), url).toBe(origin);
}
});
it("skips hosted endpoints, never-grantable addresses and origins already granted", () => {
vi.stubEnv("AI_PRIVATE_ORIGINS", "http://10.0.0.5:11434");
for (const url of [
"https://openrouter.ai/api/v1",
"https://api.openai.com/v1",
"http://169.254.169.254/latest",
"http://10.0.0.5:11434/v1",
"not a url",
]) {
expect(originNeedingAiGrant(url), url).toBeNull();
}
});
});
@@ -53,8 +53,8 @@ describe("AIProvider contract", () => {
vi.restoreAllMocks();
// The local-provider contract test points at a localhost OpenAI-
// compatible server (Ollama / LM Studio). safeFetch enforces the
// SSRF guard unless the operator explicitly opts in.
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
// SSRF guard unless the operator grants that exact origin.
vi.stubEnv("AI_PRIVATE_ORIGINS", "http://localhost:11434");
});
afterEach(() => {
@@ -0,0 +1,197 @@
/**
* v1.39.3 — the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` at dial time.
*
* Provider resolution decides whose configuration a base URL is, and the
* client dials accordingly. Under `=true` a private endpoint is still reached
* for the operator's own configurations (the instance-wide admin provider and
* settings saved on an admin account), always through the pinned
* operator-approved dispatcher; any other account's saved URL is pinned to
* public hosts, so `safeFetch` refuses it. Checked at the wire: the options
* each resolved provider hands to `safeFetch`.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { safeFetch } = vi.hoisted(() => ({ safeFetch: vi.fn() }));
vi.mock("@/lib/safe-fetch", async (importOriginal) => ({
...(await importOriginal<typeof import("@/lib/safe-fetch")>()),
safeFetch,
}));
vi.mock("@/lib/db", () => ({
prisma: {
user: { findUnique: vi.fn(), update: vi.fn() },
appSettings: { findUnique: vi.fn() },
},
}));
vi.mock("@/lib/crypto", () => ({
decrypt: vi.fn((v: string) => `decrypted:${v}`),
encrypt: vi.fn((v: string) => `encrypted:${v}`),
}));
vi.mock("@/lib/ai/codex-oauth", () => ({
refreshDeviceTokens: vi.fn(),
encryptCodexCreds: vi.fn(),
decryptCodexCreds: vi.fn(),
}));
vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
getEvent: vi.fn(() => undefined),
}));
import {
AITestConfigError,
resolveProvider,
resolveProviderForTest,
} from "../provider";
import { _resetAiGrantsForTests } from "../local-host-allowlist";
import { singleUserTurn } from "../types";
import { prisma } from "@/lib/db";
function row(role: "ADMIN" | "USER", overrides: Record<string, unknown>) {
return {
aiProvider: null,
aiModel: "m",
aiBaseUrl: null,
aiAnthropicKeyEncrypted: null,
aiLocalKeyEncrypted: null,
aiOpenaiKeyEncrypted: null,
aiCompatBaseUrl: null,
aiCompatKeyEncrypted: null,
aiCompatModel: null,
managedProfileAt: null,
role,
...overrides,
} as never;
}
async function dialOptions(): Promise<Record<string, unknown>> {
const provider = await resolveProvider("u-1");
await provider
.generateCompletion(singleUserTurn({ system: "s", user: "u" }))
.catch(() => undefined);
return safeFetch.mock.calls[0][2] as Record<string, unknown>;
}
beforeEach(() => {
_resetAiGrantsForTests();
safeFetch.mockReset();
safeFetch.mockResolvedValue(
new Response(
JSON.stringify({ choices: [{ message: { content: "ok" } }] }),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
);
vi.mocked(prisma.user.findUnique).mockReset();
vi.mocked(prisma.appSettings.findUnique).mockReset();
vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never);
vi.stubEnv("AI_PRIVATE_ORIGINS", "");
vi.stubEnv("ALLOW_LOCAL_AI_PRIVATE_HOSTS", "true");
vi.spyOn(console, "warn").mockImplementation(() => {});
});
afterEach(() => {
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
describe("ALLOW_LOCAL_AI_PRIVATE_HOSTS=true at dial time", () => {
it.each([
["LOCAL", { aiProvider: "LOCAL", aiBaseUrl: "http://10.0.0.5:11434/v1" }],
[
"OPENAI_COMPATIBLE",
{
aiProvider: "OPENAI_COMPATIBLE",
aiCompatBaseUrl: "http://10.0.0.5:4000/v1",
aiCompatModel: "m",
},
],
])(
"an admin account's %s endpoint goes through the operator-approved pin",
async (_label, overrides) => {
vi.mocked(prisma.user.findUnique).mockResolvedValue(
row("ADMIN", overrides),
);
const opts = await dialOptions();
expect(opts.requirePublicHost).toBe(false);
expect(opts.operatorApprovedPrivateOrigin).toMatch(
/^http:\/\/10\.0\.0\.5:/,
);
},
);
it.each([
["LOCAL", { aiProvider: "LOCAL", aiBaseUrl: "http://10.0.0.5:11434/v1" }],
[
"OPENAI_COMPATIBLE",
{
aiProvider: "OPENAI_COMPATIBLE",
aiCompatBaseUrl: "http://10.0.0.5:4000/v1",
aiCompatModel: "m",
},
],
])(
"a non-admin account's %s endpoint stays on the public pin",
async (_label, overrides) => {
vi.mocked(prisma.user.findUnique).mockResolvedValue(
row("USER", overrides),
);
const opts = await dialOptions();
expect(opts.requirePublicHost).toBe(true);
expect(opts).not.toHaveProperty("operatorApprovedPrivateOrigin");
},
);
it("the metadata address stays on the public pin even for an admin account", async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue(
row("ADMIN", {
aiProvider: "LOCAL",
aiBaseUrl: "http://169.254.169.254/latest/v1",
}),
);
const opts = await dialOptions();
expect(opts.requirePublicHost).toBe(true);
});
it("the instance-wide admin provider is the operator's own", async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue(
row("USER", { aiProvider: null }),
);
vi.mocked(prisma.appSettings.findUnique).mockResolvedValue({
adminAiKeyEncrypted: "k",
adminAiModel: "m",
adminAiBaseUrl: "http://10.0.0.7:8080/v1",
} as never);
const opts = await dialOptions();
expect(opts).toMatchObject({
requirePublicHost: false,
operatorApprovedPrivateOrigin: "http://10.0.0.7:8080",
});
});
});
describe("the connection test under ALLOW_LOCAL_AI_PRIVATE_HOSTS=true", () => {
it("refuses a non-admin account's private endpoint", async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue(
row("USER", {
aiProvider: "LOCAL",
aiBaseUrl: "http://10.0.0.5:11434/v1",
}),
);
await expect(resolveProviderForTest("u-1")).rejects.toBeInstanceOf(
AITestConfigError,
);
});
it("resolves an admin account's private endpoint", async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue(
row("ADMIN", {
aiProvider: "LOCAL",
aiBaseUrl: "http://10.0.0.5:11434/v1",
}),
);
await expect(
resolveProviderForTest("u-1", { provider: "LOCAL" }),
).resolves.toBeDefined();
});
});
@@ -104,15 +104,12 @@ describe("listConversations — filter + DTO", () => {
});
// v1.30.2 (QoL H1) — server-side title search for the full-history rail.
it("applies a case-insensitive title-contains filter when q is given", async () => {
it("keeps the title out of SQL when q is given (it is encrypted) and stays owner-scoped", async () => {
findMany.mockResolvedValue([]);
await listConversations({ userId: "user-1", q: "blood pressure" });
const where = findMany.mock.calls[0][0].where;
expect(where.userId).toBe("user-1");
expect(where.title).toEqual({
contains: "blood pressure",
mode: "insensitive",
});
expect("title" in where).toBe(false);
});
it("trims q and omits the title filter for an empty/whitespace-only query", async () => {
@@ -1,5 +1,8 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
const { updateMany } = vi.hoisted(() => ({ updateMany: vi.fn() }));
vi.mock("@/lib/db", () => ({
@@ -9,6 +12,7 @@ vi.mock("@/lib/db", () => ({
}));
import { renameConversation } from "../persistence";
import { decryptFromBytes } from "../bytes-codec";
beforeEach(() => {
updateMany.mockReset();
@@ -21,10 +25,12 @@ describe("renameConversation", () => {
await expect(
renameConversation("owner-1", "conversation-1", "Renamed"),
).resolves.toEqual({ id: "conversation-1", title: "Renamed" });
expect(updateMany).toHaveBeenCalledWith({
where: { id: "conversation-1", userId: "owner-1" },
data: { title: "Renamed" },
});
const arg = updateMany.mock.calls[0][0];
expect(arg.where).toEqual({ id: "conversation-1", userId: "owner-1" });
// v1.39.3 — the new title is stored as ciphertext and the readable column
// is cleared, so a legacy row never keeps its old title beside the new one.
expect(arg.data.title).toBeNull();
expect(decryptFromBytes(arg.data.titleEncrypted)).toBe("Renamed");
});
it("returns null when the owned update matched no row", async () => {
@@ -0,0 +1,199 @@
/**
* v1.39.3 — the Coach conversation title is encrypted at rest. These pin the
* three things that change with it:
* 1. every write stores the title as ciphertext and never in the readable
* column;
* 2. every read prefers the ciphertext, falls back to the readable column
* only for a row the backfill has not reached, and never falls back past
* a ciphertext that does not open;
* 3. the history search, which used to be an SQL `ILIKE`, still finds titles
* (sealed and legacy alike), case-insensitively, and pages with the same
* cursor contract.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
const { findMany, create } = vi.hoisted(() => ({
findMany: vi.fn(),
create: vi.fn(),
}));
vi.mock("@/lib/db", () => ({
prisma: {
coachConversation: { findMany },
$transaction: async (
fn: (tx: { coachConversation: { create: typeof create } }) => unknown,
) => fn({ coachConversation: { create } }),
},
}));
import {
createConversation,
listConversations,
readConversationTitle,
} from "../persistence";
import { decryptFromBytes, encryptToBytes } from "../bytes-codec";
const now = new Date("2026-09-20T10:00:00.000Z");
interface StoredRow {
id: string;
title: string | null;
titleEncrypted: Uint8Array | null;
}
function fullRow(row: StoredRow) {
return {
...row,
createdAt: now,
updatedAt: now,
documentScoped: false,
_count: { messages: 2 },
attachments: [],
};
}
/**
* A findMany that answers both reads of the search path: the id + title scan
* (select) and the page fetch (`id: { in }`).
*/
function serve(rows: StoredRow[]) {
findMany.mockImplementation(
async (args: {
where: { id?: { in: string[] } };
select?: unknown;
take?: number;
}) => {
if (args.where.id) {
// Returned out of order on purpose: the page must follow the scan order.
return rows
.filter((r) => args.where.id!.in.includes(r.id))
.reverse()
.map(fullRow);
}
if (args.select) return rows.slice(0, args.take);
return rows.slice(0, args.take).map(fullRow);
},
);
}
beforeEach(() => {
findMany.mockReset();
create.mockReset();
});
describe("createConversation", () => {
it("stores the summarised title as ciphertext only", async () => {
create.mockResolvedValue({
id: "c1",
title: null,
titleEncrypted: null,
createdAt: now,
updatedAt: now,
documentScoped: false,
});
const dto = await createConversation({
userId: "u1",
title: "Is my resting pulse\nhigher since the new job?",
});
const data = create.mock.calls[0][0].data;
expect(data).not.toHaveProperty("title");
expect(decryptFromBytes(data.titleEncrypted)).toBe(
"Is my resting pulse higher since the new job?",
);
expect(dto.title).toBe("Is my resting pulse higher since the new job?");
});
});
describe("readConversationTitle", () => {
it("prefers the ciphertext over a readable value", () => {
expect(
readConversationTitle({
title: "stale",
titleEncrypted: encryptToBytes("sealed"),
}),
).toBe("sealed");
});
it("falls back to the readable column only when there is no ciphertext", () => {
expect(
readConversationTitle({ title: "legacy", titleEncrypted: null }),
).toBe("legacy");
expect(readConversationTitle({ title: null, titleEncrypted: null })).toBe(
"",
);
});
it("throws on a ciphertext that does not open instead of showing the readable value", () => {
expect(() =>
readConversationTitle({
title: "must not leak",
titleEncrypted: new TextEncoder().encode("not-a-ciphertext"),
}),
).toThrow();
});
});
describe("listConversations title search", () => {
const rows: StoredRow[] = [
{ id: "c5", title: null, titleEncrypted: encryptToBytes("Blood pressure") },
{ id: "c4", title: null, titleEncrypted: encryptToBytes("Sleep debt") },
// A row the backfill has not reached yet is still searchable.
{ id: "c3", title: "Morning BLOOD pressure spikes", titleEncrypted: null },
{ id: "c2", title: null, titleEncrypted: encryptToBytes("Knee pain") },
{
id: "c1",
title: null,
titleEncrypted: encryptToBytes("blood pressure and salt"),
},
];
it("matches decrypted and legacy titles case-insensitively, in rail order", async () => {
serve(rows);
const page = await listConversations({ userId: "u1", q: "Blood Pressure" });
expect(page.conversations.map((c) => c.id)).toEqual(["c5", "c3", "c1"]);
expect(page.conversations.map((c) => c.title)).toEqual([
"Blood pressure",
"Morning BLOOD pressure spikes",
"blood pressure and salt",
]);
expect(page.nextCursor).toBeNull();
// The scan stays owner-scoped and never matches in SQL.
const scan = findMany.mock.calls[0][0];
expect(scan.where.userId).toBe("u1");
expect("title" in scan.where).toBe(false);
});
it("pages the matches with the cursor contract of the unfiltered rail", async () => {
serve(rows);
const first = await listConversations({
userId: "u1",
q: "blood",
limit: 2,
});
expect(first.conversations.map((c) => c.id)).toEqual(["c5", "c3"]);
expect(first.nextCursor).toBe("c3");
const second = await listConversations({
userId: "u1",
q: "blood",
limit: 2,
cursor: first.nextCursor,
});
expect(second.conversations.map((c) => c.id)).toEqual(["c1"]);
expect(second.nextCursor).toBeNull();
});
it("answers an unknown cursor with an empty page", async () => {
serve(rows);
const page = await listConversations({
userId: "u1",
q: "blood",
cursor: "gone",
});
expect(page.conversations).toEqual([]);
expect(page.nextCursor).toBeNull();
});
});
@@ -96,10 +96,14 @@ describe("recordProactiveNudge", () => {
createdAt: now,
});
// Conversation owned by the user, title summarised.
expect(txCreate.coachConversation.create).toHaveBeenCalledWith({
data: { userId: "user_1", title: "Time to weigh in" },
});
// Conversation owned by the user, title summarised and stored as
// ciphertext only (v1.39.3); the readable column is never written.
const convArg = txCreate.coachConversation.create.mock.calls[0][0];
expect(convArg.data.userId).toBe("user_1");
expect(convArg.data).not.toHaveProperty("title");
expect(new TextDecoder().decode(convArg.data.titleEncrypted)).toBe(
"enc:Time to weigh in",
);
// The body is encrypted at rest (Bytes), role is assistant, and the
// message hangs off the new conversation. No raw plaintext column.
+126 -41
View File
@@ -31,9 +31,9 @@ import type {
/**
* Title-from-message — first 80 chars trimmed, ellipsis on overflow.
* Stays plain text (the history rail needs to render without paying
* the per-message decrypt cost), so callers should pass already-
* sanitised input.
* Callers should pass already-sanitised input. The result is stored
* encrypted (`titleEncrypted`, v1.39.3): it is the opening words of the
* person's first message, which is as sensitive as the message itself.
*/
export function summariseTitle(input: string): string {
const collapsed = input.replace(/\s+/g, " ").trim();
@@ -301,15 +301,44 @@ export interface AppendMessageParams {
* Create a brand-new conversation row owned by `userId`. Caller is
* expected to immediately append the first user message.
*/
/**
* v1.39.3 — the title's at-rest form. Always a ciphertext: `summariseTitle`
* never returns an empty string and a rename is validated non-empty, so there
* is no "no title" case to store as NULL.
*/
export function encryptConversationTitle(
title: string,
): Uint8Array<ArrayBuffer> {
return encryptToBytes(title);
}
/**
* v1.39.3 — read a conversation title, ciphertext first. The readable column
* is consulted only when there is no ciphertext, which is a row the free-text
* encryption backfill has not reached yet. A present ciphertext that does not
* decrypt throws, the same fail-closed posture as every note reader
* (`readNote`); it never falls back to the readable column.
*/
export function readConversationTitle(row: {
title: string | null;
titleEncrypted: Uint8Array | null;
}): string {
if (row.titleEncrypted && row.titleEncrypted.byteLength > 0) {
return decryptFromBytes(row.titleEncrypted);
}
return row.title ?? "";
}
export async function createConversation(
params: CreateConversationParams,
): Promise<CoachConversationDTO> {
const attachmentIds = params.attachmentIds ?? [];
const title = summariseTitle(params.title);
const row = await prisma.$transaction(async (tx) => {
const conversation = await tx.coachConversation.create({
data: {
userId: params.userId,
title: summariseTitle(params.title),
titleEncrypted: encryptConversationTitle(title),
documentScoped: params.documentScoped ?? false,
},
});
@@ -327,7 +356,7 @@ export async function createConversation(
});
return {
id: row.id,
title: row.title,
title,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
messageCount: 0,
@@ -439,7 +468,7 @@ export async function recordProactiveNudge(
const conversation = await tx.coachConversation.create({
data: {
userId: params.userId,
title: summariseTitle(params.title),
titleEncrypted: encryptConversationTitle(summariseTitle(params.title)),
},
});
const message = await tx.coachMessage.create({
@@ -559,7 +588,7 @@ export async function fetchConversationWithMessages(
const attachments = mapAttachments(row.attachments);
return {
id: row.id,
title: row.title,
title: readConversationTitle(row),
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
messageCount: messages.length,
@@ -586,15 +615,31 @@ export interface ListConversationsParams {
/**
* v1.30.2 (QoL H1) — optional server-side title search for the history
* rail + the standalone conversations page. Case-insensitive substring
* match against `title` only — the only plaintext column on the row;
* `CoachMessage.encryptedContent` cannot be searched without decrypting
* every message, so message BODIES are out of scope for this pass (see
* the route's doc comment). Trimmed empty string is treated as "no
* filter", matching the pre-existing cursor/limit handling style.
* match against the title only; message BODIES are out of scope (see the
* route's doc comment). Since v1.39.3 the title is encrypted at rest, so
* the match runs on the decrypted titles in memory rather than in SQL (see
* `TITLE_SEARCH_SCAN_CAP`). Trimmed empty string is treated as "no filter",
* matching the pre-existing cursor/limit handling style.
*/
q?: string;
}
/**
* v1.39.3 — how many of an account's conversations a title search decrypts,
* newest first. The title is encrypted at rest, so SQL cannot match it; the
* search reads id + title for the account, decrypts, and filters in memory.
* A title is at most 80 characters, so a scan of the whole cap is a few
* milliseconds of AES. An account past the cap has its oldest threads left out
* of search results (never out of the unfiltered rail), which no real account
* comes near: the rail pages 20 at a time and a person opens a few a day.
*/
export const TITLE_SEARCH_SCAN_CAP = 5000;
const CONVERSATION_LIST_ORDER = [
{ updatedAt: "desc" as const },
{ id: "desc" as const },
];
/**
* Cursor-paginated list of conversations for the rail. Default limit
* 20, cap 50. Cursor is the id of the last item on the previous page;
@@ -608,43 +653,81 @@ export async function listConversations(
}> {
const limit = Math.min(Math.max(params.limit ?? 20, 1), 50);
const q = params.q?.trim();
const rows = await prisma.coachConversation.findMany({
where: {
userId: params.userId,
...(params.attachedDocumentId
? { attachments: { some: { documentId: params.attachedDocumentId } } }
: {}),
...(q ? { title: { contains: q, mode: "insensitive" } } : {}),
},
orderBy: [{ updatedAt: "desc" }, { id: "desc" }],
take: limit + 1,
...(params.cursor
? {
cursor: { id: params.cursor },
skip: 1,
}
const scope = {
userId: params.userId,
...(params.attachedDocumentId
? { attachments: { some: { documentId: params.attachedDocumentId } } }
: {}),
include: {
_count: { select: { messages: true } },
// v1.29.x (S7) — the live attachment set (label columns only) so the rail
// can badge a fenced thread with a paperclip + the first document's title.
// Empty on a health thread.
attachments: {
orderBy: { addedAt: "asc" },
include: { document: { select: { title: true, filename: true } } },
},
};
const include = {
_count: { select: { messages: true } },
// v1.29.x (S7) — the live attachment set (label columns only) so the rail
// can badge a fenced thread with a paperclip + the first document's title.
// Empty on a health thread.
attachments: {
orderBy: { addedAt: "asc" as const },
include: { document: { select: { title: true, filename: true } } },
},
});
};
const page = rows.slice(0, limit);
const nextCursor = rows.length > limit ? page[page.length - 1].id : null;
let page;
let nextCursor: string | null;
if (q) {
// Match on the decrypted titles, then page the matching ids with the same
// cursor contract the SQL path has: the cursor is the last id of the
// previous page, and an unknown cursor yields an empty page.
const needle = q.toLocaleLowerCase();
const candidates = await prisma.coachConversation.findMany({
where: scope,
orderBy: CONVERSATION_LIST_ORDER,
take: TITLE_SEARCH_SCAN_CAP,
select: { id: true, title: true, titleEncrypted: true },
});
const matching = candidates
.filter((c) =>
readConversationTitle(c).toLocaleLowerCase().includes(needle),
)
.map((c) => c.id);
let start = 0;
if (params.cursor) {
const at = matching.indexOf(params.cursor);
start = at === -1 ? matching.length : at + 1;
}
const pageIds = matching.slice(start, start + limit);
nextCursor =
matching.length > start + limit ? pageIds[pageIds.length - 1] : null;
const rows = await prisma.coachConversation.findMany({
where: { ...scope, id: { in: pageIds } },
include,
});
const byId = new Map(rows.map((r) => [r.id, r]));
page = pageIds.flatMap((id) => {
const row = byId.get(id);
return row ? [row] : [];
});
} else {
const rows = await prisma.coachConversation.findMany({
where: scope,
orderBy: CONVERSATION_LIST_ORDER,
take: limit + 1,
...(params.cursor
? {
cursor: { id: params.cursor },
skip: 1,
}
: {}),
include,
});
page = rows.slice(0, limit);
nextCursor = rows.length > limit ? page[page.length - 1].id : null;
}
return {
conversations: page.map((r) => {
const attachments = mapAttachments(r.attachments);
return {
id: r.id,
title: r.title,
title: readConversationTitle(r),
createdAt: r.createdAt.toISOString(),
updatedAt: r.updatedAt.toISOString(),
messageCount: r._count.messages,
@@ -669,7 +752,9 @@ export async function renameConversation(
): Promise<{ id: string; title: string } | null> {
const { count } = await prisma.coachConversation.updateMany({
where: { id: conversationId, userId },
data: { title },
// The readable column is cleared on a rename so a legacy row never keeps
// its old title beside the new ciphertext.
data: { title: null, titleEncrypted: encryptConversationTitle(title) },
});
return count === 1 ? { id: conversationId, title } : null;
}
+4 -1
View File
@@ -119,7 +119,10 @@ export async function surfaceCoachReminders(
await prisma.$transaction(async (tx) => {
const conversation = await tx.coachConversation.create({
data: { userId, title: title.slice(0, 80) },
data: {
userId,
titleEncrypted: encryptToBytes(title.slice(0, 80)),
},
});
await tx.coachMessage.create({
data: {
+46
View File
@@ -0,0 +1,46 @@
/**
* v1.39.3 — what the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is
* currently standing in for, for the admin AI settings page.
*
* Null when `true` is not set. Otherwise the distinct origins of every saved
* AI base URL on the instance (each account's Local and gateway URL, and the
* instance-wide admin base URL) that point at the operator's network and that
* no `AI_PRIVATE_ORIGINS` or legacy host-list entry covers yet. Listing them is
* the migration path: the operator copies them into `AI_PRIVATE_ORIGINS`, and
* `=true` can go. A non-admin account's URL is included on purpose; `=true` no
* longer covers it, so it is exactly the one that needs an entry.
*
* An origin is scheme, host and port, nothing more, so no path or credential
* a person typed into a base URL reaches the page.
*/
import { prisma } from "@/lib/db";
import {
legacyAnyHostConfigured,
originNeedingAiGrant,
} from "@/lib/ai/local-host-allowlist";
export interface LegacyPrivateHostReport {
originsInUse: string[];
}
export async function legacyPrivateHostReport(
adminBaseUrl: string | null | undefined,
): Promise<LegacyPrivateHostReport | null> {
if (!legacyAnyHostConfigured()) return null;
const rows = await prisma.user.findMany({
where: {
OR: [{ aiBaseUrl: { not: null } }, { aiCompatBaseUrl: { not: null } }],
},
select: { aiBaseUrl: true, aiCompatBaseUrl: true },
});
const origins = new Set<string>();
for (const url of [
adminBaseUrl,
...rows.flatMap((r) => [r.aiBaseUrl, r.aiCompatBaseUrl]),
]) {
if (!url) continue;
const origin = originNeedingAiGrant(url);
if (origin) origins.add(origin);
}
return { originsInUse: [...origins].sort() };
}
+21 -9
View File
@@ -1,6 +1,6 @@
import { safeFetch } from "@/lib/safe-fetch";
import { annotate } from "@/lib/logging/context";
import { isLocalAiHostAllowed } from "./local-host-allowlist";
import { aiEgressPolicyFor } from "./local-host-allowlist";
import type {
AIProvider,
AiMessage,
@@ -27,6 +27,13 @@ interface LocalClientConfig {
apiKey?: string | null;
model: string;
baseUrl: string;
/**
* v1.39.3 — true when the base URL belongs to the operator: the
* instance-wide admin provider, or provider settings saved on an admin
* account. Only such a URL is still covered by the deprecated
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` (see `local-host-allowlist.ts`).
*/
operatorTrusted?: boolean;
}
const STRICT_JSON_PREFIX =
@@ -193,11 +200,14 @@ export class LocalOpenAICompatibleClient implements AIProvider {
// pin (issue #217) extends the same flag with a connect-time
// resolved-IP check to also defeat DNS rebinding. Operators who
// legitimately point at a self-hosted Ollama / LM Studio on an
// RFC1918 address opt in via `ALLOW_LOCAL_AI_PRIVATE_HOSTS` — either the
// legacy `=true` (any private host) or a comma-separated host allowlist
// (only those hostnames). Enforced at write-time in /api/user/ai-provider
// too. v1.18.7 (SECURITY LOW) — the binary flag became a host allowlist.
const allowPrivate = isLocalAiHostAllowed(url);
// RFC1918 address opt in via `AI_PRIVATE_ORIGINS` (exact origins) or the
// legacy host list. A granted origin is dialled through the pinned
// operator-approved dispatcher, never unpinned, so the metadata range and
// link-local stay unreachable even for a granted name (v1.39.3). Enforced
// at write-time in /api/user/ai-provider too.
const egress = aiEgressPolicyFor(url, {
operatorTrusted: this.config.operatorTrusted,
});
const res = await safeFetch(
url,
{
@@ -210,7 +220,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
{
// v1.22 (#89) — honour the caller's per-request timeout override; default 60 s.
timeoutMs: params.timeoutMs ?? 60_000,
requirePublicHost: !allowPrivate,
...egress,
signal: params.signal,
},
);
@@ -335,7 +345,9 @@ export class LocalOpenAICompatibleClient implements AIProvider {
onDelta: (delta: string) => void,
): Promise<CompletionResult> {
const { url, headers, body } = this.buildRequest(params, true);
const allowPrivate = isLocalAiHostAllowed(url);
const egress = aiEgressPolicyFor(url, {
operatorTrusted: this.config.operatorTrusted,
});
// Idle controller: aborts the upstream when no chunk arrives within the
// per-idle window. Composed with the caller's cancel signal so a client
@@ -355,7 +367,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
// Absolute backstop only — the per-idle timer below is the real
// ceiling. A server that streams forever still cannot pin a worker.
timeoutMs: STREAM_ABSOLUTE_CEILING_MS,
requirePublicHost: !allowPrivate,
...egress,
signal,
},
);
+220 -56
View File
@@ -1,76 +1,240 @@
/**
* v1.18.7 (SECURITY LOW) — host allowlist for local/private AI endpoints.
* Which private hosts an AI base URL may reach, and how the call is dialled.
*
* The operator escape-hatch for pointing the server at a self-hosted
* Ollama / LM Studio on an RFC1918 address used to be a single binary flag,
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true`, which opened EVERY private/internal
* host at once — including the cloud-metadata endpoint and any internal admin
* panel. This narrows it to an explicit allowlist while keeping the binary
* `true` for backward compatibility:
* A base URL for the Local provider, the OpenAI-compatible gateway and the
* operator's admin key is typed by a person, and for the first two that
* person is any user of the instance. By default only public hosts are
* reachable. An operator who runs Ollama, LM Studio or a LiteLLM gateway on
* their own network grants it through the environment, the same exact-origin
* grammar `NOTIFICATION_PRIVATE_ORIGINS` and `NIGHTSCOUT_PRIVATE_ORIGINS` use
* (`src/lib/private-origin-policy.ts`):
*
* - unset / "" / "false" → no private host allowed (the secure default).
* - "true" → ANY private host allowed (legacy behaviour).
* - "ollama.lan, 10.0.0.5" → ONLY those exact hostnames allowed; every
* other private host is still rejected.
* AI_PRIVATE_ORIGINS="http://ollama.lan:11434,http://10.0.0.5:4000"
*
* This is layered ON TOP of the `safeFetch` / `isPublicUrl` floor: a public
* URL never needs the allowlist, and a private host not on the list is
* rejected exactly as before. The allowlist only ever WIDENS what the
* operator deliberately permits, never narrows the public-host floor.
* A grant is one `scheme://host[:port]`. It is never a wildcard, suffix or
* range. A granted call is dialled through the pinned operator-approved
* dispatcher with redirects forbidden, so a listed name that resolves to the
* metadata range, link-local or the unspecified address is dropped at connect
* time, and a literal in those ranges cannot be granted at all. Nothing in a
* request or a settings field can confer a grant.
*
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS` (v1.18.7) stays readable. A comma-separated
* host list (`ollama.lan,10.0.0.5`) still grants those exact hosts, on any
* port, dialled the same pinned way.
*
* Its `=true` form used to open every private host to every user of the
* instance. From v1.39.3 it is deprecated and narrowed for one release cycle
* rather than removed, so a single-user self-host with Ollama on the LAN keeps
* working after a patch update: it still grants a private host, but only to
* the configurations the operator owns (the instance-wide admin provider, and
* provider settings saved on an admin account), and only through the same
* pinned dispatcher, so the metadata range, link-local and the unspecified
* address stay unreachable. A non-admin account's own base URL needs an exact
* origin in `AI_PRIVATE_ORIGINS`. The boot summary and the admin AI settings
* page say so and name the origins in use.
*/
import { isIP } from "node:net";
const ENV_VAR = "ALLOW_LOCAL_AI_PRIVATE_HOSTS";
import {
isNeverGrantableHost,
originOfUrl,
parsePrivateOrigins,
} from "@/lib/private-origin-policy";
import { isPublicUrl } from "@/lib/validations/notifications";
/** Parsed allowlist policy from the env var. */
type LocalHostPolicy =
| { kind: "none" }
| { kind: "any" }
| { kind: "hosts"; hosts: ReadonlySet<string> };
const ORIGINS_ENV = "AI_PRIVATE_ORIGINS";
const LEGACY_ENV = "ALLOW_LOCAL_AI_PRIVATE_HOSTS";
function parsePolicy(): LocalHostPolicy {
const raw = process.env[ENV_VAR]?.trim();
if (!raw || raw.toLowerCase() === "false") return { kind: "none" };
if (raw.toLowerCase() === "true") return { kind: "any" };
const hosts = raw
.split(",")
.map((h) => h.trim().toLowerCase())
.filter((h) => h.length > 0);
if (hosts.length === 0) return { kind: "none" };
return { kind: "hosts", hosts: new Set(hosts) };
/** The operator-facing sentence for a legacy `=true`. Shared with the boot summary. */
export const LEGACY_ANY_HOST_WARNING =
`${LEGACY_ENV}=true is deprecated, set ${ORIGINS_ENV}=<origin> ` +
`(for example ${ORIGINS_ENV}="http://ollama.lan:11434"); =true will be ` +
`removed in a later release. Until then it only covers the admin AI ` +
`provider and AI settings saved on an admin account.`;
/**
* Who a base URL belongs to. `operatorTrusted` is true for the instance-wide
* admin provider and for provider settings saved on an admin account: the
* configurations the deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` still
* covers. Everything else (any non-admin account's own base URL) is not.
*/
export interface AiUrlOwner {
operatorTrusted?: boolean;
}
/** Extract the lowercased hostname from a URL string, or null if unparseable. */
function hostnameOf(url: string): string | null {
try {
return new URL(url).hostname.toLowerCase();
} catch {
return null;
interface Grants {
origins: ReadonlySet<string>;
legacyHosts: ReadonlySet<string>;
}
let cachedKey: string | null = null;
let cached: Grants = { origins: new Set(), legacyHosts: new Set() };
const warned = new Set<string>();
function warnOnce(message: string): void {
if (warned.has(message)) return;
warned.add(message);
console.warn(message);
}
/** True when the legacy variable carries its retired `=true` form. */
export function legacyAnyHostConfigured(
env: Record<string, string | undefined> = process.env,
): boolean {
return env[LEGACY_ENV]?.trim().toLowerCase() === "true";
}
function parseLegacyHosts(raw: string | undefined): ReadonlySet<string> {
const value = raw?.trim() ?? "";
if (!value || value.toLowerCase() === "false") return new Set();
// `true` is handled per call in `grantedAiOrigin` (operator-owned
// configurations only); it is not a host list.
if (value.toLowerCase() === "true") {
warnOnce(LEGACY_ANY_HOST_WARNING);
return new Set();
}
const hosts = new Set<string>();
for (const entry of value.split(",")) {
const host = entry.trim().toLowerCase();
if (!host) continue;
// A host, not a URL: anything with a scheme, path, port or wildcard is a
// typo for the origin list, and granting a guess would widen it.
const bare = host.replace(/^\[|\]$/g, "");
const isIpv6 = isIP(bare) === 6;
if (!/^[a-z0-9.\-[\]:]+$/.test(host) || (host.includes(":") && !isIpv6)) {
warnOnce(
`${LEGACY_ENV}: ignoring entry "${host.slice(0, 200)}": a host name or address, nothing else; use ${ORIGINS_ENV} for a full origin`,
);
continue;
}
if (isNeverGrantableHost(host)) {
warnOnce(
`${LEGACY_ENV}: ignoring entry "${host}": the unspecified address, link-local and metadata addresses cannot be granted`,
);
continue;
}
hosts.add(bare);
}
return hosts;
}
function grants(): Grants {
const originsRaw = process.env.AI_PRIVATE_ORIGINS;
const legacyRaw = process.env.ALLOW_LOCAL_AI_PRIVATE_HOSTS;
const key = `${originsRaw ?? ""}\u0000${legacyRaw ?? ""}`;
if (key === cachedKey) return cached;
cached = {
origins: parsePrivateOrigins(originsRaw, (entry, reason) => {
warnOnce(`${ORIGINS_ENV}: ignoring entry "${entry}": ${reason}`);
}),
legacyHosts: parseLegacyHosts(legacyRaw),
};
cachedKey = key;
return cached;
}
/**
* True when the operator has explicitly permitted this (private) host for a
* local AI endpoint. `"true"` permits any host (legacy); a host list permits
* only listed hostnames. An unparseable URL is never permitted.
*
* NOTE: this answers "did the operator opt this host in?", NOT "is this host
* public?". Callers still run the `isPublicUrl` / `safeFetch` floor; this is
* the private-host override on top of it.
* The granted origin for this URL, or null. A URL is granted when its exact
* origin is listed in `AI_PRIVATE_ORIGINS`, or its host is listed in the
* legacy host list. A host no grant may name is never granted, whatever the
* lists say.
*/
export function isLocalAiHostAllowed(url: string): boolean {
const policy = parsePolicy();
if (policy.kind === "none") return false;
if (policy.kind === "any") return true;
const host = hostnameOf(url);
return host !== null && policy.hosts.has(host);
export function grantedAiOrigin(
url: string,
owner: AiUrlOwner = {},
): string | null {
// Parse the grants first, so a malformed entry is reported on the first
// call that could have used it rather than never.
const { origins, legacyHosts } = grants();
const origin = originOfUrl(url);
if (!origin) return null;
const hostname = new URL(origin).hostname.toLowerCase();
// Both lists refuse a never-grantable literal when they are parsed, so
// neither can hold one; a listed name that resolves into those ranges is
// dropped by the pinned dispatcher at dial time.
if (origins.has(origin)) return origin;
if (legacyHosts.has(hostname.replace(/^\[|\]$/g, ""))) return origin;
// Deprecated `=true`: an operator-owned configuration only, and never a
// literal no grant may name. A name that resolves into those ranges is
// dropped by the pinned dispatcher at dial time.
if (
owner.operatorTrusted === true &&
legacyAnyHostConfigured() &&
!isNeverGrantableHost(hostname)
) {
return origin;
}
return null;
}
/**
* Whether `safeFetch` must require a public host for this URL: true unless the
* operator has opted this private host into the allowlist. Kept as a thin
* wrapper so the call sites read `requirePublicHost: requirePublicHostFor(url)`
* and the conditional intent stays obvious.
* True when the operator granted this URL's origin. Answers "may this private
* host be saved and called?", never "is it public?": callers still accept a
* public URL without any grant.
*/
export function requirePublicHostFor(url: string): boolean {
return !isLocalAiHostAllowed(url);
export function isLocalAiHostAllowed(
url: string,
owner: AiUrlOwner = {},
): boolean {
return grantedAiOrigin(url, owner) !== null;
}
/**
* The `safeFetch` destination policy for an AI call to `url`: the pinned
* public dispatcher unless the operator granted the origin, and then the
* pinned operator-approved dispatcher for that one origin. Spread into the
* `safeFetch` options.
*/
export function aiEgressPolicyFor(
url: string,
owner: AiUrlOwner = {},
):
| { requirePublicHost: true }
| { requirePublicHost: false; operatorApprovedPrivateOrigin: string } {
const origin = grantedAiOrigin(url, owner);
return origin === null
? { requirePublicHost: true }
: { requirePublicHost: false, operatorApprovedPrivateOrigin: origin };
}
/**
* Host names that only resolve on a local network. `isPublicUrl` judges the
* text of a name, not where it resolves, so `ollama.lan` reads as public to it
* while the connect-time pin would refuse it.
*/
const LAN_SUFFIXES = [
".lan",
".local",
".home",
".home.arpa",
".internal",
".intranet",
".localdomain",
".corp",
];
/**
* The origin an operator would add to `AI_PRIVATE_ORIGINS` for this saved
* base URL, or null when it needs none: a hosted endpoint, an address no grant
* can open, an origin the exact list or the legacy host list already covers,
* or an unparseable value. Used by the admin AI settings page to name what the
* deprecated `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` is currently standing in for.
*/
export function originNeedingAiGrant(url: string): string | null {
const origin = originOfUrl(url);
if (!origin) return null;
const hostname = new URL(origin).hostname.toLowerCase();
if (isNeverGrantableHost(hostname)) return null;
const lanName =
!hostname.includes(".") ||
LAN_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
if (isPublicUrl(url) && !lanName) return null;
if (grantedAiOrigin(url) !== null) return null;
return origin;
}
/** Test helper: forget the parsed grants and the warnings already printed. */
export function _resetAiGrantsForTests(): void {
cachedKey = null;
warned.clear();
}
+15 -6
View File
@@ -8,7 +8,7 @@ import {
jsonModeDialectFor,
rememberJsonModeDialect,
} from "./json-dialect";
import { requirePublicHostFor } from "./local-host-allowlist";
import { aiEgressPolicyFor } from "./local-host-allowlist";
import {
buildOpenAIMessages,
buildOpenAITools,
@@ -46,6 +46,13 @@ interface OpenAIClientConfig {
* `codex` keeps the fully pinned posture.
*/
providerType?: OpenAIProviderType;
/**
* v1.39.3 — true when the base URL belongs to the operator: the
* instance-wide admin provider, or provider settings saved on an admin
* account. Only such a URL is still covered by the deprecated
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` (see `local-host-allowlist.ts`).
*/
operatorTrusted?: boolean;
}
type OpenAIProviderType = "admin-key" | "codex" | "openai-compatible";
@@ -152,7 +159,8 @@ export class OpenAIClient implements AIProvider {
// v1.21.5 — honour the caller's per-request timeout override; default 60 s.
// v1.33.1 (#470) — the gateway tag reuses the Local provider's host
// policy verbatim: public hosts always; a private host only when the
// operator allowlisted it via ALLOW_LOCAL_AI_PRIVATE_HOSTS.
// operator granted its origin (`AI_PRIVATE_ORIGINS` or the legacy host
// list), and then only through the pinned operator-approved dispatcher.
//
// v1.37.30 — the `admin-key` tag consults the same policy. Its base URL
// is the one other operator-typed URL in this client (an operator can
@@ -167,10 +175,11 @@ export class OpenAIClient implements AIProvider {
// operator legitimately redirects.
{
timeoutMs: params.timeoutMs ?? 60_000,
requirePublicHost:
this.isGateway || this.type === "admin-key"
? requirePublicHostFor(this.config.baseUrl)
: true,
...(this.isGateway || this.type === "admin-key"
? aiEgressPolicyFor(url, {
operatorTrusted: this.config.operatorTrusted,
})
: { requirePublicHost: true as const }),
signal: params.signal,
},
);
+34 -4
View File
@@ -55,8 +55,20 @@ type UserAIRow = {
aiCompatBaseUrl: string | null;
aiCompatKeyEncrypted: string | null;
aiCompatModel: string | null;
/**
* v1.39.3 — the owning account's role. Provider settings saved on an admin
* account are the operator's own, and only those (plus the instance-wide
* admin provider) are still covered by the deprecated
* `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true`.
*/
role: string;
};
/** v1.39.3 — see `UserAIRow.role`. */
function isOperatorOwned(row: { role: string } | null | undefined): boolean {
return row?.role === "ADMIN";
}
/**
* v1.33.1 (#470) — build the OpenAI-compatible gateway client from the user's
* three dedicated columns. The signature names every column this provider is
@@ -83,6 +95,7 @@ function buildCompatProvider(row: {
aiCompatBaseUrl: string | null;
aiCompatKeyEncrypted: string | null;
aiCompatModel: string | null;
role: string;
}): AIProvider | null {
const model = row.aiCompatModel?.trim() || row.aiModel?.trim() || null;
if (!row.aiCompatBaseUrl || !model) return null;
@@ -91,6 +104,7 @@ function buildCompatProvider(row: {
model,
baseUrl: row.aiCompatBaseUrl,
providerType: "openai-compatible",
operatorTrusted: isOperatorOwned(row),
});
}
@@ -122,6 +136,7 @@ function buildUserProvider(row: UserAIRow): AIProvider | null {
: null,
model: row.aiModel ?? "local-model",
baseUrl: row.aiBaseUrl,
operatorTrusted: isOperatorOwned(row),
});
}
case "OPENAI_COMPATIBLE": {
@@ -212,6 +227,8 @@ async function resolveAdminProvider(): Promise<AIProvider> {
apiKey: decrypt(settings.adminAiKeyEncrypted),
model: settings.adminAiModel ?? "gpt-4o",
baseUrl,
// The instance-wide provider is the operator's own configuration.
operatorTrusted: true,
});
}
@@ -480,6 +497,7 @@ export async function resolveProvider(userId: string): Promise<AIProvider> {
aiCompatBaseUrl: true,
aiCompatKeyEncrypted: true,
aiCompatModel: true,
role: true,
managedProfileAt: true,
},
});
@@ -548,6 +566,7 @@ export async function resolveProviderChain(
aiCompatBaseUrl: true,
aiCompatKeyEncrypted: true,
aiCompatModel: true,
role: true,
aiProviderChain: true,
useCentralCodex: true,
managedProfileAt: true,
@@ -1023,6 +1042,7 @@ async function resolveProviderForType(
aiCompatBaseUrl: string | null;
aiCompatKeyEncrypted: string | null;
aiCompatModel: string | null;
role: string;
} | null;
policy: ProviderCredentialPolicy;
},
@@ -1062,6 +1082,7 @@ async function resolveProviderForType(
: null,
model: ctx.userRow.aiModel ?? "local-model",
baseUrl: ctx.userRow.aiBaseUrl,
operatorTrusted: isOperatorOwned(ctx.userRow),
});
}
case "openai-compatible": {
@@ -1138,8 +1159,12 @@ export async function resolveProviderForTest(
aiCompatBaseUrl: true,
aiCompatKeyEncrypted: true,
aiCompatModel: true,
role: true,
},
});
// v1.39.3 — an unsaved override is tested under the same owner rule as a
// saved one: it belongs to the calling account.
const owner = { operatorTrusted: isOperatorOwned(stored) };
const provider = (override.provider ?? stored?.aiProvider ?? "")
.toString()
@@ -1189,9 +1214,9 @@ export async function resolveProviderForTest(
if (!baseUrl) {
throw new AITestConfigError(422, "Local provider requires a base URL");
}
// v1.18.7 (SECURITY LOW) — host allowlist (`true` = any private host;
// a comma-separated host list = only those) replaces the binary flag.
const allowPrivate = isLocalAiHostAllowed(baseUrl);
// A private host only when the operator granted its origin
// (`AI_PRIVATE_ORIGINS` or the legacy host list; v1.39.3).
const allowPrivate = isLocalAiHostAllowed(baseUrl, owner);
if (!allowPrivate && !isPublicUrl(baseUrl)) {
throw new AITestConfigError(
422,
@@ -1207,6 +1232,7 @@ export async function resolveProviderForTest(
apiKey,
model: model || "local-model",
baseUrl,
...owner,
});
}
case "OPENAI_COMPATIBLE": {
@@ -1227,7 +1253,10 @@ export async function resolveProviderForTest(
"OpenAI-compatible provider requires a base URL",
);
}
if (!isLocalAiHostAllowed(compatBaseUrl) && !isPublicUrl(compatBaseUrl)) {
if (
!isLocalAiHostAllowed(compatBaseUrl, owner) &&
!isPublicUrl(compatBaseUrl)
) {
throw new AITestConfigError(
422,
"Base URL points to an internal/private host",
@@ -1253,6 +1282,7 @@ export async function resolveProviderForTest(
model: compatModel,
baseUrl: compatBaseUrl,
providerType: "openai-compatible",
...owner,
});
}
case "CHATGPT_OAUTH": {
@@ -129,3 +129,23 @@ describe("formatReadiness + logReadinessSummary", () => {
expect(errors).toHaveLength(1);
});
});
describe("collectReadiness — retired AI grant", () => {
it("warns, without blocking, when ALLOW_LOCAL_AI_PRIVATE_HOSTS is still `true`", () => {
const env = { ...baseEnv(), ALLOW_LOCAL_AI_PRIVATE_HOSTS: " TRUE " };
const line = lineFor(env, "ALLOW_LOCAL_AI_PRIVATE_HOSTS");
expect(line?.status).toBe("warn");
expect(line?.detail).toContain("AI_PRIVATE_ORIGINS");
expect(collectReadiness(env).hasBlocker).toBe(false);
});
it("says nothing for a host list or an unset variable", () => {
expect(
lineFor(
{ ...baseEnv(), ALLOW_LOCAL_AI_PRIVATE_HOSTS: "ollama.lan" },
"ALLOW_LOCAL_AI_PRIVATE_HOSTS",
),
).toBeUndefined();
expect(lineFor(baseEnv(), "ALLOW_LOCAL_AI_PRIVATE_HOSTS")).toBeUndefined();
});
});
+18
View File
@@ -15,6 +15,11 @@
* signpost in front of them.
*/
import {
LEGACY_ANY_HOST_WARNING,
legacyAnyHostConfigured,
} from "@/lib/ai/local-host-allowlist";
export interface ReadinessLine {
/** Short label, e.g. "ENCRYPTION_KEY". */
label: string;
@@ -121,6 +126,19 @@ export function collectReadiness(env: Env = process.env): ReadinessReport {
: "Secure cookie flag OFF — fine for plain-HTTP LAN; do NOT expose plain HTTP to the internet",
});
// --- Deprecated AI grant (a warning, never a blocker) -------------------
// `ALLOW_LOCAL_AI_PRIVATE_HOSTS=true` used to open every private host on
// the network to every user's AI base URL. Since v1.39.3 it only covers the
// operator's own AI configurations and is on its way out; this line names
// the replacement. The admin AI settings page lists the origins in use.
if (legacyAnyHostConfigured(env)) {
lines.push({
label: "ALLOW_LOCAL_AI_PRIVATE_HOSTS",
status: "warn",
detail: LEGACY_ANY_HOST_WARNING,
});
}
// --- Optional subsystems (informational) --------------------------------
const optional: Array<[string, boolean]> = [
["SMTP (email)", allSet(env, ["SMTP_HOST", "SMTP_PORT", "SMTP_FROM"])],
+8
View File
@@ -216,6 +216,8 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
// ───── Coach (Bytes columns) ─────
{ model: "CoachMessage", field: "encryptedContent", kind: "bytes" },
{ model: "CoachConversation", field: "summaryEncrypted", kind: "bytes" },
// v1.39.3 — the title: the opening words of the first message, or a rename.
{ model: "CoachConversation", field: "titleEncrypted", kind: "bytes" },
{ model: "CoachFact", field: "factEncrypted", kind: "bytes" },
// v1.21.3 (B1) — Coach goal / if-then plan free-text columns.
{ model: "CoachPlan", field: "ifCueEncrypted", kind: "bytes" },
@@ -318,6 +320,12 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
{ model: "MedicationDoseChange", field: "noteEncrypted", kind: "bytes" },
{ model: "MedicationInventoryItem", field: "notesEncrypted", kind: "bytes" },
// ───── v1.39.3 custom-metric reading note (Bytes column) ─────
// The one free-text field on a reading of a metric the person defined. The
// catalog columns (name, unit, description) stay plaintext: they are the
// definition of a series, listed and sorted by.
{ model: "CustomMetricEntry", field: "noteEncrypted", kind: "bytes" },
// ───── v1.25 mental-health screener item answers (Bytes column) ─────
// The PHQ-9 / GAD-7 per-item responses (incl. the safety-critical item 9)
// ride a single AES-256-GCM blob. The most sensitive payload in the wave;
+11 -5
View File
@@ -1,15 +1,19 @@
/**
* v1.25.5 — server-side serialisers for the user-defined custom-metric store.
*
* Custom metrics are PLAINTEXT (name / unit / description), so — unlike the
* Biomarker catalog with its AES-256-GCM `contextEncrypted` codec — there is no
* encrypt/decrypt layer here. These helpers just map a Prisma row into the
* stable wire DTO the web + iOS clients render.
* The catalog columns (name / unit / description) are plaintext: they are the
* definition of a series and are listed and sorted by. The one free-text field
* on a reading, its note, is AES-256-GCM at rest since v1.39.3
* (`noteEncrypted`), read here through the shared note boundary. These
* helpers map a Prisma row into the stable wire DTO the web + iOS clients
* render.
*
* The store is deliberately ISOLATED from the closed `MeasurementType` system:
* no rollup, no sync, no FHIR, no insights. Charts read entries LIVE.
*/
import { readNote } from "@/lib/crypto/note-cipher";
/** A custom-metric catalog row as the API serialises it. */
export interface CustomMetricRow {
id: string;
@@ -37,7 +41,9 @@ export interface CustomMetricEntryRow {
value: number;
unit: string;
measuredAt: Date;
/** Legacy readable note, set only on rows the backfill has not reached. */
note: string | null;
noteEncrypted: Uint8Array | null;
createdAt: Date;
}
@@ -79,7 +85,7 @@ export function serialiseCustomMetricEntry(row: CustomMetricEntryRow) {
value: row.value,
unit: row.unit,
measuredAt: row.measuredAt.toISOString(),
note: row.note,
note: readNote(row.noteEncrypted, row.note),
createdAt: row.createdAt.toISOString(),
};
}
@@ -0,0 +1,152 @@
/**
* v1.39.3 — the Coach conversation title in both backup formats.
*
* A disaster-recovery file carries the title as stored: the ciphertext, or
* for a row the free-text backfill has not reached yet the old readable value
* (sealing it on the way out would make the export non-deterministic). A
* portable file carries it readable, for the person who owns it. A restore of
* any of these writes the ciphertext column and never the readable one, and a
* file written before this release (readable title only) still restores.
*/
import { describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import {
buildCoachBackupSection,
restoreCoachData,
type RestoredCoachConversation,
} from "../coach-backup";
import { parseBackupPayload } from "@/lib/validations/backup";
const at = "2026-09-20T10:00:00.000Z";
function row(id: string, title: string | null, sealed: string | null) {
return {
id,
title,
titleEncrypted: sealed === null ? null : encryptToBytes(sealed),
documentScoped: false,
summaryEncrypted: null,
summaryUpdatedAt: null,
summaryTurnCount: 0,
createdAt: new Date(at),
updatedAt: new Date(at),
messages: [],
attachments: [],
};
}
function prismaWith(rows: ReturnType<typeof row>[]) {
return {
coachConversation: { findMany: vi.fn().mockResolvedValue(rows) },
} as never;
}
const rows = [
row("c1", null, "Sealed title"),
row("c2", "Legacy readable title", null),
];
describe("coach backup title", () => {
it("a disaster-recovery file carries the title as stored", async () => {
const section = await buildCoachBackupSection(prismaWith(rows), "u1", {
purpose: "disaster-recovery",
});
const [sealed, legacy] = section.coachConversations;
expect(JSON.stringify(sealed)).not.toContain("Sealed title");
expect(sealed).not.toHaveProperty("title");
expect(
decryptFromBytes(Buffer.from(sealed.titleEncrypted!, "base64")),
).toBe("Sealed title");
expect(legacy).toMatchObject({ title: "Legacy readable title" });
expect(legacy).not.toHaveProperty("titleEncrypted");
// The same input exports byte-identically twice: nothing is sealed with
// a fresh IV on the way out.
const again = await buildCoachBackupSection(prismaWith(rows), "u1", {
purpose: "disaster-recovery",
});
expect(JSON.stringify(again)).toBe(JSON.stringify(section));
});
it("a portable file carries the readable title and no ciphertext", async () => {
const section = await buildCoachBackupSection(prismaWith(rows), "u1", {
purpose: "portable-export",
});
expect(section.coachConversations.map((c) => c.title)).toEqual([
"Sealed title",
"Legacy readable title",
]);
for (const c of section.coachConversations) {
expect(c).not.toHaveProperty("titleEncrypted");
}
});
it("a restore of either format, and of an older file, writes only the ciphertext", async () => {
const created: Array<Record<string, unknown>> = [];
const tx = {
coachMessage: { count: vi.fn().mockResolvedValue(0) },
coachConversationDocument: {
count: vi.fn().mockResolvedValue(0),
createMany: vi.fn(),
},
coachConversation: {
deleteMany: vi.fn().mockResolvedValue({ count: 0 }),
create: vi.fn(async (args: { data: Record<string, unknown> }) => {
created.push(args.data);
return args.data;
}),
},
};
const base = {
documentScoped: false,
createdAt: at,
updatedAt: at,
messages: [],
attachments: [],
};
const conversations: RestoredCoachConversation[] = [
{
...base,
id: "dr",
titleEncrypted: Buffer.from(encryptToBytes("From DR")).toString(
"base64",
),
},
{ ...base, id: "portable", title: "From a portable file" },
];
await restoreCoachData(
tx as never,
"u1",
{ coachConversations: conversations },
new Set(),
{ entries: [] } as never,
);
expect(created.map((d) => d.title)).toEqual([undefined, undefined]);
expect(
created.map((d) => decryptFromBytes(d.titleEncrypted as Uint8Array)),
).toEqual(["From DR", "From a portable file"]);
});
it("the wire schema accepts a pre-release file with a readable title", () => {
const parsed = parseBackupPayload({
schemaVersion: "2",
exportedAt: at,
userId: "u1",
coachConversations: [
{
id: "old",
title: "Written before the title was sealed",
documentScoped: false,
createdAt: at,
updatedAt: at,
},
],
});
expect(parsed.coachConversations[0].title).toBe(
"Written before the title was sealed",
);
});
});
@@ -1286,7 +1286,10 @@ describe("buildFullBackupPayload — profile and custom metrics", () => {
value: 44.5,
unit: "kg",
measuredAt: "2026-07-19T06:30:00.000Z",
// v1.39.3 — the fixture row predates the encryption backfill:
// a DR file carries it as stored, and the restore seals it.
note: "felt strong",
noteEncrypted: null,
}),
],
}),
@@ -578,8 +578,11 @@ describe("backup round trip — export, wire schema, restore", () => {
expect(entries[0]).toMatchObject({
value: 44.5,
unit: "kg",
note: "felt strong",
});
// v1.39.3 — the note comes back sealed; the readable column is never
// written, even for a row that was still readable when it was exported.
expect(entries[0]).not.toHaveProperty("note");
expect(entries[0].noteEncrypted).toBeInstanceOf(Uint8Array);
});
it("writes persisted pattern identity and dismissal evidence back", async () => {
+51 -5
View File
@@ -92,7 +92,14 @@ export interface CoachConversationBackupEntry {
/** Always carried: messages, attachments and the Coach's own facts, plans
* and reminders all address a conversation by it. */
id: string;
title: string;
/**
* The readable title. Present on a portable payload, on every file written
* before v1.39.3, and on a disaster-recovery payload for a row the free-text
* backfill had not reached yet.
*/
title?: string;
/** v1.39.3 — the title's ciphertext as base64. Disaster-recovery only. */
titleEncrypted?: string;
/**
* The permanent fence marker. Carried and restored verbatim; see the file
* header for why letting this default is a security regression and not a
@@ -130,6 +137,7 @@ export interface CoachBackupCounts {
const COACH_CONVERSATION_BACKUP_SELECT = {
id: true,
title: true,
titleEncrypted: true,
documentScoped: true,
summaryEncrypted: true,
summaryUpdatedAt: true,
@@ -174,6 +182,17 @@ function decryptTurnSoft(bytes: Uint8Array | null): string | null {
}
}
/** A title for a reader: ciphertext first, legacy readable column after. */
function readTitleSoft(row: {
title: string | null;
titleEncrypted: Uint8Array | null;
}): string {
if (row.titleEncrypted && row.titleEncrypted.byteLength > 0) {
return decryptTurnSoft(row.titleEncrypted) ?? "";
}
return row.title ?? "";
}
export async function buildCoachBackupSection(
prisma: Pick<PrismaClient, "coachConversation">,
userId: string,
@@ -190,7 +209,18 @@ export async function buildCoachBackupSection(
return {
coachConversations: rows.map((row) => ({
id: row.id,
title: row.title,
// A disaster-recovery file carries the stored bytes: the ciphertext,
// or, for a row the free-text backfill has not reached yet, the old
// readable title exactly as stored (the restore seals it). Sealing it
// here instead would make the export non-deterministic, and the
// streaming writer and this builder must produce the same file.
...(disasterRecovery && row.titleEncrypted
? {
titleEncrypted: Buffer.from(row.titleEncrypted).toString("base64"),
}
: disasterRecovery
? { title: row.title ?? "" }
: { title: readTitleSoft(row) }),
documentScoped: row.documentScoped,
...(disasterRecovery
? {
@@ -293,12 +323,17 @@ export type RestoredCoachMessage = Pick<
export type RestoredCoachConversation = Pick<
CoachConversationBackupEntry,
"id" | "title" | "documentScoped" | "createdAt" | "updatedAt"
"id" | "documentScoped" | "createdAt" | "updatedAt"
> &
OptionalNullable<
Pick<
CoachConversationBackupEntry,
"summaryEncrypted" | "summary" | "summaryUpdatedAt" | "summaryTurnCount"
| "title"
| "titleEncrypted"
| "summaryEncrypted"
| "summary"
| "summaryUpdatedAt"
| "summaryTurnCount"
>
> & {
messages: RestoredCoachMessage[];
@@ -343,7 +378,9 @@ export async function restoreCoachData(
data: {
id: conversation.id,
userId: ownerId,
title: conversation.title,
// v1.39.3 — the title is stored encrypted; the readable column is
// never written on a restore.
titleEncrypted: resolveTitleBytes(conversation),
// Verbatim. See the file header: this is the one field whose loss is a
// security regression rather than a gap in the history.
documentScoped: conversation.documentScoped,
@@ -420,6 +457,15 @@ function resolveTurnBytes(
return encryptToBytes(message.content ?? "");
}
function resolveTitleBytes(
conversation: RestoredCoachConversation,
): Uint8Array<ArrayBuffer> {
if (conversation.titleEncrypted !== undefined) {
return decodeBase64(conversation.titleEncrypted);
}
return encryptToBytes(conversation.title ?? "");
}
function resolveSummaryBytes(
conversation: RestoredCoachConversation,
): Uint8Array<ArrayBuffer> | null {
+60 -2
View File
@@ -25,6 +25,7 @@ import { Buffer } from "node:buffer";
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
import { decryptFromBytes, encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { encryptNote } from "@/lib/crypto/note-cipher";
import { getEvent } from "@/lib/logging/context";
import {
DEFAULT_HEALTH_PROFILE_AI_SECTIONS,
@@ -116,7 +117,16 @@ export interface CustomMetricEntryBackupEntry {
/** Unit snapshot taken from the metric at write time, not a live join. */
unit: string;
measuredAt: string;
/**
* The readable note: portable exports, and DR payloads for a row the
* free-text backfill had not reached yet. Null otherwise.
*/
note: string | null;
/**
* v1.39.3 — base64 ciphertext of the note, present only in disaster-recovery
* payloads (null when the reading has no note).
*/
noteEncrypted?: string | null;
createdAt?: string;
/** v1.37.20 (A3-11) — soft-delete tombstone; DR payloads only. */
deletedAt?: string | null;
@@ -227,6 +237,32 @@ function decryptProfileFieldSoft(
}
}
/**
* v1.39.3 — a custom-metric reading's note for a portable export: ciphertext
* first, the legacy readable column for a row the backfill has not reached.
* An unreadable ciphertext is recorded in the manifest and exported as no
* note, never replaced by the readable column.
*/
function readEntryNoteSoft(
entry: { id: string; note: string | null; noteEncrypted: Uint8Array | null },
failures: string[],
): string | null {
if (entry.noteEncrypted && entry.noteEncrypted.byteLength > 0) {
try {
return decryptFromBytes(entry.noteEncrypted);
} catch (err) {
getEvent()?.addWarning(
`custom metric entry note decrypt failed: ${
err instanceof Error ? err.message : String(err)
}`,
);
failures.push(`customMetricEntry.${entry.id}.note`);
return null;
}
}
return entry.note;
}
function toBase64(buf: Uint8Array | null): string | null {
return buf && buf.byteLength > 0 ? Buffer.from(buf).toString("base64") : null;
}
@@ -447,12 +483,21 @@ export async function buildProfileBackupSection(
id: entry.id,
createdAt: entry.createdAt.toISOString(),
deletedAt: entry.deletedAt?.toISOString() ?? null,
// v1.39.3 — the ciphertext rides verbatim. A row the free-text
// backfill has not reached yet carries its old readable note in
// `note` instead, as stored; the restore seals it. Sealing it
// here would make the export non-deterministic.
noteEncrypted: toBase64(entry.noteEncrypted),
}
: {}),
value: entry.value,
unit: entry.unit,
measuredAt: entry.measuredAt.toISOString(),
note: entry.note,
note: disasterRecovery
? entry.noteEncrypted
? null
: entry.note
: readEntryNoteSoft(entry, decryptFailures),
})),
}));
@@ -571,6 +616,16 @@ function resolveProfileColumn(
return plaintext == null ? null : encryptToBytes(plaintext);
}
function resolveEntryNote(entry: {
note?: string | null;
noteEncrypted?: string | null;
}): Uint8Array<ArrayBuffer> | null {
if (entry.noteEncrypted !== undefined && entry.noteEncrypted !== null) {
return decodeEncryptedField(entry.noteEncrypted, "customMetricEntry.note");
}
return encryptNote(entry.note);
}
function resolveFactValue(
entry: HealthProfileFactBackupEntry,
): Uint8Array<ArrayBuffer> {
@@ -848,7 +903,10 @@ export async function restoreProfileData(
value: entry.value,
unit: entry.unit,
measuredAt: new Date(entry.measuredAt),
note: entry.note,
// v1.39.3 — ciphertext wins when a DR payload carried it; a
// portable file's readable note is sealed under this instance's
// key. The readable column is never written.
noteEncrypted: resolveEntryNote(entry),
deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null,
...(entry.createdAt
? { createdAt: new Date(entry.createdAt) }
@@ -0,0 +1,184 @@
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { readNote } from "@/lib/crypto/note-cipher";
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
// ── in-memory store backing a minimal prisma mock ──────────────────────────
interface ConversationRow {
id: string;
userId: string;
title: string | null;
titleEncrypted: Uint8Array | null;
}
interface EntryRow {
id: string;
userId: string;
note: string | null;
noteEncrypted: Uint8Array | null;
}
const store = vi.hoisted(() => ({
conversations: [] as ConversationRow[],
entries: [] as EntryRow[],
}));
vi.mock("@/lib/jobs/boss-instance", () => ({ getGlobalBoss: () => null }));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/db", () => {
function delegate<R extends { id: string; userId: string }>(
rows: () => R[],
readable: keyof R,
) {
return {
// The handler pages on `{ userId, <readable>: { not: null } }`.
findMany: async (args: { where: { userId: string }; take: number }) =>
rows()
.filter((r) => r.userId === args.where.userId && r[readable] !== null)
.slice(0, args.take)
.map((r) => ({ id: r.id })),
findUnique: async (args: { where: { id: string } }) =>
rows().find((x) => x.id === args.where.id) ?? null,
update: async (args: { where: { id: string }; data: Partial<R> }) => {
const r = rows().find((x) => x.id === args.where.id)!;
Object.assign(r, args.data);
return r;
},
};
}
const delegates = {
coachConversation: delegate(() => store.conversations, "title"),
customMetricEntry: delegate(() => store.entries, "note"),
};
return {
prisma: {
...delegates,
$transaction: async (fn: (tx: typeof delegates) => unknown) =>
fn(delegates),
},
};
});
import { runFreeTextEncryptionBackfillForUser } from "@/lib/jobs/free-text-encryption-backfill";
const KEY = "a".repeat(64);
beforeEach(() => {
vi.stubEnv("ENCRYPTION_KEYS", "");
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
vi.stubEnv("ENCRYPTION_KEY", KEY);
store.conversations = [
{
id: "c1",
userId: "u1",
title: "Why is my pressure up after the new tablets?",
titleEncrypted: null,
},
// Already migrated: left alone.
{
id: "c2",
userId: "u1",
title: null,
titleEncrypted: encryptToBytes("Sleep this week"),
},
{ id: "c3", userId: "u2", title: "other user", titleEncrypted: null },
];
store.entries = [
{
id: "e1",
userId: "u1",
note: "after the long ride",
noteEncrypted: null,
},
{ id: "e2", userId: "u1", note: null, noteEncrypted: null }, // no note
// An empty legacy note is still a readable value; it migrates to "none".
{ id: "e3", userId: "u1", note: "", noteEncrypted: null },
{ id: "e4", userId: "u2", note: "other user", noteEncrypted: null },
];
});
afterEach(() => {
vi.unstubAllEnvs();
});
describe("runFreeTextEncryptionBackfillForUser", () => {
it("seals titles and notes, nulls the readable column, and reads back the same text", async () => {
const summary = await runFreeTextEncryptionBackfillForUser("u1");
expect(summary).toEqual({
conversationTitlesMigrated: 1,
metricNotesMigrated: 2,
});
const c1 = store.conversations.find((r) => r.id === "c1")!;
expect(c1.title).toBeNull();
expect(readNote(c1.titleEncrypted, null)).toBe(
"Why is my pressure up after the new tablets?",
);
const e1 = store.entries.find((r) => r.id === "e1")!;
expect(e1.note).toBeNull();
expect(readNote(e1.noteEncrypted, null)).toBe("after the long ride");
const e3 = store.entries.find((r) => r.id === "e3")!;
expect(e3.note).toBeNull();
expect(e3.noteEncrypted).toBeNull();
});
it("keeps an already sealed row and a row without a note as they were", async () => {
const before = store.conversations.find(
(r) => r.id === "c2",
)!.titleEncrypted;
await runFreeTextEncryptionBackfillForUser("u1");
const c2 = store.conversations.find((r) => r.id === "c2")!;
expect(c2.titleEncrypted).toBe(before);
const e2 = store.entries.find((r) => r.id === "e2")!;
expect(e2).toMatchObject({ note: null, noteEncrypted: null });
});
it("seals the readable value when a row holds both, since only an older writer can have put it there", async () => {
store.conversations[1].title = "Renamed by the previous release";
await runFreeTextEncryptionBackfillForUser("u1");
const c2 = store.conversations.find((r) => r.id === "c2")!;
expect(c2.title).toBeNull();
expect(readNote(c2.titleEncrypted, null)).toBe(
"Renamed by the previous release",
);
});
it("does not touch another account's rows", async () => {
await runFreeTextEncryptionBackfillForUser("u1");
expect(store.conversations.find((r) => r.id === "c3")).toMatchObject({
title: "other user",
titleEncrypted: null,
});
expect(store.entries.find((r) => r.id === "e4")).toMatchObject({
note: "other user",
noteEncrypted: null,
});
});
it("is idempotent: a second run changes nothing", async () => {
await runFreeTextEncryptionBackfillForUser("u1");
const sealed = store.conversations.find(
(r) => r.id === "c1",
)!.titleEncrypted;
const second = await runFreeTextEncryptionBackfillForUser("u1");
expect(second).toEqual({
conversationTitlesMigrated: 0,
metricNotesMigrated: 0,
});
expect(store.conversations.find((r) => r.id === "c1")!.titleEncrypted).toBe(
sealed,
);
});
it("is fail-closed: without a key every readable row stays intact", async () => {
vi.stubEnv("ENCRYPTION_KEY", "");
vi.stubEnv("ENCRYPTION_KEYS", "");
await expect(runFreeTextEncryptionBackfillForUser("u1")).rejects.toThrow();
expect(store.conversations.find((r) => r.id === "c1")).toMatchObject({
title: "Why is my pressure up after the new tablets?",
titleEncrypted: null,
});
});
});
@@ -107,6 +107,7 @@ describe("maintenance registrar queue policies", () => {
["INTAKE_SLOT_DEDUP_QUEUE", "exclusive"],
["NOTE_ENCRYPTION_BACKFILL_QUEUE", "exclusive"],
["MED_NOTES_ENCRYPTION_BACKFILL_QUEUE", "exclusive"],
["FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE", "exclusive"],
["DOCUMENT_THUMBNAIL_BACKFILL_QUEUE", "exclusive"],
["CONTENT_INDEX_BACKFILL_QUEUE", "exclusive"],
["ENCRYPTION_KEY_ROTATE_QUEUE", "exclusive"],
@@ -0,0 +1,245 @@
/**
* v1.39.3 — boot-time converging backfill that moves the last two readable
* free-text columns to AES-256-GCM at rest:
* `CoachConversation.title` -> `titleEncrypted` (the opening words of the
* first Coach message, or a rename) and
* `CustomMetricEntry.note` -> `noteEncrypted`.
*
* Modelled on `med-notes-encryption-backfill.ts`: a discovery query enqueues
* one job per user still holding an un-migrated row, the per-user handler walks
* that user's rows, and the pass is idempotent across reboots — once a row is
* migrated it drops off the discovery + candidate sets.
*
* DATA-LOSS SAFETY:
* - Per row, the encrypt-then-null happens in a SINGLE interactive
* transaction guarded by a re-read inside the tx, so the readable column is
* only nulled AFTER the ciphertext is written in the same atomic unit. A
* row that had text is never left with neither.
* - FAIL-CLOSED: the encryption helper throws on a missing / malformed key.
* The throw aborts the transaction (the row is untouched) and propagates so
* pg-boss retries.
* - IDEMPOTENT: the guard (readable column IS NOT NULL) is re-checked inside
* the tx, so a re-run, or two workers racing, migrates a row at most once;
* a second pass migrates zero rows.
* - The guard is the readable column alone, not "and no ciphertext yet".
* From this release on nothing writes the readable column, so a row holding
* both can only come from a writer of the previous release still running
* after the upgrade (a rename, an edited note). Its readable value is then
* the newer one, and it is the one sealed.
*
* The readable columns are NOT dropped in this release. That is a follow-up
* release, once this backfill reports zero remaining rows on every instance;
* both columns carry the marker in `schema.prisma`.
*
* Boot discovery is staggered past the startup storm (`startAfter`), like the
* other boot backfills; the daily discovery tick passes no offset.
*
* The queue name MUST be registered in the maintenance registrar
* (`src/lib/jobs/reminder/register-maintenance.ts`) so pg-boss provisions it at
* boot; an unregistered queue silently never drains.
*/
import { prisma } from "@/lib/db";
import { annotate } from "@/lib/logging/context";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { encryptNote } from "@/lib/crypto/note-cipher";
export const FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE =
"free-text-encryption-backfill";
/** Serial — one transaction per row, kept off the request pool. */
export const FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY = 1;
/** How many candidate rows to pull per page within a user's pass. */
const PAGE_SIZE = 200;
export interface FreeTextEncryptionBackfillPayload {
/** Absent on the daily discovery tick; present for a per-user job. */
userId?: string;
enqueuedAt?: string;
}
export interface FreeTextEncryptionBackfillSummary {
conversationTitlesMigrated: number;
metricNotesMigrated: number;
}
/**
* Migrate one conversation's readable title into `titleEncrypted` and null the
* readable column, atomically and idempotently. Returns true if it changed the
* row, false if a concurrent pass already did.
*/
async function migrateConversationTitle(id: string): Promise<boolean> {
return prisma.$transaction(async (tx) => {
const fresh = await tx.coachConversation.findUnique({
where: { id },
select: { title: true },
});
if (!fresh || fresh.title === null) return false;
// FAIL-CLOSED: a key error throws here and rolls the tx back.
await tx.coachConversation.update({
where: { id },
data: { titleEncrypted: encryptToBytes(fresh.title), title: null },
});
return true;
});
}
/**
* Migrate one custom-metric reading's readable note into `noteEncrypted` and
* null the readable column, atomically and idempotently.
*/
async function migrateMetricNote(id: string): Promise<boolean> {
return prisma.$transaction(async (tx) => {
const fresh = await tx.customMetricEntry.findUnique({
where: { id },
select: { note: true },
});
if (!fresh || fresh.note === null) return false;
await tx.customMetricEntry.update({
where: { id },
// An empty note reads back as "no note" either way, so it stores none.
data: { noteEncrypted: encryptNote(fresh.note), note: null },
});
return true;
});
}
/**
* Per-user queue handler. Walks every un-migrated title and note of one
* account and migrates each in its own transaction. Safe to re-run.
*/
export async function runFreeTextEncryptionBackfillForUser(
userId: string,
): Promise<FreeTextEncryptionBackfillSummary> {
let conversationTitlesMigrated = 0;
let metricNotesMigrated = 0;
// Every row this pass has sealed. A row that comes back on a later page was
// not cleared, so the pass would never converge; it stops loudly instead of
// re-sealing the same rows forever.
const sealed = new Set<string>();
const once = (id: string) => {
if (sealed.has(id)) {
throw new Error(
`free-text encryption backfill: row ${id} still readable after sealing`,
);
}
sealed.add(id);
};
// Each migration nulls the readable column, so a migrated row drops out of
// the next `IS NOT NULL` page.
for (;;) {
const rows = await prisma.coachConversation.findMany({
where: { userId, title: { not: null } },
select: { id: true },
take: PAGE_SIZE,
});
if (rows.length === 0) break;
let migratedThisPage = 0;
for (const { id } of rows) {
once(id);
if (await migrateConversationTitle(id)) {
conversationTitlesMigrated += 1;
migratedThisPage += 1;
}
}
// A page where nothing moved (every row taken by a racing worker) ends
// the loop, so it always terminates.
if (migratedThisPage === 0) break;
}
for (;;) {
const rows = await prisma.customMetricEntry.findMany({
where: { userId, note: { not: null } },
select: { id: true },
take: PAGE_SIZE,
});
if (rows.length === 0) break;
let migratedThisPage = 0;
for (const { id } of rows) {
once(id);
if (await migrateMetricNote(id)) {
metricNotesMigrated += 1;
migratedThisPage += 1;
}
}
if (migratedThisPage === 0) break;
}
annotate({
action: {
name: "free_text.encryption.backfill",
details: {
conversation_titles_migrated: conversationTitlesMigrated,
metric_notes_migrated: metricNotesMigrated,
},
},
});
return { conversationTitlesMigrated, metricNotesMigrated };
}
/**
* Discovery. Finds every user still holding a readable title or note and
* enqueues one backfill job per account. The 0357 partial indexes match both
* predicates, so the scan is index-only and converges to empty. pg-boss
* `singletonKey` coalesces duplicate sends. Best-effort: errors come back
* through the result value so worker boot never fails on a miss.
*/
export async function enqueueBootTimeFreeTextEncryptionBackfill(
// Boot-storm stagger in seconds; 0 (the cron tick) keeps immediate semantics.
startAfterSeconds: number = 0,
): Promise<{ enqueued: number; skipped: number; error: string | null }> {
const boss = getGlobalBoss();
if (!boss) {
return { enqueued: 0, skipped: 0, error: null };
}
try {
const [titleUsers, noteUsers] = await Promise.all([
prisma.$queryRaw<{ user_id: string }[]>`
SELECT DISTINCT user_id FROM coach_conversations
WHERE title IS NOT NULL`,
prisma.$queryRaw<{ user_id: string }[]>`
SELECT DISTINCT user_id FROM custom_metric_entries
WHERE note IS NOT NULL`,
]);
const userIds = new Set<string>();
for (const { user_id } of titleUsers) userIds.add(user_id);
for (const { user_id } of noteUsers) userIds.add(user_id);
let enqueued = 0;
let skipped = 0;
for (const userId of userIds) {
const payload: FreeTextEncryptionBackfillPayload = {
userId,
enqueuedAt: new Date().toISOString(),
};
const jobId = await boss.send(
FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE,
payload,
{
retryLimit: 5,
retryDelay: 60,
retryBackoff: true,
singletonKey: `free-text-encryption-backfill|${userId}`,
...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}),
},
);
if (jobId) {
enqueued += 1;
} else {
skipped += 1;
}
}
return { enqueued, skipped, error: null };
} catch (err) {
return {
enqueued: 0,
skipped: 0,
error: err instanceof Error ? err.message : String(err),
};
}
}
+2
View File
@@ -57,6 +57,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"context_surfaced",
"continuation",
"continued",
"conversation_titles_migrated",
"created",
"daily_rows_retired",
"days_consolidated",
@@ -130,6 +131,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"medications",
"medications_evaluated",
"metric_assessments_warmed",
"metric_notes_migrated",
"missed_doses_minted",
"module_off",
"mood_entries_migrated",
@@ -97,6 +97,13 @@ import {
enqueueBootTimeMedNotesEncryptionBackfill,
type MedNotesEncryptionBackfillPayload,
} from "@/lib/jobs/med-notes-encryption-backfill";
import {
FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE,
FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY,
runFreeTextEncryptionBackfillForUser,
enqueueBootTimeFreeTextEncryptionBackfill,
type FreeTextEncryptionBackfillPayload,
} from "@/lib/jobs/free-text-encryption-backfill";
import {
CONTENT_INDEX_BACKFILL_QUEUE,
CONTENT_INDEX_BACKFILL_CONCURRENCY,
@@ -343,6 +350,13 @@ const MCP_TOKEN_CLEANUP_CRON = "0 4 * * *";
// boss poll.
const MED_NOTES_ENCRYPTION_BACKFILL_CRON = "5 4 * * *";
// v1.39.3 — converging backfill that moves the Coach conversation title and the
// custom-metric reading note to AES-256-GCM at rest. Boot discovery (staggered
// past the startup storm) enqueues one per-user job; a daily 04:07
// Europe/Berlin discovery tick re-fans for rows written by a previous-release
// process after the upgrade. Two minutes after the medication-note tick so the
// two do not share a poll.
const FREE_TEXT_ENCRYPTION_BACKFILL_CRON = "7 4 * * *";
const allQueues = [
// v1.25 (W-ENV) — nightly environmental-context fetch. A daily discovery tick
@@ -445,6 +459,10 @@ const allQueues = [
// plaintext medication note; without this entry pg-boss never provisions the
// queue and both the boot enqueue and the cron silently no-op.
MED_NOTES_ENCRYPTION_BACKFILL_QUEUE,
// v1.39.3 — Coach title + custom-metric note encryption backfill. Without
// this entry pg-boss never provisions the queue and both the boot enqueue
// and the cron silently no-op.
FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE,
// Document vault — daily physical purge for tombstones past the 30-day
// undo grace (returns the encrypted blob's TOAST space). Without this entry
// the daily schedule silently no-ops and "deleted" documents hold backup
@@ -605,6 +623,9 @@ const schedules: ScheduleEntry[] = [
// fan out one per-user job per account still holding a plaintext medication
// note.
[MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, MED_NOTES_ENCRYPTION_BACKFILL_CRON],
// v1.39.3 — daily 04:07 Europe/Berlin discovery for the Coach title +
// custom-metric note backfill. Empty payload = fan out per user.
[FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, FREE_TEXT_ENCRYPTION_BACKFILL_CRON],
// Document vault — daily 04:10 Europe/Berlin purge for tombstoned
// documents past the 30-day undo grace.
[DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON, cronIsTheRetry],
@@ -663,6 +684,11 @@ const queuePolicies: QueuePolicyTable = {
reason:
"Per-user medication-note encryption backfill; discovery drops the user once no plaintext note remains.",
},
[FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE]: {
policy: "exclusive",
reason:
"Per-user Coach title + custom-metric note encryption backfill; discovery drops the user once no readable value remains.",
},
[DOCUMENT_THUMBNAIL_BACKFILL_QUEUE]: {
policy: "exclusive",
reason:
@@ -1206,6 +1232,65 @@ export async function registerMaintenanceQueues(
},
);
// v1.39.3 — Coach title + custom-metric note encryption backfill worker.
// Same shape as the medication-note worker above: an empty-userId payload is
// the daily discovery tick, a userId payload migrates that account's rows.
await createAndWork<Partial<FreeTextEncryptionBackfillPayload>>(
boss,
FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE,
{ localConcurrency: FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY },
async (jobs) => {
let users = 0;
let discoveryEnqueued = 0;
let discoverySkipped = 0;
let discoveryFailed = 0;
let titles = 0;
let notes = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) {
const result = await enqueueBootTimeFreeTextEncryptionBackfill();
discoveryEnqueued += result.enqueued;
discoverySkipped += result.skipped;
if (result.error) discoveryFailed++;
workerLog(
"info",
`[free-text-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`,
);
continue;
}
try {
const { conversationTitlesMigrated, metricNotesMigrated } =
await runFreeTextEncryptionBackfillForUser(userId);
users++;
titles += conversationTitlesMigrated;
notes += metricNotesMigrated;
workerLog(
"info",
`[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated}`,
);
} catch (err) {
recordError();
workerLog(
"error",
`[free-text-encryption-backfill] user=${userId} failed`,
err,
);
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
discovery_enqueued: discoveryEnqueued,
discovery_skipped: discoverySkipped,
discovery_failed: discoveryFailed,
conversation_titles_migrated: titles,
metric_notes_migrated: notes,
});
},
);
// Document vault P2 — on-demand content-search index backfill worker. The
// trigger endpoint sends one per-user job; this handler indexes that user's
// not-yet-indexed documents (consent + budget gated, bounded + resumable).
@@ -1568,6 +1653,33 @@ export async function enqueueMaintenanceBootDiscovery(): Promise<void> {
);
}
// v1.39.3 — Coach title + custom-metric note encryption backfill. Staggered
// one step past the note-encryption backfill so it starts after the boot
// storm rather than on the first poll.
try {
const { enqueued, skipped, error } =
await enqueueBootTimeFreeTextEncryptionBackfill(
BOOT_BACKFILL_STAGGER_SECONDS * 6,
);
if (error) {
workerLog(
"error",
`[free-text-encryption-backfill] boot discovery failed: ${error}`,
);
} else {
workerLog(
"info",
`[free-text-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`,
);
}
} catch (err) {
workerLog(
"error",
"[free-text-encryption-backfill] boot discovery threw an unexpected error",
err,
);
}
// Document vault — preview-thumbnail backfill. Finds every account holding a
// thumbnailable document (image or PDF) without a preview and enqueues one
// per-user backfill pass. Idempotent across reboots: a rendered thumbnail
+1
View File
@@ -128,6 +128,7 @@ const subsystemSurface = {
"encryption-key-rotate": { audience: "system" },
"mcp-token-cleanup": { audience: "system" },
"med-notes-encryption-backfill": { audience: "system" },
"free-text-encryption-backfill": { audience: "system" },
"document-tombstone-purge": { audience: "system" },
// v1.37.19 (A7-4) — hourly heal of summary rows stuck on PENDING.
"document-summary-reaper": { audience: "system" },
+1 -1
View File
@@ -2011,7 +2011,7 @@ export const authPaths: NonNullable<ZodOpenApiObject["paths"]> = {
responses: {
"302": {
description:
"Always a redirect. To the provider on success. On failure to `/auth/login?error=<reason>` (browser) or `healthlog://oidc-callback?error=<reason>` (native), where reason is one of `oidc_disabled`, `oidc_rate_limited`, `oidc_invalid_request`, `oidc_failed`.",
"Always a redirect. To the provider on success. On failure to `/auth/login?error=<reason>` (browser) or `healthlog://oidc-callback?error=<reason>` (native), where reason is one of `oidc_disabled`, `oidc_rate_limited`, `oidc_invalid_request`, `oidc_failed`. The provider then returns to the callback (not listed here, because the identity provider drives it rather than a client), which ends the native leg at `healthlog://oidc-callback` with `code=<hlh_…>`, with `mfa_ticket=<ticket>&methods=<list>`, or with `error=<reason>`, where reason is one of `oidc_disabled`, `oidc_rate_limited`, `oidc_denied`, `oidc_no_email`, `oidc_email_unverified`, `oidc_identity_conflict`, `oidc_registration_disabled`, `oidc_failed`. The reasons are this exact snake_case vocabulary on both legs; a client maps a value it does not know to a generic sign-in failure.",
},
},
},
+1 -1
View File
@@ -183,7 +183,7 @@ export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
tags: ["Import"],
summary: "Import measurements from CSV",
description:
"Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤ 10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`. `measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to `kg`). `glucoseContext` is optional on a `BLOOD_GLUCOSE` row — a blank cell stores no context, which is the normal shape of a continuous-sensor export; a non-empty value is still checked against the enum, and the column is refused on any other type. An `externalId` column makes re-upload idempotent (upsert on `(userId, type, source=IMPORT, externalId)`); without it a re-upload duplicates. `?dryRun=1` validates + previews without writing. Shares the 5/hour `import:` rate bucket.",
"Bulk-import measurements from a CSV file (web cold-start escape hatch). Body is raw `text/csv` (≤ 16 MB, ≤ 10 000 valid rows). Header (order-independent): `type,value,unit,measuredAt[,glucoseContext,notes,externalId]`. `measuredAt` must carry an explicit ISO-8601 offset and is bounded (no future beyond a 5-min skew, no instant before 1900). Glucose accepts `mmol/L` (converted to canonical `mg/dL`); weight accepts `lb` (converted to `kg`). `glucoseContext` is optional on a `BLOOD_GLUCOSE` row — a blank cell stores no context, which is the normal shape of a continuous-sensor export; a non-empty value is still checked against the enum, and the column is refused on any other type. Re-uploading a file is idempotent. With an `externalId` column a row upserts on `(userId, type, source=IMPORT, externalId)`: a changed row is `updated`, an unchanged one is `skipped` with reason `duplicate` and not written. Without it a row is deduplicated on `(type, measuredAt)` and a repeat is `skipped` / `duplicate`. Writes commit in chunks rather than as one transaction; a failure part-way leaves earlier chunks written, and re-uploading the same file completes the rest. `?dryRun=1` validates + previews without writing. Shares the 5/hour `import:` rate bucket.",
parameters: [
{
name: "dryRun",
+6 -1
View File
@@ -799,6 +799,8 @@ const customMetricEntryBackupSchema = z
unit: z.string().min(1),
measuredAt: isoDateTime,
note: z.string().nullable().default(null),
/// v1.39.3 — the note's ciphertext, DR payloads only.
noteEncrypted: base64BytesSchema.nullable().optional(),
createdAt: isoDateTime.optional(),
/// v1.37.20 (A3-11) — the entry tombstone rides DR payloads so a restore
/// brings the account back exactly as it stood, undo affordance included.
@@ -1366,7 +1368,10 @@ const coachConversationDocumentBackupSchema = z
const coachConversationBackupSchema = z
.object({
id: z.string().min(1),
title: z.string(),
// v1.39.3 — a portable file (and every file written before) carries the
// readable title; a disaster-recovery file carries only the ciphertext.
title: z.string().optional(),
titleEncrypted: base64BytesSchema.optional(),
// NOT optional with a default. The fence is permanent and a file that does
// not state it is a file that cannot be trusted to re-fence the
// conversation, so the absence has to be visible rather than defaulted to