feat(ai): UI-configurable response timeout, local streaming, dedicated OCR provider

Add three per-user AI settings, all configured in the app (no new env vars),
behind one migration (0198):

- Response timeout (seconds). A per-user upstream timeout threaded onto the
  provider call. A local/self-hosted backend can take more than the legacy
  60 s default to load a model on the first request, which timed the Coach
  out before the first token. Defaults to a generous ~180 s when unset;
  validated to 10–600 s.

- True token streaming for the local OpenAI-compatible client. The client now
  POSTs with stream:true and parses the SSE delta chunks, emitting tokens as
  they arrive, with a per-idle-gap timeout instead of a whole-call ceiling and
  a graceful fallback to the buffered path when a server cannot stream. The
  Coach route runs the provider call inside the response stream behind a
  keepalive heartbeat so a reverse proxy no longer idle-drops the connection
  during model load or generation. Grounding, refusal, and sentinel-stripping
  still run on the complete reply before any token reaches the client.

- Dedicated document-scan (Lab-OCR) provider. Off by default — the OCR path
  uses the main provider chain. When enabled, lab-report processing resolves a
  separate provider/model/key, with a hint when the chosen model is not
  vision-capable. The new key column is AES-256-GCM at rest and registered in
  the encrypted-columns registry and the rotation script.

i18n: EN + DE translations for the new Settings → AI strings (other locales
carry interim copies to keep locale parity green).
This commit is contained in:
Marc-André Bombeck committed 2026-06-27 15:21:40 +02:00
1 parent 3b74b108a1
commit b91649d6de
30 files changed
+2281 -569

No files matched your search

@@ -50,12 +50,12 @@ vi.mock("@/lib/i18n/server-locale", () => ({
resolveServerLocale: vi.fn(async () => "en"),
}));
const { runRawCompletionWithFallback } = vi.hoisted(() => ({
runRawCompletionWithFallback: vi.fn(),
const { runStreamingRawCompletionWithFallback } = vi.hoisted(() => ({
runStreamingRawCompletionWithFallback: vi.fn(),
}));
vi.mock("@/lib/ai/provider-runner", () => ({
AllProvidersFailedError: class extends Error {},
runRawCompletionWithFallback,
runStreamingRawCompletionWithFallback,
}));
vi.mock("@/lib/ai/provider", () => ({
// v1.20.0 (F1) — pin a no-tools provider so this suite exercises the legacy
@@ -138,17 +138,18 @@ vi.mock("@/lib/validations/coach-prefs", () => ({
parseCoachPrefs: vi.fn(() => ({ defaultWindow: undefined })),
DEFAULT_REMINDER_SUGGESTION_PREFS: {},
}));
// v1.22 (#89) — the provider call now runs INSIDE the stream producer. Capture
// the producer promise so the test can await it before asserting on call args.
const sse = vi.hoisted(() => ({ done: Promise.resolve() as Promise<unknown> }));
vi.mock("@/lib/sse/create-stream", () => ({
// Drain the producer synchronously so the handler completes; we only care
// about the provider call args, not the wire frames.
createSseStream: (
producer: (c: {
signal: { aborted: boolean };
enqueue: () => void;
}) => void | Promise<void>,
) => {
void Promise.resolve(
producer({ signal: { aborted: true }, enqueue: () => {} }),
sse.done = Promise.resolve(
producer({ signal: { aborted: false }, enqueue: () => {} }),
);
return new ReadableStream();
},
@@ -158,6 +159,12 @@ import { POST } from "../route";
const post = POST as unknown as (req: Request) => Promise<Response>;
async function postAndDrain(body: Record<string, unknown>): Promise<Response> {
const res = await post(chatReq(body));
await sse.done;
return res;
}
function chatReq(body: Record<string, unknown>): Request {
return new Request("http://localhost/api/insights/chat", {
method: "POST",
@@ -167,7 +174,8 @@ function chatReq(body: Record<string, unknown>): Request {
}
function lastUserPrompt(): string {
const calls = runRawCompletionWithFallback.mock.calls as unknown as Array<
const calls = runStreamingRawCompletionWithFallback.mock
.calls as unknown as Array<
[{ params: { messages: Array<{ role: string; content: string }> } }]
>;
const last = calls[calls.length - 1];
@@ -178,14 +186,14 @@ function lastUserPrompt(): string {
describe("coach chat — snapshot sent once per conversation (C4)", () => {
beforeEach(() => {
vi.clearAllMocks();
runRawCompletionWithFallback.mockResolvedValue({
runStreamingRawCompletionWithFallback.mockResolvedValue({
result: { content: "Your BP looks stable.", tokensUsed: 42, model: "m" },
workingProvider: { providerType: "admin-openai" },
});
});
it("ships the full SNAPSHOT + grounding on the first turn", async () => {
await post(chatReq({ message: "How is my BP?" }));
await postAndDrain({ message: "How is my BP?" });
const prompt = lastUserPrompt();
expect(prompt).toContain(SNAPSHOT_JSON);
expect(prompt).toContain(GROUNDING);
@@ -201,7 +209,7 @@ describe("coach chat — snapshot sent once per conversation (C4)", () => {
],
});
await post(chatReq({ conversationId: "c1", message: "High?" }));
await postAndDrain({ conversationId: "c1", message: "High?" });
const prompt = lastUserPrompt();
// The expensive figures + grounding must be gone …
expect(prompt).not.toContain(SNAPSHOT_JSON);
@@ -52,12 +52,12 @@ vi.mock("@/lib/i18n/server-locale", () => ({
resolveServerLocale: vi.fn(async () => "en"),
}));
const { runRawCompletionWithFallback } = vi.hoisted(() => ({
runRawCompletionWithFallback: vi.fn(),
const { runStreamingRawCompletionWithFallback } = vi.hoisted(() => ({
runStreamingRawCompletionWithFallback: vi.fn(),
}));
vi.mock("@/lib/ai/provider-runner", () => ({
AllProvidersFailedError: class extends Error {},
runRawCompletionWithFallback,
runStreamingRawCompletionWithFallback,
}));
const { resolveProviderChain } = vi.hoisted(() => ({
@@ -194,6 +194,10 @@ const { parseKeyValuesSentinel } = await import("@/lib/ai/coach/keyvalues");
const { parseSuggestReminder } =
await import("@/lib/ai/coach/suggest-reminder");
// v1.22 (#89) — the provider call + guards + persistence now run INSIDE the
// stream producer (heartbeat-fronted). The mock captures the producer's promise
// so tests can await it after `post()` before asserting on persisted effects.
const sse = vi.hoisted(() => ({ done: Promise.resolve() as Promise<unknown> }));
vi.mock("@/lib/sse/create-stream", () => ({
createSseStream: (
producer: (c: {
@@ -201,8 +205,9 @@ vi.mock("@/lib/sse/create-stream", () => ({
enqueue: () => void;
}) => void | Promise<void>,
) => {
void Promise.resolve(
producer({ signal: { aborted: true }, enqueue: () => {} }),
// aborted:false so the producer streams to completion (persist + frames).
sse.done = Promise.resolve(
producer({ signal: { aborted: false }, enqueue: () => {} }),
);
return new ReadableStream();
},
@@ -212,6 +217,13 @@ import { POST } from "../route";
const post = POST as unknown as (req: Request) => Promise<Response>;
/** Run a POST and wait for the in-stream producer to finish. */
async function postAndDrain(body: Record<string, unknown>): Promise<Response> {
const res = await post(chatReq(body));
await sse.done;
return res;
}
function chatReq(body: Record<string, unknown>): Request {
return new Request("http://localhost/api/insights/chat", {
method: "POST",
@@ -233,10 +245,10 @@ describe("coach chat — tool-mode routing (F1)", () => {
resolveProviderChain.mockResolvedValue([
{ providerType: "anthropic", instance: {} }, // no supportsTools=false ⇒ tools on
]);
await post(chatReq({ message: "How is my BP?" }));
await postAndDrain({ message: "How is my BP?" });
expect(runCoachToolLoop).toHaveBeenCalledTimes(1);
expect(runRawCompletionWithFallback).not.toHaveBeenCalled();
expect(runStreamingRawCompletionWithFallback).not.toHaveBeenCalled();
// The base context carries the inventory, NOT the snapshot figures.
const loopArgs = (runCoachToolLoop.mock.calls[0] as unknown[])[0] as {
messages: Array<{ content: string }>;
@@ -277,6 +289,7 @@ describe("coach chat — tool-mode routing (F1)", () => {
);
const res = await post(chatReq({ message: "How is my BP?" }));
await sse.done;
// No throw → no 500. The provider-error frame streams over a 200.
expect(res.status).toBe(200);
@@ -293,7 +306,7 @@ describe("coach chat — tool-mode routing (F1)", () => {
resolveProviderChain.mockResolvedValue([
{ providerType: "anthropic", instance: {} },
]);
await post(chatReq({ message: "How is my BP?" }));
await postAndDrain({ message: "How is my BP?" });
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
const assistantCall = calls.find(
(c) => (c[0] as { role: string }).role === "assistant",
@@ -338,7 +351,7 @@ describe("coach chat — tool-mode routing (F1)", () => {
],
}),
);
await post(chatReq({ message: "How is my BP?" }));
await postAndDrain({ message: "How is my BP?" });
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
const assistantCall = calls.find(
(c) => (c[0] as { role: string }).role === "assistant",
@@ -353,14 +366,14 @@ describe("coach chat — tool-mode routing (F1)", () => {
{ providerType: "anthropic", instance: {} },
{ providerType: "local", instance: { supportsTools: false } },
]);
runRawCompletionWithFallback.mockResolvedValue({
runStreamingRawCompletionWithFallback.mockResolvedValue({
result: { content: "Your BP is steady.", tokensUsed: 42, model: "m" },
workingProvider: { providerType: "anthropic" },
});
await post(chatReq({ message: "How is my BP?" }));
await postAndDrain({ message: "How is my BP?" });
expect(runCoachToolLoop).not.toHaveBeenCalled();
expect(runRawCompletionWithFallback).toHaveBeenCalledTimes(1);
expect(runStreamingRawCompletionWithFallback).toHaveBeenCalledTimes(1);
// Single-round budget reservation in the fallback path; the 4th arg is the
// provider-aware daily cap (F1).
expect(reserveBudget).toHaveBeenCalledWith(
@@ -371,7 +384,7 @@ describe("coach chat — tool-mode routing (F1)", () => {
);
// The legacy path ships the snapshot figures in the user turn.
const params = (
(runRawCompletionWithFallback.mock.calls[0] as unknown[])[0] as {
(runStreamingRawCompletionWithFallback.mock.calls[0] as unknown[])[0] as {
params: { messages: Array<{ content: string }> };
}
).params;
File diff suppressed because it is too large. Load diff
@@ -18,7 +18,7 @@ vi.mock("@/lib/api-handler", () => ({
vi.mock("@/lib/db", () => ({
prisma: {
user: { findUnique: vi.fn() },
user: { findUnique: vi.fn(), update: vi.fn() },
},
}));
@@ -35,14 +35,26 @@ vi.mock("@/lib/validations/notifications", () => ({
isPublicUrl: vi.fn(() => true),
}));
vi.mock("@/lib/ai/local-host-allowlist", () => ({
isLocalAiHostAllowed: vi.fn(() => false),
}));
vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
}));
import { GET } from "../route";
import { GET, PATCH } from "../route";
import { prisma } from "@/lib/db";
import { resolveProviderAvailability } from "@/lib/ai/provider";
function patchRequest(body: unknown): Request {
return new Request("http://localhost/api/user/ai-provider", {
method: "PATCH",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
});
}
interface AiProviderResponse {
data: {
provider: string | null;
@@ -126,4 +138,101 @@ describe("GET /api/user/ai-provider availability", () => {
expect(body.data.aiAvailable).toBe(false);
expect(body.data.managedBy).toBeNull();
});
it("returns the v1.22 timeout + OCR fields", async () => {
vi.mocked(prisma.user.findUnique).mockResolvedValue({
aiProvider: "OPENAI",
aiModel: "gpt-4o",
aiBaseUrl: null,
aiAnthropicKeyEncrypted: null,
aiLocalKeyEncrypted: null,
aiOpenaiKeyEncrypted: null,
aiResponseTimeoutSeconds: 240,
aiOcrEnabled: true,
aiOcrProvider: "ANTHROPIC",
aiOcrModel: "claude-sonnet-4-6",
aiOcrBaseUrl: null,
aiOcrKeyEncrypted: "enc-ocr-1234",
} as never);
vi.mocked(resolveProviderAvailability).mockResolvedValue({
aiAvailable: true,
managedBy: "user",
});
const res = await (GET as () => Promise<Response>)();
const body = (await res.json()) as {
data: {
responseTimeoutSeconds: number | null;
ocrEnabled: boolean;
ocrProvider: string | null;
hasOcrKey: boolean;
ocrKeyPreview: string | null;
};
};
expect(body.data.responseTimeoutSeconds).toBe(240);
expect(body.data.ocrEnabled).toBe(true);
expect(body.data.ocrProvider).toBe("ANTHROPIC");
expect(body.data.hasOcrKey).toBe(true);
// Preview is `...` + last 4 of the decrypted value (decrypt → `dec:...`).
expect(body.data.ocrKeyPreview).toMatch(/^\.\.\./);
});
});
describe("PATCH /api/user/ai-provider — v1.22 fields", () => {
const patch = PATCH as (req: Request) => Promise<Response>;
it("persists a valid response timeout", async () => {
const res = await patch(patchRequest({ responseTimeoutSeconds: 120 }));
expect(res.status).toBe(200);
expect(vi.mocked(prisma.user.update)).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ aiResponseTimeoutSeconds: 120 }),
}),
);
});
it("rejects an out-of-range response timeout", async () => {
await expect(
patch(patchRequest({ responseTimeoutSeconds: 5 })),
).rejects.toThrow(/between 10 and 600/);
expect(vi.mocked(prisma.user.update)).not.toHaveBeenCalled();
});
it("clears the timeout when null is sent", async () => {
await patch(patchRequest({ responseTimeoutSeconds: null }));
expect(vi.mocked(prisma.user.update)).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({ aiResponseTimeoutSeconds: null }),
}),
);
});
it("persists OCR provider config and encrypts the key", async () => {
const res = await patch(
patchRequest({
ocrEnabled: true,
ocrProvider: "OPENAI",
ocrModel: "gpt-4o",
ocrKey: "sk-ocr-secret",
}),
);
expect(res.status).toBe(200);
expect(vi.mocked(prisma.user.update)).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({
aiOcrEnabled: true,
aiOcrProvider: "OPENAI",
aiOcrModel: "gpt-4o",
aiOcrKeyEncrypted: "enc:sk-ocr-secret",
}),
}),
);
});
it("rejects an invalid OCR provider", async () => {
await expect(
patch(patchRequest({ ocrProvider: "CHATGPT_OAUTH" })),
).rejects.toThrow(/Invalid OCR provider/);
});
});
+103
View File
@@ -11,6 +11,9 @@ import { annotate } from "@/lib/logging/context";
export const dynamic = "force-dynamic";
const ALLOWED = new Set(["OPENAI", "ANTHROPIC", "LOCAL", "CHATGPT_OAUTH"]);
// v1.22 (#90) — the dedicated document-scan provider has no OAuth path, so the
// allowlist is the three keyed/self-hosted providers only (no CHATGPT_OAUTH).
const OCR_PROVIDERS = new Set(["OPENAI", "ANTHROPIC", "LOCAL"]);
export const GET = apiHandler(async () => {
const { user } = await requireAuth();
@@ -25,6 +28,13 @@ export const GET = apiHandler(async () => {
aiAnthropicKeyEncrypted: true,
aiLocalKeyEncrypted: true,
aiOpenaiKeyEncrypted: true,
// v1.22 (#89 + #90)
aiResponseTimeoutSeconds: true,
aiOcrEnabled: true,
aiOcrProvider: true,
aiOcrModel: true,
aiOcrBaseUrl: true,
aiOcrKeyEncrypted: true,
},
});
@@ -50,6 +60,18 @@ export const GET = apiHandler(async () => {
openaiKeyPreview: u?.aiOpenaiKeyEncrypted
? `...${decrypt(u.aiOpenaiKeyEncrypted).slice(-4)}`
: null,
// v1.22 (#89) — per-user response timeout, in seconds (null = default).
responseTimeoutSeconds: u?.aiResponseTimeoutSeconds ?? null,
// v1.22 (#90) — dedicated document-scan (Lab-OCR) provider config. Key
// material is presence + 4-char preview only, like every other provider.
ocrEnabled: Boolean(u?.aiOcrEnabled),
ocrProvider: u?.aiOcrProvider ?? null,
ocrModel: u?.aiOcrModel ?? null,
ocrBaseUrl: u?.aiOcrBaseUrl ?? null,
hasOcrKey: Boolean(u?.aiOcrKeyEncrypted),
ocrKeyPreview: u?.aiOcrKeyEncrypted
? `...${decrypt(u.aiOcrKeyEncrypted).slice(-4)}`
: null,
});
});
@@ -132,6 +154,87 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
}
}
// ── v1.22 (#89) — response timeout (seconds) ──────────────────
if (body.responseTimeoutSeconds !== undefined) {
if (body.responseTimeoutSeconds === null) {
updates.aiResponseTimeoutSeconds = null;
} else if (
typeof body.responseTimeoutSeconds === "number" &&
Number.isInteger(body.responseTimeoutSeconds)
) {
// Sane bounds: 10 s floor (a stricter value just trips the timeout) and
// 600 s ceiling (a generous cap for a slow self-hosted backend).
if (
body.responseTimeoutSeconds < 10 ||
body.responseTimeoutSeconds > 600
) {
throw new HttpError(
422,
"Response timeout must be between 10 and 600 seconds",
);
}
updates.aiResponseTimeoutSeconds = body.responseTimeoutSeconds;
} else {
throw new HttpError(422, "Invalid response timeout");
}
}
// ── v1.22 (#90) — dedicated document-scan (Lab-OCR) provider ──
if (body.ocrEnabled !== undefined) {
if (typeof body.ocrEnabled === "boolean") {
updates.aiOcrEnabled = body.ocrEnabled;
} else {
throw new HttpError(422, "Invalid ocrEnabled");
}
}
if (body.ocrProvider !== undefined) {
if (body.ocrProvider === null || body.ocrProvider === "") {
updates.aiOcrProvider = null;
} else if (
typeof body.ocrProvider === "string" &&
OCR_PROVIDERS.has(body.ocrProvider)
) {
updates.aiOcrProvider = body.ocrProvider;
} else {
throw new HttpError(422, "Invalid OCR provider");
}
}
if (body.ocrModel !== undefined) {
if (body.ocrModel === null || body.ocrModel === "") {
updates.aiOcrModel = null;
} else if (typeof body.ocrModel === "string") {
updates.aiOcrModel = body.ocrModel.trim() || null;
}
}
if (body.ocrBaseUrl !== undefined) {
if (body.ocrBaseUrl === null || body.ocrBaseUrl === "") {
updates.aiOcrBaseUrl = null;
} else if (typeof body.ocrBaseUrl === "string") {
const trimmed = body.ocrBaseUrl.trim();
// Same SSRF guard as the main LOCAL base URL: reject private/internal
// hosts unless the operator opted in via ALLOW_LOCAL_AI_PRIVATE_HOSTS.
const allowPrivate = isLocalAiHostAllowed(trimmed);
if (!allowPrivate && !isPublicUrl(trimmed)) {
return apiError(
"OCR base URL points to an internal/private host. Ops must allow it on this instance via ALLOW_LOCAL_AI_PRIVATE_HOSTS.",
422,
);
}
updates.aiOcrBaseUrl = trimmed;
}
}
if (body.ocrKey !== undefined) {
if (body.ocrKey === null || body.ocrKey === "") {
updates.aiOcrKeyEncrypted = null;
} else if (typeof body.ocrKey === "string") {
updates.aiOcrKeyEncrypted = encrypt(body.ocrKey.trim());
}
}
if (Object.keys(updates).length === 0) {
throw new HttpError(422, "No valid fields");
}