From 887928211bd5fef757b342fd9af6432386c08e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 10 May 2026 01:44:43 +0200 Subject: [PATCH] feat(admin): AI quality preview showing per-(severity x provider) helpful-rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1.4.16 phase B5e: ship the `/admin/ai-quality` route that surfaces the daily aggregator's helpful-rate per (severity x metricSourceType x provider x prompt-version) bucket. Renders as a compact 7-column table with severity tinted by clinical-importance colour and the helpful-rate column tinted green/yellow/orange by 0.8/0.5 thresholds. New endpoint `GET /api/admin/ai-quality` (requireAdmin-gated) returns the raw `AppSettings.adminAiInsightsFeedbackSummary` blob so the v1.4.17 ratchet that consumes the same shape for prompt-tuning can read what the worker writes without an envelope-reshape step. Empty state ("no summary yet" / "buckets[] is empty") rendered as a quiet message rather than a fake zero row so the operator can tell "no feedback" apart from "all-zeros". Loading + error banners follow the existing host-metrics pattern. Section wiring: - new `ai-quality` slug in ADMIN_SECTION_SLUGS (between integrations and feedback so the IA flows General → Integrations → AI Quality → Feedback Inbox) - new Sparkles icon entry in admin-shell - new switch case in [section]/renderer.tsx - EN+DE i18n keys: admin.section.ai-quality.{title,subtitle} + admin.aiQuality.{title,loading,loadError,empty,windowLabel, generatedAtLabel,col*} Tests: - 5 new ai-quality-section tests (loading, empty-null, empty-buckets, table-render with multiple buckets, error) - 1 admin-sections smoke test ensures the empty branch renders + 1 slug-list parity test Co-Authored-By: Claude Opus 4.7 (1M context) --- messages/de.json | 19 ++ messages/en.json | 19 ++ src/app/admin/[section]/renderer.tsx | 10 + src/app/api/admin/ai-quality/route.ts | 42 ++++ .../__tests__/ai-quality-section.test.tsx | 123 +++++++++++ .../admin/__tests__/sections.test.tsx | 15 ++ src/components/admin/admin-shell.tsx | 6 + src/components/admin/ai-quality-section.tsx | 202 ++++++++++++++++++ src/components/admin/section-slugs.ts | 1 + 9 files changed, 437 insertions(+) create mode 100644 src/app/api/admin/ai-quality/route.ts create mode 100644 src/components/admin/__tests__/ai-quality-section.test.tsx create mode 100644 src/components/admin/ai-quality-section.tsx diff --git a/messages/de.json b/messages/de.json index 23b2a5c8f..1f007d772 100644 --- a/messages/de.json +++ b/messages/de.json @@ -1526,6 +1526,21 @@ "empty": "Erfassung läuft — Daten erscheinen in 1–2 Minuten.", "loadError": "Host-Metriken konnten nicht geladen werden. Bitte die Seite neu laden." }, + "aiQuality": { + "title": "KI-Qualität", + "loading": "Feedback-Zusammenfassung wird geladen...", + "loadError": "KI-Qualitätszusammenfassung konnte nicht geladen werden. Bitte die Seite neu laden.", + "empty": "Noch kein Feedback. Der Aggregator läuft täglich um 04:00 — sobald Nutzer Empfehlungen bewerten, erscheinen hier die Hilfreich-Quoten.", + "windowLabel": "Letzte {days} Tage", + "generatedAtLabel": "Erstellt", + "colSeverity": "Schweregrad", + "colMetric": "Metrik", + "colProvider": "Provider", + "colPromptVersion": "Prompt", + "colHelpful": "Hilfreich", + "colNotHelpful": "Nicht hilfreich", + "colHelpfulRate": "Hilfreich-Quote" + }, "workerStatus": "Job-Worker", "workerRunning": "Aktiv", "workerStopped": "Gestoppt", @@ -1762,6 +1777,10 @@ "title": "Integrationen", "subtitle": "Umami, GlitchTip, Web Push und Bug-Reports konfigurieren." }, + "ai-quality": { + "title": "KI-Qualität", + "subtitle": "Hilfreich-Quote pro (Schweregrad x Provider) aus Nutzer-Feedback der letzten 30 Tage." + }, "feedback": { "title": "Feedback", "subtitle": "Feedback und Bug-Reports der Nutzer sichten." diff --git a/messages/en.json b/messages/en.json index a06d15b4e..b801122ea 100644 --- a/messages/en.json +++ b/messages/en.json @@ -1526,6 +1526,21 @@ "empty": "Sampling started — data appears in 1–2 minutes.", "loadError": "Failed to load host metrics. Try refreshing the page." }, + "aiQuality": { + "title": "AI Quality", + "loading": "Loading feedback summary...", + "loadError": "Failed to load AI quality summary. Try refreshing the page.", + "empty": "No feedback yet. The aggregator runs daily at 04:00 — once users start rating recommendations, helpful-rate bands appear here.", + "windowLabel": "Last {days} days", + "generatedAtLabel": "Generated", + "colSeverity": "Severity", + "colMetric": "Metric", + "colProvider": "Provider", + "colPromptVersion": "Prompt", + "colHelpful": "Helpful", + "colNotHelpful": "Not helpful", + "colHelpfulRate": "Helpful rate" + }, "workerStatus": "Job Worker", "workerRunning": "Running", "workerStopped": "Stopped", @@ -1762,6 +1777,10 @@ "title": "Integrations", "subtitle": "Configure Umami, GlitchTip, Web Push, and bug reports." }, + "ai-quality": { + "title": "AI Quality", + "subtitle": "Helpful-rate per (severity x provider) from user feedback over the last 30 days." + }, "feedback": { "title": "Feedback", "subtitle": "Triage user-submitted feedback and bug reports." diff --git a/src/app/admin/[section]/renderer.tsx b/src/app/admin/[section]/renderer.tsx index bdc990d7e..fec261752 100644 --- a/src/app/admin/[section]/renderer.tsx +++ b/src/app/admin/[section]/renderer.tsx @@ -16,6 +16,7 @@ import type { JSX } from "react"; import { useAuth } from "@/hooks/use-auth"; import { useTranslations } from "@/lib/i18n/context"; +import { AiQualitySection } from "@/components/admin/ai-quality-section"; import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-section"; import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section"; import { BackupsSection } from "@/components/admin/backups-section"; @@ -83,6 +84,15 @@ export function AdminSectionRenderer({ ); + case "ai-quality": + return ( + + + + ); case "feedback": return ( { + await requireAdmin(); + + const settings = await prisma.appSettings.findUnique({ + where: { id: "singleton" }, + select: { adminAiInsightsFeedbackSummary: true }, + }); + + const summary = settings?.adminAiInsightsFeedbackSummary ?? null; + + annotate({ + action: { name: "admin.ai_quality.read" }, + meta: { + has_summary: summary !== null, + }, + }); + + return apiSuccess({ summary }); +}); diff --git a/src/components/admin/__tests__/ai-quality-section.test.tsx b/src/components/admin/__tests__/ai-quality-section.test.tsx new file mode 100644 index 000000000..364af534c --- /dev/null +++ b/src/components/admin/__tests__/ai-quality-section.test.tsx @@ -0,0 +1,123 @@ +import { describe, it, expect, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { I18nProvider } from "@/lib/i18n/context"; + +/** + * v1.4.16 phase B5e — coverage for the admin AI quality preview. + * + * Three branches: + * 1. loading state (Loader2 spinner + label) + * 2. empty state (no summary OR summary with empty buckets) + * 3. table-render (one row per bucket with the helpful-rate column) + * + * The fetch is replaced via a hoisted vi.fn so each test can stage a + * different response; the component reads through TanStack Query but + * we mock the hook to bypass the suspend-and-fetch dance and feed the + * data directly. + */ + +const mockQueryState = vi.hoisted(() => ({ + data: null as null | object, + isLoading: false, + isError: false, +})); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: () => ({ + data: mockQueryState.data, + isLoading: mockQueryState.isLoading, + isError: mockQueryState.isError, + }), +})); + +vi.mock("@/lib/i18n/context", async (orig) => { + const actual = (await orig()) as Record; + return actual; +}); + +import { AiQualitySection } from "../ai-quality-section"; + +function render(node: React.ReactElement) { + return renderToStaticMarkup( + {node}, + ); +} + +describe("", () => { + it("renders the loading state when query.isLoading", () => { + mockQueryState.data = null; + mockQueryState.isLoading = true; + mockQueryState.isError = false; + const html = render(); + expect(html).toContain("Loading feedback summary"); + }); + + it("renders the empty state when summary is null", () => { + mockQueryState.data = null; + mockQueryState.isLoading = false; + mockQueryState.isError = false; + const html = render(); + expect(html).toContain("AI Quality"); + expect(html).toContain("No feedback yet"); + }); + + it("renders the empty state when buckets[] is empty", () => { + mockQueryState.data = { + generatedAt: "2026-05-09T04:00:00.000Z", + windowDays: 30, + buckets: [], + }; + mockQueryState.isLoading = false; + mockQueryState.isError = false; + const html = render(); + expect(html).toContain("No feedback yet"); + }); + + it("renders one row per bucket with helpful-rate %", () => { + mockQueryState.data = { + generatedAt: "2026-05-09T04:00:00.000Z", + windowDays: 30, + buckets: [ + { + severity: "important", + metricSourceType: "bloodPressure", + providerType: "codex", + promptVersion: "4.16.0", + helpful: 8, + notHelpful: 2, + total: 10, + helpfulRate: 0.8, + }, + { + severity: "info", + metricSourceType: "weight", + providerType: "openai", + promptVersion: "4.16.0", + helpful: 1, + notHelpful: 4, + total: 5, + helpfulRate: 0.2, + }, + ], + }; + mockQueryState.isLoading = false; + mockQueryState.isError = false; + const html = render(); + expect(html).toContain("important"); + expect(html).toContain("bloodPressure"); + expect(html).toContain("codex"); + expect(html).toContain("80%"); + expect(html).toContain("20%"); + // Two `data-slot="ai-quality-bucket"` rows. + const matches = html.match(/data-slot="ai-quality-bucket"/g); + expect(matches?.length).toBe(2); + }); + + it("renders a load-error banner when query.isError", () => { + mockQueryState.data = null; + mockQueryState.isLoading = false; + mockQueryState.isError = true; + const html = render(); + expect(html).toContain("Failed to load AI quality summary"); + }); +}); diff --git a/src/components/admin/__tests__/sections.test.tsx b/src/components/admin/__tests__/sections.test.tsx index e76ec5c35..324809d7c 100644 --- a/src/components/admin/__tests__/sections.test.tsx +++ b/src/components/admin/__tests__/sections.test.tsx @@ -42,6 +42,7 @@ vi.mock("@/hooks/use-auth", () => ({ import { I18nProvider } from "@/lib/i18n/context"; import { ADMIN_SECTION_SLUGS } from "../section-slugs"; +import { AiQualitySection } from "../ai-quality-section"; import { ApiTokenOverviewSection } from "../api-token-overview-section"; import { BackupsSection } from "../backups-section"; import { DangerZoneSection } from "../danger-zone-section"; @@ -133,4 +134,18 @@ describe("admin sections — SSR smoke", () => { const html = render(); expect(html).toContain("Danger Zone"); }); + + // v1.4.16 phase B5e — admin AI quality preview. The mocked useQuery + // returns `data: null, isLoading: false` so the empty-state branch + // paints; the table-render branch is exercised by the dedicated + // ai-quality-section test. + it(" renders the empty-state when no summary exists", () => { + const html = render(); + expect(html).toContain("AI Quality"); + expect(html).toContain("No feedback yet"); + }); + + it("admin slug list contains ai-quality (renderer parity)", () => { + expect(ADMIN_SECTION_SLUGS).toContain("ai-quality"); + }); }); diff --git a/src/components/admin/admin-shell.tsx b/src/components/admin/admin-shell.tsx index 6217cb910..406184da4 100644 --- a/src/components/admin/admin-shell.tsx +++ b/src/components/admin/admin-shell.tsx @@ -28,6 +28,7 @@ import { Server, Settings, ShieldAlert, + Sparkles, Users, type LucideIcon, } from "lucide-react"; @@ -69,6 +70,11 @@ export const ADMIN_SECTIONS: readonly AdminSection[] = [ titleKey: "admin.section.integrations.title", icon: AlertTriangle, }, + { + slug: "ai-quality", + titleKey: "admin.section.ai-quality.title", + icon: Sparkles, + }, { slug: "feedback", titleKey: "admin.section.feedback.title", diff --git a/src/components/admin/ai-quality-section.tsx b/src/components/admin/ai-quality-section.tsx new file mode 100644 index 000000000..fc3be70bc --- /dev/null +++ b/src/components/admin/ai-quality-section.tsx @@ -0,0 +1,202 @@ +"use client"; + +/** + * `` — admin AI quality preview (v1.4.16 phase B5e). + * + * Surfaces the aggregator-written helpful-rate per (severity x + * provider x prompt-version) bucket. The pg-boss `feedback-aggregator` + * queue writes the underlying summary daily at 04:00 Europe/Berlin + * to `AppSettings.adminAiInsightsFeedbackSummary`; this component + * reads it via `GET /api/admin/ai-quality` (admin-gated). + * + * Empty-state ("no summary yet") rendered as a quiet message rather + * than a fake zero row so the operator can tell "no feedback" apart + * from "all-zeros". + * + * Layout: a compact table with one row per bucket. Mobile ( = { + info: "text-dracula-cyan", + suggestion: "text-dracula-purple", + important: "text-dracula-orange", + urgent: "text-dracula-red", +}; + +function helpfulRateColour(rate: number): string { + if (rate >= 0.8) return "text-dracula-green"; + if (rate >= 0.5) return "text-dracula-yellow"; + return "text-dracula-orange"; +} + +export function AiQualitySection() { + const { t } = useTranslations(); + const fmt = useFormatters(); + + const query = useQuery({ + queryKey: ["admin", "ai-quality"], + queryFn: async () => { + const res = await fetch("/api/admin/ai-quality"); + const json = (await res.json()) as AiQualityResponse; + if (!res.ok) throw new Error(json.error ?? "ai_quality_failed"); + return json.data?.summary ?? null; + }, + }); + + if (query.isLoading) { + return ( +
+ + + {t("admin.aiQuality.loading")} + +
+ ); + } + + if (query.isError) { + return ( +
+ {t("admin.aiQuality.loadError")} +
+ ); + } + + const summary = query.data; + + if (!summary || summary.buckets.length === 0) { + return ( +
+
+ +
+ {t("admin.aiQuality.title")} +
+
+

+ {t("admin.aiQuality.empty")} +

+
+ ); + } + + return ( +
+
+
+ +
+ {t("admin.aiQuality.title")} +
+
+

+ {t("admin.aiQuality.windowLabel", { + days: String(summary.windowDays), + })} + {" · "} + {t("admin.aiQuality.generatedAtLabel")}{" "} + {fmt.dateTime(summary.generatedAt)} +

+
+ +
+ + + + + + + + + + + + + + {summary.buckets.map((bucket, idx) => { + const ratePct = Math.round(bucket.helpfulRate * 100); + return ( + + + + + + + + + + ); + })} + +
+ {t("admin.aiQuality.colSeverity")} + + {t("admin.aiQuality.colMetric")} + + {t("admin.aiQuality.colProvider")} + + {t("admin.aiQuality.colPromptVersion")} + + {t("admin.aiQuality.colHelpful")} + + {t("admin.aiQuality.colNotHelpful")} + + {t("admin.aiQuality.colHelpfulRate")} +
+ {bucket.severity} + {bucket.metricSourceType}{bucket.providerType} + {bucket.promptVersion} + {bucket.helpful}{bucket.notHelpful} + {ratePct}% +
+
+
+ ); +} diff --git a/src/components/admin/section-slugs.ts b/src/components/admin/section-slugs.ts index 84d7ca5d1..68d13b470 100644 --- a/src/components/admin/section-slugs.ts +++ b/src/components/admin/section-slugs.ts @@ -14,6 +14,7 @@ export const ADMIN_SECTION_SLUGS = [ "general", "services", "integrations", + "ai-quality", "feedback", "reminders", "users",