diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index eca41144d..a82a525de 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -149,6 +149,49 @@ components: - data - error type: object + CoachMessageFeedbackRequest: + description: Per-message helpful/unhelpful feedback (v1.4.23 H7). Optional `reason` is free-form prose, capped at 200 chars. + properties: + rating: + enum: + - helpful + - unhelpful + type: string + reason: + maxLength: 200 + minLength: 1 + type: string + required: + - rating + type: object + CoachMessageFeedbackResponse: + additionalProperties: false + properties: + data: + additionalProperties: false + properties: + createdAt: + format: date-time + pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + type: string + id: + type: string + required: + - id + - createdAt + type: object + error: + type: "null" + meta: + additionalProperties: false + properties: + requestId: + type: string + type: object + required: + - data + - error + type: object CreateMeasurementResponse: additionalProperties: false properties: @@ -857,6 +900,42 @@ paths: summary: Register native device + APNs token tags: - Devices + /api/insights/chat/messages/{id}/feedback: + post: + description: Persists a helpful/unhelpful rating for a single Coach reply. Reuses the v1.4.16 RecommendationFeedback + table via the polymorphic `targetType` column. The aggregator buckets ratings by (promptVersion, tone, + verbosity). + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CoachMessageFeedbackRequest" + required: true + responses: + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/CoachMessageFeedbackResponse" + description: Feedback saved. + "401": *a1 + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + description: Message not found or not owned by the caller. + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + description: Caller has already rated this message text. + "422": *a2 + "429": *a3 + summary: Rate a Coach assistant message + tags: + - Insights /api/insights/comprehensive: get: description: Full Insights surface — daily briefing, recommendations with rationale, optional weekly report + storyboard diff --git a/messages/de.json b/messages/de.json index f15cef525..9306601c4 100644 --- a/messages/de.json +++ b/messages/de.json @@ -919,7 +919,10 @@ "settingsEvidenceHint": "Die Belege-Box unter jeder Antwort automatisch öffnen.", "settingsSave": "Speichern", "settingsSaved": "Gespeichert.", - "settingsCancel": "Abbrechen" + "settingsCancel": "Abbrechen", + "feedbackHelpful": "Hilfreich", + "feedbackUnhelpful": "Nicht ganz", + "feedbackThanks": "Danke für das Signal." }, "viewAll": "Alle ansehen", "previewEmpty": "Noch keine Empfehlungen.", @@ -1778,6 +1781,21 @@ "colNotHelpful": "Nicht hilfreich", "colHelpfulRate": "Hilfreich-Quote" }, + "coachFeedback": { + "title": "Coach-Feedback", + "loading": "Coach-Feedback-Zusammenfassung wird geladen...", + "loadError": "Coach-Feedback-Zusammenfassung konnte nicht geladen werden. Bitte die Seite neu laden.", + "empty": "Noch kein Coach-Feedback. Der Aggregator läuft täglich um 04:00 — sobald Nutzer Coach-Antworten bewerten, erscheinen hier die Hilfreich-Quoten.", + "windowLabel": "Letzte {days} Tage", + "generatedAtLabel": "Erstellt", + "colPromptVersion": "Prompt-Version", + "colTone": "Tonfall", + "colVerbosity": "Ausführlichkeit", + "colHelpful": "Hilfreich", + "colNotHelpful": "Nicht hilfreich", + "colN": "n", + "colHelpfulRate": "Hilfreich-Quote" + }, "workerStatus": "Job-Worker", "workerRunning": "Aktiv", "workerStopped": "Gestoppt", @@ -2024,6 +2042,10 @@ "title": "KI-Qualität", "subtitle": "Hilfreichkeit nach Schweregrad und Provider — Nutzer-Feedback der letzten 30 Tage." }, + "coach-feedback": { + "title": "Coach-Feedback", + "subtitle": "Hilfreich-Quote pro Coach-Prompt-Version, Tonfall und Ausführlichkeit der letzten 30 Tage." + }, "feedback": { "title": "Feedback", "subtitle": "Feedback, Bug-Reports und Feature-Wünsche der Nutzer sichten." diff --git a/messages/en.json b/messages/en.json index c6df0438c..341e54b67 100644 --- a/messages/en.json +++ b/messages/en.json @@ -919,7 +919,10 @@ "settingsEvidenceHint": "Open the disclosure under every reply automatically.", "settingsSave": "Save", "settingsSaved": "Saved.", - "settingsCancel": "Cancel" + "settingsCancel": "Cancel", + "feedbackHelpful": "Helpful", + "feedbackUnhelpful": "Not quite", + "feedbackThanks": "Thanks for the signal." }, "viewAll": "View all", "previewEmpty": "No recommendations yet.", @@ -1778,6 +1781,21 @@ "colNotHelpful": "Not helpful", "colHelpfulRate": "Helpful rate" }, + "coachFeedback": { + "title": "Coach Feedback", + "loading": "Loading Coach feedback summary...", + "loadError": "Failed to load Coach feedback summary. Try refreshing the page.", + "empty": "No Coach feedback yet. The aggregator runs daily at 04:00 — once users rate Coach replies, helpful-rate bands appear here.", + "windowLabel": "Last {days} days", + "generatedAtLabel": "Generated", + "colPromptVersion": "Prompt version", + "colTone": "Tone", + "colVerbosity": "Verbosity", + "colHelpful": "Helpful", + "colNotHelpful": "Not helpful", + "colN": "n", + "colHelpfulRate": "Helpful rate" + }, "workerStatus": "Job Worker", "workerRunning": "Running", "workerStopped": "Stopped", @@ -2024,6 +2042,10 @@ "title": "AI Quality", "subtitle": "Helpfulness scores grouped by severity and provider, from the last 30 days of user feedback." }, + "coach-feedback": { + "title": "Coach Feedback", + "subtitle": "Helpful-rate per Coach prompt version, tone, and verbosity from the last 30 days." + }, "feedback": { "title": "Feedback", "subtitle": "Triage user feedback, bug reports, and feature requests." diff --git a/prisma/migrations/0040_recommendation_feedback_target_type/migration.sql b/prisma/migrations/0040_recommendation_feedback_target_type/migration.sql new file mode 100644 index 000000000..b9bd942c7 --- /dev/null +++ b/prisma/migrations/0040_recommendation_feedback_target_type/migration.sql @@ -0,0 +1,18 @@ +-- v1.4.23 H7 — polymorphic feedback target. +-- +-- Reuses the v1.4.16 B5e RecommendationFeedback table for Coach +-- assistant-message thumbs so the aggregator only has to slice one +-- table. `target_type` defaults to "recommendation" so every +-- pre-v1.4.23 row reads as the existing surface; new Coach rows write +-- "coach" + the message id in the existing `recommendation_id` slot. +-- `reason` is optional free-form prose (capped to 200 chars by the +-- route). +-- +-- Index on (target_type, created_at) lets the admin aggregator scan +-- per-surface buckets without a full table sweep. +ALTER TABLE "recommendation_feedback" +ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'recommendation', +ADD COLUMN IF NOT EXISTS "reason" TEXT; + +CREATE INDEX IF NOT EXISTS "recommendation_feedback_target_type_created_at_idx" +ON "recommendation_feedback" ("target_type", "created_at"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 91c4669f6..1e7c7115d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -913,11 +913,24 @@ model RecommendationFeedback { promptVersion String @map("prompt_version") createdAt DateTime @default(now()) @map("created_at") + // v1.4.23 H7 — polymorphic target so Coach replies share the table. + // "recommendation" (default) is the v1.4.16 B5e shape; "coach" is a + // Coach assistant message. The aggregator slices on targetType to + // bucket each surface independently. Default value lets every + // pre-v1.4.23 row read as "recommendation" without a backfill. + targetType String @default("recommendation") @map("target_type") + // Optional free-form reason — populated when the user clicks + // "unhelpful" and selects a category from the small chip strip. + // Capped at 200 chars upstream so the table can't bloat from a + // pasted essay. + reason String? @map("reason") + user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@unique([userId, recommendationId, recommendationText], map: "rec_feedback_user_rec_text_key") @@index([userId, createdAt]) @@index([recommendationSeverity, helpful, createdAt]) + @@index([targetType, createdAt]) @@map("recommendation_feedback") } diff --git a/src/app/admin/[section]/renderer.tsx b/src/app/admin/[section]/renderer.tsx index fec261752..f4178eb6f 100644 --- a/src/app/admin/[section]/renderer.tsx +++ b/src/app/admin/[section]/renderer.tsx @@ -17,6 +17,7 @@ import type { JSX } from "react"; import { useAuth } from "@/hooks/use-auth"; import { useTranslations } from "@/lib/i18n/context"; import { AiQualitySection } from "@/components/admin/ai-quality-section"; +import { CoachFeedbackSection } from "@/components/admin/coach-feedback-section"; import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-section"; import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section"; import { BackupsSection } from "@/components/admin/backups-section"; @@ -93,6 +94,15 @@ export function AdminSectionRenderer({ ); + case "coach-feedback": + return ( + + + + ); case "feedback": return ( ; +} + +async function handlePost(request: NextRequest, ctx: RouteContext) { + const { user } = await requireAuth(); + const { id: messageId } = await ctx.params; + + // Per-user rate limit — same shape as the recommendation feedback + // route (60/h). Keeps a chatty client from drowning the bucketing + // signal. + const rl = await checkRateLimit( + `coach-feedback:${user.id}`, + 60, + 60 * 60 * 1000, + ); + if (!rl.allowed) { + annotate({ + action: { name: "insights.coach.message.feedback" }, + meta: { outcome: "rate_limited" }, + }); + return apiError("Too many feedback submissions, try again later", 429); + } + + const { data: rawBody, error } = await safeJson(request); + if (error) return error; + const parsed = coachMessageFeedbackSchema.safeParse(rawBody); + if (!parsed.success) { + annotate({ + action: { name: "insights.coach.message.feedback" }, + meta: { outcome: "validation_failed" }, + }); + throw new HttpError(422, "feedback.body.invalid"); + } + const body = parsed.data; + + // Verify ownership + pull the prose snapshot. The encrypted-content + // model stores the message body via AES-256-GCM; decrypt to use as + // the `recommendationText` slot so the per-(user, message, text) + // dedup remains stable across re-rates. + const message = await prisma.coachMessage.findFirst({ + where: { + id: messageId, + role: "assistant", + conversation: { userId: user.id }, + }, + select: { id: true, encryptedContent: true }, + }); + if (!message) { + annotate({ + action: { name: "insights.coach.message.feedback" }, + meta: { outcome: "not_found" }, + }); + throw new HttpError(404, "coach.message.notFound"); + } + + let proseSnapshot = ""; + try { + const ciphertext = Buffer.from(message.encryptedContent).toString("utf8"); + proseSnapshot = decrypt(ciphertext); + } catch { + // Decryption failure shouldn't block the feedback row — fall back + // to a stable id-keyed marker so the dedup still works. + proseSnapshot = ``; + } + // Cap the snapshot at the same byte budget the rec-feedback table + // already accepts (a Coach reply is usually ~400-1500 chars; clip + // hard at 4 KB so a runaway provider can't bloat the table). + if (proseSnapshot.length > 4096) { + proseSnapshot = proseSnapshot.slice(0, 4096); + } + + const attribution = await resolveCoachFeedbackAttribution(user.id, messageId); + + try { + const created = await prisma.recommendationFeedback.create({ + data: { + userId: user.id, + recommendationId: messageId, + recommendationText: proseSnapshot, + recommendationSeverity: "coach", + metricSourceType: attribution.metricSourceType, + metricSourceTimeRange: "single_message", + helpful: body.rating === "helpful", + providerType: attribution.providerType, + promptVersion: attribution.promptVersion, + targetType: "coach", + reason: body.reason ?? null, + }, + select: { id: true, createdAt: true }, + }); + + await auditLog("insights.coach.message.feedback", { + userId: user.id, + ipAddress: getClientIp(request), + details: { + feedbackId: created.id, + messageId, + rating: body.rating, + providerType: attribution.providerType, + promptVersion: attribution.promptVersion, + metricSourceType: attribution.metricSourceType, + }, + }); + + annotate({ + action: { name: "insights.coach.message.feedback" }, + meta: { + outcome: "created", + rating: body.rating, + providerType: attribution.providerType, + promptVersion: attribution.promptVersion, + }, + }); + + return apiSuccess({ id: created.id, createdAt: created.createdAt }, 201); + } catch (err) { + if ( + err instanceof Prisma.PrismaClientKnownRequestError && + err.code === "P2002" + ) { + annotate({ + action: { name: "insights.coach.message.feedback" }, + meta: { outcome: "already_rated" }, + }); + return apiError("already_rated", 409); + } + throw err; + } +} + +export const POST = apiHandler(handlePost); diff --git a/src/components/admin/__tests__/coach-feedback-section.test.tsx b/src/components/admin/__tests__/coach-feedback-section.test.tsx new file mode 100644 index 000000000..47b9d88e8 --- /dev/null +++ b/src/components/admin/__tests__/coach-feedback-section.test.tsx @@ -0,0 +1,110 @@ +import { describe, expect, it, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +/** + * v1.4.23 H7 — admin Coach feedback section. + * + * Three render paths matter: + * 1. Loading — spinner + "Loading Coach feedback summary..." + * 2. Empty — quiet message when the aggregator hasn't run or no + * Coach rows exist in the rolling window. + * 3. Populated — table renders one row per (promptVersion, tone, + * verbosity) bucket with helpful-rate column tinted by band. + */ + +const useQueryMock = vi.fn(); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: (...args: unknown[]) => useQueryMock(...args), +})); + +import { I18nProvider } from "@/lib/i18n/context"; +import { CoachFeedbackSection } from "../coach-feedback-section"; + +function render() { + return renderToStaticMarkup( + + + , + ); +} + +describe("CoachFeedbackSection", () => { + it("renders the loading state when the query is pending", () => { + useQueryMock.mockReturnValue({ + isLoading: true, + isError: false, + data: null, + }); + const html = render(); + expect(html).toContain("Loading Coach feedback summary"); + expect(html).toContain("animate-spin"); + }); + + it("renders the empty state when the summary has no Coach buckets", () => { + useQueryMock.mockReturnValue({ + isLoading: false, + isError: false, + data: { + generatedAt: "2026-05-10T04:00:00Z", + windowDays: 30, + coachBuckets: [], + }, + }); + const html = render(); + expect(html).toContain("No Coach feedback yet"); + expect(html).toContain("Coach Feedback"); + }); + + it("renders one row per bucket with helpful-rate tint", () => { + useQueryMock.mockReturnValue({ + isLoading: false, + isError: false, + data: { + generatedAt: "2026-05-10T04:00:00Z", + windowDays: 30, + coachBuckets: [ + { + promptVersion: "4.23.0", + tone: "warm", + verbosity: "default", + helpful: 12, + notHelpful: 3, + total: 15, + helpfulRate: 0.8, + }, + { + promptVersion: "4.23.0", + tone: "concise", + verbosity: "brief", + helpful: 4, + notHelpful: 4, + total: 8, + helpfulRate: 0.5, + }, + ], + }, + }); + const html = render(); + // Two bucket rows in the table. + const matches = (html.match(/coach-feedback-bucket/g) ?? []).length; + expect(matches).toBe(2); + // Helpful-rate column carries the band tint. + expect(html).toContain("text-dracula-green"); // 80% bucket + expect(html).toContain("text-dracula-yellow"); // 50% bucket + // Numbers visible in the markup. + expect(html).toContain("80%"); + expect(html).toContain("50%"); + }); + + it("renders the load-error path when the fetch failed", () => { + useQueryMock.mockReturnValue({ + isLoading: false, + isError: true, + data: null, + }); + const html = render(); + expect(html).toContain("Failed to load Coach feedback summary"); + expect(html).toContain('role="alert"'); + }); +}); diff --git a/src/components/admin/coach-feedback-section.tsx b/src/components/admin/coach-feedback-section.tsx new file mode 100644 index 000000000..3bf76b386 --- /dev/null +++ b/src/components/admin/coach-feedback-section.tsx @@ -0,0 +1,188 @@ +"use client"; + +/** + * `` — admin view onto the v1.4.23 H7 Coach + * helpful/unhelpful aggregate. Reads the same feedback-aggregator row + * that the AI Quality view consumes; this section just slices the + * Coach-only buckets so the operator can see prompt-version × helpful- + * rate × n at a glance. + * + * First useful question this answers in the v1.4.23 first-week window: + * "Is the v1.4.22 Coach prose rewrite (PROMPT_VERSION 4.22.0 → 4.23.x) + * landing well, or did the warm tone overshoot?" If the helpful-rate + * for the new prompt version drops below 50 % within the first 100 + * ratings, v1.4.24 walks the persona back. + * + * Empty state when the aggregator hasn't run or no Coach feedback has + * landed in the rolling window. Cross-user aggregation only — the + * admin view never exposes per-user prose. + */ + +import { useQuery } from "@tanstack/react-query"; +import { Loader2, Sparkles } from "lucide-react"; + +import { useTranslations, useFormatters } from "@/lib/i18n/context"; + +interface CoachFeedbackBucket { + promptVersion: string; + tone: string; + verbosity: string; + helpful: number; + notHelpful: number; + total: number; + helpfulRate: number; +} + +interface FeedbackSummary { + generatedAt: string; + windowDays: number; + coachBuckets?: CoachFeedbackBucket[]; +} + +interface CoachFeedbackResponse { + data: { summary: FeedbackSummary | null } | null; + error?: string | null; +} + +function helpfulRateColour(rate: number): string { + if (rate >= 0.8) return "text-dracula-green"; + if (rate >= 0.5) return "text-dracula-yellow"; + return "text-dracula-orange"; +} + +export function CoachFeedbackSection() { + const { t } = useTranslations(); + const fmt = useFormatters(); + + const query = useQuery({ + queryKey: ["admin", "coach-feedback"], + queryFn: async () => { + const res = await fetch("/api/admin/ai-quality"); + const json = (await res.json()) as CoachFeedbackResponse; + if (!res.ok) throw new Error(json.error ?? "coach_feedback_failed"); + return json.data?.summary ?? null; + }, + }); + + if (query.isLoading) { + return ( +
+ + + {t("admin.coachFeedback.loading")} + +
+ ); + } + + if (query.isError) { + return ( +
+ {t("admin.coachFeedback.loadError")} +
+ ); + } + + const summary = query.data; + const buckets = summary?.coachBuckets ?? []; + + if (!summary || buckets.length === 0) { + return ( +
+
+ +
+ {t("admin.coachFeedback.title")} +
+
+

+ {t("admin.coachFeedback.empty")} +

+
+ ); + } + + return ( +
+
+
+ +
+ {t("admin.coachFeedback.title")} +
+
+

+ {t("admin.coachFeedback.windowLabel", { + days: String(summary.windowDays), + })} + {" · "} + {t("admin.coachFeedback.generatedAtLabel")}{" "} + {fmt.dateTime(summary.generatedAt)} +

+
+ +
+ + + + + + + + + + + + + + {buckets.map((bucket, idx) => { + const ratePct = Math.round(bucket.helpfulRate * 100); + return ( + + + + + + + + + + ); + })} + +
+ {t("admin.coachFeedback.colPromptVersion")} + + {t("admin.coachFeedback.colTone")} + + {t("admin.coachFeedback.colVerbosity")} + + {t("admin.coachFeedback.colHelpful")} + + {t("admin.coachFeedback.colNotHelpful")} + + {t("admin.coachFeedback.colN")} + + {t("admin.coachFeedback.colHelpfulRate")} +
+ {bucket.promptVersion} + {bucket.tone}{bucket.verbosity}{bucket.helpful} + {bucket.notHelpful} + {bucket.total} + {ratePct}% +
+
+
+ ); +} diff --git a/src/components/admin/section-slugs.ts b/src/components/admin/section-slugs.ts index 68d13b470..895ddfded 100644 --- a/src/components/admin/section-slugs.ts +++ b/src/components/admin/section-slugs.ts @@ -15,6 +15,7 @@ export const ADMIN_SECTION_SLUGS = [ "services", "integrations", "ai-quality", + "coach-feedback", "feedback", "reminders", "users", diff --git a/src/components/insights/coach-panel/message-thread.tsx b/src/components/insights/coach-panel/message-thread.tsx index beccb1b7e..629801cdc 100644 --- a/src/components/insights/coach-panel/message-thread.tsx +++ b/src/components/insights/coach-panel/message-thread.tsx @@ -1,8 +1,15 @@ "use client"; import { useEffect, useRef, useState } from "react"; -import { useQuery } from "@tanstack/react-query"; -import { Bot, ChevronRight, Sparkles, User } from "lucide-react"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { + Bot, + ChevronRight, + Sparkles, + ThumbsDown, + ThumbsUp, + User, +} from "lucide-react"; import { cn } from "@/lib/utils"; import { useTranslations } from "@/lib/i18n/context"; @@ -180,6 +187,7 @@ export function MessageThread({ metricSource={m.metricSource} providerType={m.providerType} evidenceDefaultOpen={evidenceDefaultOpen} + messageId={m.id} /> ); })} @@ -228,6 +236,12 @@ interface ChatBubbleProps { errorCode?: string | null; /** v1.4.23 H4 — when true the evidence `
` mounts open. */ evidenceDefaultOpen?: boolean; + /** + * v1.4.23 H7 — present only on persisted assistant messages. + * Streaming bubbles (no message id yet) skip the thumbs row so the + * user can't rate before the message lands on disk. + */ + messageId?: string; } function ChatBubble({ @@ -238,6 +252,7 @@ function ChatBubble({ inProgress, errorCode, evidenceDefaultOpen, + messageId, }: ChatBubbleProps) { const { t } = useTranslations(); const { user } = useAuth(); @@ -383,7 +398,87 @@ function ChatBubble({
)} + {/* v1.4.23 H7 — per-message thumbs feedback. Only persisted + assistant messages get the row (skipped for refusals, + errors, in-flight stream bubbles). The aggregator buckets + the rating by (promptVersion, tone, verbosity). */} + {messageId && + !inProgress && + !errorCode && + providerType !== "refusal" && ( + + )} ); } + +interface CoachMessageFeedbackProps { + messageId: string; +} + +function CoachMessageFeedback({ messageId }: CoachMessageFeedbackProps) { + const { t } = useTranslations(); + const [submittedRating, setSubmittedRating] = useState< + "helpful" | "unhelpful" | null + >(null); + + const submit = useMutation({ + mutationFn: async (rating: "helpful" | "unhelpful") => { + const res = await fetch( + `/api/insights/chat/messages/${messageId}/feedback`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ rating }), + }, + ); + // Treat 409 (already_rated) as a successful no-op so the user + // never sees an error toast for double-clicking the same chip. + if (!res.ok && res.status !== 409) { + throw new Error("coach-feedback.failed"); + } + return rating; + }, + onSuccess: (rating) => setSubmittedRating(rating), + }); + + if (submittedRating) { + return ( +

+ {t("insights.coach.feedbackThanks")} +

+ ); + } + + return ( +
+ + +
+ ); +} diff --git a/src/lib/ai/feedback-attribution.ts b/src/lib/ai/feedback-attribution.ts index 315148550..a00760942 100644 --- a/src/lib/ai/feedback-attribution.ts +++ b/src/lib/ai/feedback-attribution.ts @@ -81,3 +81,65 @@ export async function resolveFeedbackAttribution( promptVersion: PROMPT_VERSION, }; } + +/** + * v1.4.23 H7 — attribution shape for a Coach assistant-message + * thumbs row. The polymorphic `RecommendationFeedback` table reuses + * its existing slots for Coach rows: + * - `recommendationId` carries the Coach message id + * - `recommendationText` carries the assistant prose snapshot + * - `recommendationSeverity` is always "coach" so dashboards can + * filter / group on the literal + * - `metricSourceType` encodes the user's active prefs at the + * moment of the rating: `coach:tone=warm:verbosity=default`. The + * aggregator buckets on this field so a future audit can slice + * helpful-rate per (tone, verbosity) without a join. + * - `metricSourceTimeRange` is always "single_message" + * - `target_type` is "coach" + */ +export interface CoachFeedbackAttribution extends FeedbackAttribution { + metricSourceType: string; +} + +export function buildCoachMetricSourceType( + tone: string, + verbosity: string, +): string { + return `coach:tone=${tone}:verbosity=${verbosity}`; +} + +/** + * Resolve the (providerType, promptVersion, metricSourceType) tuple + * for a Coach feedback row. Reads the user's latest persisted + * Coach assistant message to recover the providerType + promptVersion + * stamped on it, and queries the user's active `coachPrefsJson` for + * the tone/verbosity slice. Falls back to safe defaults when either + * lookup misses. + */ +export async function resolveCoachFeedbackAttribution( + userId: string, + messageId: string, +): Promise { + const [message, userRow] = await Promise.all([ + prisma.coachMessage.findFirst({ + where: { id: messageId, conversation: { userId } }, + select: { providerType: true, promptVersion: true }, + }), + prisma.user.findUnique({ + where: { id: userId }, + select: { coachPrefsJson: true }, + }), + ]); + + // Lazy-import to avoid circular imports — coach-prefs validation + // doesn't depend on attribution but the inverse would create a cycle. + const { parseCoachPrefs } = await import("@/lib/validations/coach-prefs"); + const prefs = parseCoachPrefs(userRow?.coachPrefsJson); + + return { + providerType: + message?.providerType ?? FEEDBACK_ATTRIBUTION_FALLBACK.providerType, + promptVersion: message?.promptVersion ?? PROMPT_VERSION, + metricSourceType: buildCoachMetricSourceType(prefs.tone, prefs.verbosity), + }; +} diff --git a/src/lib/jobs/__tests__/feedback-aggregator.test.ts b/src/lib/jobs/__tests__/feedback-aggregator.test.ts index dde4a2a5f..fb998cdb9 100644 --- a/src/lib/jobs/__tests__/feedback-aggregator.test.ts +++ b/src/lib/jobs/__tests__/feedback-aggregator.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi } from "vitest"; import type { PrismaClient } from "@/generated/prisma/client"; import { aggregateRecommendationFeedback, + buildCoachFeedbackBuckets, buildFeedbackBuckets, DEFAULT_FEEDBACK_AGGREGATION_WINDOW_DAYS, } from "../feedback-aggregator"; @@ -12,6 +13,7 @@ interface FeedbackRow { providerType: string; promptVersion: string; helpful: boolean; + targetType?: string; } function makePrismaMock(rows: FeedbackRow[]) { @@ -165,5 +167,98 @@ describe("aggregateRecommendationFeedback", () => { expect(upsertCall.update.adminAiInsightsFeedbackSummary.buckets).toEqual( [], ); + // v1.4.23 H7 — coachBuckets always present, empty by default. + expect( + upsertCall.update.adminAiInsightsFeedbackSummary.coachBuckets, + ).toEqual([]); + }); +}); + +/** + * v1.4.23 H7 — Coach-only bucket view. The aggregator slices the + * unified RecommendationFeedback rows on `targetType === "coach"` and + * groups by (promptVersion, tone, verbosity). + */ +describe("buildCoachFeedbackBuckets", () => { + it("ignores non-Coach rows", () => { + const rows: FeedbackRow[] = [ + { + recommendationSeverity: "important", + metricSourceType: "bloodPressure", + providerType: "codex", + promptVersion: "4.16.0", + helpful: true, + targetType: "recommendation", + }, + ]; + expect(buildCoachFeedbackBuckets(rows)).toEqual([]); + }); + + it("groups Coach rows by (promptVersion, tone, verbosity)", () => { + const rows: FeedbackRow[] = [ + { + recommendationSeverity: "coach", + metricSourceType: "coach:tone=warm:verbosity=default", + providerType: "codex", + promptVersion: "4.23.0", + helpful: true, + targetType: "coach", + }, + { + recommendationSeverity: "coach", + metricSourceType: "coach:tone=warm:verbosity=default", + providerType: "codex", + promptVersion: "4.23.0", + helpful: true, + targetType: "coach", + }, + { + recommendationSeverity: "coach", + metricSourceType: "coach:tone=warm:verbosity=default", + providerType: "codex", + promptVersion: "4.23.0", + helpful: false, + targetType: "coach", + }, + { + recommendationSeverity: "coach", + metricSourceType: "coach:tone=concise:verbosity=brief", + providerType: "codex", + promptVersion: "4.23.0", + helpful: true, + targetType: "coach", + }, + ]; + const buckets = buildCoachFeedbackBuckets(rows); + expect(buckets).toHaveLength(2); + const warm = buckets.find( + (b) => b.tone === "warm" && b.verbosity === "default", + ); + expect(warm).toBeDefined(); + expect(warm?.helpful).toBe(2); + expect(warm?.notHelpful).toBe(1); + expect(warm?.helpfulRate).toBeCloseTo(0.67, 2); + const concise = buckets.find( + (b) => b.tone === "concise" && b.verbosity === "brief", + ); + expect(concise?.total).toBe(1); + expect(concise?.helpfulRate).toBe(1); + }); + + it("falls back to tone=unknown when metricSourceType doesn't match the encoded shape", () => { + const rows: FeedbackRow[] = [ + { + recommendationSeverity: "coach", + metricSourceType: "weird-shape", + providerType: "codex", + promptVersion: "4.23.0", + helpful: false, + targetType: "coach", + }, + ]; + const buckets = buildCoachFeedbackBuckets(rows); + expect(buckets).toHaveLength(1); + expect(buckets[0].tone).toBe("unknown"); + expect(buckets[0].verbosity).toBe("unknown"); }); }); diff --git a/src/lib/jobs/feedback-aggregator.ts b/src/lib/jobs/feedback-aggregator.ts index d551ced42..6e4d7ec8b 100644 --- a/src/lib/jobs/feedback-aggregator.ts +++ b/src/lib/jobs/feedback-aggregator.ts @@ -35,12 +35,41 @@ export interface FeedbackBucket { total: number; /** Fraction in [0..1], rounded to 2 decimals. Always 0 when total=0. */ helpfulRate: number; + /** + * v1.4.23 H7 — surface origin so the admin dashboard can group + * recommendation rows separately from Coach assistant-message rows + * (the existing buckets blended both before this field landed). + */ + targetType: string; +} + +/** + * v1.4.23 H7 — slim view onto the Coach-only rows so the admin + * coach-feedback dashboard can render the first-week aggregate without + * re-deriving the slice from the recommendation buckets. + */ +export interface CoachFeedbackBucket { + promptVersion: string; + /** Encoded as `tone=warm|neutral|concise` */ + tone: string; + /** Encoded as `verbosity=brief|default|detailed` */ + verbosity: string; + helpful: number; + notHelpful: number; + total: number; + helpfulRate: number; } export interface FeedbackAggregationSummary { generatedAt: string; windowDays: number; buckets: FeedbackBucket[]; + /** + * v1.4.23 H7 — Coach assistant-message buckets sliced by + * (promptVersion, tone, verbosity). Empty array when the + * aggregation window has no Coach feedback rows. + */ + coachBuckets: CoachFeedbackBucket[]; } interface MinimalFeedbackRow { @@ -49,6 +78,13 @@ interface MinimalFeedbackRow { providerType: string; promptVersion: string; helpful: boolean; + /** + * v1.4.23 H7 — present on every row from the new schema. Defaults to + * "recommendation" for legacy rows so the bucketing still works + * during the upgrade window when the migration has run but the route + * hasn't started writing the field yet. + */ + targetType?: string; } /** @@ -67,7 +103,9 @@ export function buildFeedbackBuckets( ): FeedbackBucket[] { const map = new Map(); for (const row of rows) { + const targetType = row.targetType ?? "recommendation"; const key = [ + targetType, row.recommendationSeverity, row.metricSourceType, row.providerType, @@ -88,6 +126,7 @@ export function buildFeedbackBuckets( notHelpful: row.helpful ? 0 : 1, total: 1, helpfulRate: 0, + targetType, }); } } @@ -100,6 +139,7 @@ export function buildFeedbackBuckets( buckets.sort((a, b) => { return ( + a.targetType.localeCompare(b.targetType) || a.severity.localeCompare(b.severity) || a.metricSourceType.localeCompare(b.metricSourceType) || a.providerType.localeCompare(b.providerType) || @@ -110,6 +150,69 @@ export function buildFeedbackBuckets( return buckets; } +/** + * v1.4.23 H7 — derive the Coach-only bucket view from the unified + * recommendation feedback rows. The Coach route writes the user's + * active prefs into `metricSourceType` as + * `coach:tone=:verbosity=`; this helper parses that back out so + * the admin dashboard can group on the typed dimensions without + * re-running a regex per row. + * + * Rows whose `targetType !== "coach"` are ignored. Rows whose + * `metricSourceType` does not match the encoded shape land under + * `tone=unknown:verbosity=unknown` so an early-write pre-migration row + * still contributes to a bucket rather than silently dropping. + */ +export function buildCoachFeedbackBuckets( + rows: MinimalFeedbackRow[], +): CoachFeedbackBucket[] { + const map = new Map(); + for (const row of rows) { + if ((row.targetType ?? "recommendation") !== "coach") continue; + const { tone, verbosity } = parseCoachMetricSource(row.metricSourceType); + const key = `${row.promptVersion}|${tone}|${verbosity}`; + const existing = map.get(key); + if (existing) { + if (row.helpful) existing.helpful += 1; + else existing.notHelpful += 1; + existing.total += 1; + } else { + map.set(key, { + promptVersion: row.promptVersion, + tone, + verbosity, + helpful: row.helpful ? 1 : 0, + notHelpful: row.helpful ? 0 : 1, + total: 1, + helpfulRate: 0, + }); + } + } + const buckets = [...map.values()]; + for (const b of buckets) { + b.helpfulRate = + b.total === 0 ? 0 : Math.round((b.helpful / b.total) * 100) / 100; + } + buckets.sort( + (a, b) => + a.promptVersion.localeCompare(b.promptVersion) || + a.tone.localeCompare(b.tone) || + a.verbosity.localeCompare(b.verbosity), + ); + return buckets; +} + +const COACH_METRIC_SOURCE_RE = /^coach:tone=([^:]+):verbosity=([^:]+)$/; + +function parseCoachMetricSource(metricSourceType: string): { + tone: string; + verbosity: string; +} { + const match = metricSourceType.match(COACH_METRIC_SOURCE_RE); + if (!match) return { tone: "unknown", verbosity: "unknown" }; + return { tone: match[1], verbosity: match[2] }; +} + export interface AggregateOptions { /** Override "now" for deterministic tests. Defaults to `new Date()`. */ now?: Date; @@ -139,6 +242,7 @@ export async function aggregateRecommendationFeedback( providerType: true, promptVersion: true, helpful: true, + targetType: true, }, }); @@ -146,6 +250,7 @@ export async function aggregateRecommendationFeedback( generatedAt: now.toISOString(), windowDays, buckets: buildFeedbackBuckets(rows), + coachBuckets: buildCoachFeedbackBuckets(rows), }; // The singleton AppSettings row is auto-created on first install diff --git a/src/lib/openapi/routes.ts b/src/lib/openapi/routes.ts index 7eea9af99..aa263f9ee 100644 --- a/src/lib/openapi/routes.ts +++ b/src/lib/openapi/routes.ts @@ -103,6 +103,17 @@ coachPrefsSchema.meta({ "Per-user Coach prompt-tuning preferences (v1.4.23 H4). All fields default to the legacy v1.4.22 behaviour when omitted.", }); +const coachMessageFeedbackBody = z + .object({ + rating: z.enum(["helpful", "unhelpful"]), + reason: z.string().min(1).max(200).optional(), + }) + .meta({ + id: "CoachMessageFeedbackRequest", + description: + "Per-message helpful/unhelpful feedback (v1.4.23 H7). Optional `reason` is free-form prose, capped at 200 chars.", + }); + // ── Sub-schemas owned here (route-specific shapes) ─────────────────── const passkeyLoginVerifyRequest = z @@ -526,6 +537,45 @@ export const openApiPaths: NonNullable = { }, }, }, + "/api/insights/chat/messages/{id}/feedback": { + post: { + tags: ["Insights"], + summary: "Rate a Coach assistant message", + description: + "Persists a helpful/unhelpful rating for a single Coach reply. Reuses the v1.4.16 RecommendationFeedback table via the polymorphic `targetType` column. The aggregator buckets ratings by (promptVersion, tone, verbosity).", + requestBody: { + required: true, + content: { + "application/json": { schema: coachMessageFeedbackBody }, + }, + }, + responses: { + "201": { + description: "Feedback saved.", + content: { + "application/json": { + schema: dataEnvelope( + z.object({ + id: z.string(), + createdAt: z.iso.datetime({ offset: true }), + }), + "CoachMessageFeedbackResponse", + ), + }, + }, + }, + "404": { + description: "Message not found or not owned by the caller.", + content: { "application/json": { schema: errorEnvelope } }, + }, + "409": { + description: "Caller has already rated this message text.", + content: { "application/json": { schema: errorEnvelope } }, + }, + ...stdResponses, + }, + }, + }, "/api/insights/comprehensive": { get: { tags: ["Insights"], diff --git a/tests/integration/coach-feedback.test.ts b/tests/integration/coach-feedback.test.ts new file mode 100644 index 000000000..eca885a48 --- /dev/null +++ b/tests/integration/coach-feedback.test.ts @@ -0,0 +1,177 @@ +/** + * v1.4.23 H7 — Coach assistant-message thumbs feedback route. + * + * Round-trip: seed a session + a coach conversation + an assistant + * message, POST a thumbs rating, assert a `RecommendationFeedback` row + * landed with `targetType="coach"` and the encoded prefs metric source. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { encrypt } from "@/lib/crypto"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +async function seedSession(username: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username, email: `${username}@example.test` }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user; +} + +async function seedAssistantMessage(userId: string, content: string) { + const prisma = getPrismaClient(); + const conversation = await prisma.coachConversation.create({ + data: { userId, title: "Test conversation" }, + }); + const ciphertext = encrypt(content); + return prisma.coachMessage.create({ + data: { + conversationId: conversation.id, + role: "assistant", + encryptedContent: Buffer.from(ciphertext, "utf8"), + providerType: "codex", + promptVersion: "4.23.0", + }, + }); +} + +describe("POST /api/insights/chat/messages/:id/feedback", () => { + it("persists a thumbs-up with target_type=coach and the encoded metric source", async () => { + const prisma = getPrismaClient(); + const user = await seedSession("coach-feedback-h7"); + const message = await seedAssistantMessage(user.id, "Your last week..."); + + const { POST } = await import( + "@/app/api/insights/chat/messages/[id]/feedback/route" + ); + const res = await ( + POST as ( + req: Request, + ctx: { params: Promise<{ id: string }> }, + ) => Promise + )( + new Request( + `http://localhost/api/insights/chat/messages/${message.id}/feedback`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ rating: "helpful" }), + }, + ), + { params: Promise.resolve({ id: message.id }) }, + ); + expect(res.status).toBe(201); + + const rows = await prisma.recommendationFeedback.findMany({ + where: { userId: user.id }, + }); + expect(rows).toHaveLength(1); + expect(rows[0].targetType).toBe("coach"); + expect(rows[0].helpful).toBe(true); + expect(rows[0].recommendationSeverity).toBe("coach"); + // Default prefs encode as warm/default — the user has no + // coachPrefsJson row so the route falls back to defaults. + expect(rows[0].metricSourceType).toBe("coach:tone=warm:verbosity=default"); + expect(rows[0].providerType).toBe("codex"); + expect(rows[0].promptVersion).toBe("4.23.0"); + }); + + it("returns 404 when the message belongs to another user", async () => { + const otherUser = await getPrismaClient().user.create({ + data: { username: "other-coach-h7" }, + }); + const otherMessage = await seedAssistantMessage(otherUser.id, "private"); + await seedSession("attacker-h7"); + + const { POST } = await import( + "@/app/api/insights/chat/messages/[id]/feedback/route" + ); + const res = await ( + POST as ( + req: Request, + ctx: { params: Promise<{ id: string }> }, + ) => Promise + )( + new Request( + `http://localhost/api/insights/chat/messages/${otherMessage.id}/feedback`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ rating: "helpful" }), + }, + ), + { params: Promise.resolve({ id: otherMessage.id }) }, + ); + expect(res.status).toBe(404); + }); + + it("rejects an unknown rating value with 422", async () => { + const user = await seedSession("coach-feedback-422"); + const message = await seedAssistantMessage(user.id, "Reply."); + + const { POST } = await import( + "@/app/api/insights/chat/messages/[id]/feedback/route" + ); + const res = await ( + POST as ( + req: Request, + ctx: { params: Promise<{ id: string }> }, + ) => Promise + )( + new Request( + `http://localhost/api/insights/chat/messages/${message.id}/feedback`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ rating: "meh" }), + }, + ), + { params: Promise.resolve({ id: message.id }) }, + ); + expect(res.status).toBe(422); + }); +});