feat(coach): per-message thumbs feedback + admin aggregate view

The v1.4.22 prose rewrite shipped without a way for users to signal
whether the new tone is landing. v1.4.23 H7 closes that loop with a
light-touch thumbs row + an admin first-week dashboard.

Schema: extend `RecommendationFeedback` with a polymorphic
`target_type` column (default "recommendation" so legacy rows stay
representative) plus a nullable `reason` field. New index on
(target_type, created_at) so the aggregator can scan per-surface
buckets without a full sweep. Migration `0040_recommendation_feedback_target_type`.

Route: POST /api/insights/chat/messages/:id/feedback. Body
{ rating: "helpful" | "unhelpful", reason?: string }. Verifies
ownership (404 on cross-account messages — no existence leak),
decrypts the message body for the dedup key, and writes the row with
target_type="coach" + a metric-source string encoding the user's
active prefs (`coach:tone=warm:verbosity=default`). Per-user 60/h rate
limit. 409 on a P2002 (already_rated) so a double-click is a no-op.

Aggregator: new `buildCoachFeedbackBuckets()` slices Coach rows by
(promptVersion, tone, verbosity). The summary blob the worker writes
to AppSettings.adminAiInsightsFeedbackSummary now carries both
`buckets` (pre-existing) and `coachBuckets` (H7 addition). Targets
the same singleton row so no second worker handler is needed.

Admin view: `/admin/coach-feedback` (new section slug) renders the
Coach buckets as a small DataTable (prompt version × tone × verbosity
× helpful × not-helpful × n × helpful-rate). Helpful-rate column
tinted via Dracula tokens (green ≥80%, yellow ≥50%, orange below).
First useful question this answers: "Is the v1.4.22 prose rewrite
landing well, or did the warm tone overshoot?" If the helpful-rate
for PROMPT_VERSION 4.23.x drops below 50% within the first 100
ratings, v1.4.24 walks the persona back.

UI: `<ThumbsUp>` / `<ThumbsDown>` buttons (lucide icons, no emoji)
under each persisted assistant bubble. Streaming bubbles + refusals
+ error bubbles skip the row so users can't rate before the message
lands on disk. Localised "Thanks for the signal." after submit.

OpenAPI: spec carries the new POST route + the request schema.
Coverage now includes the H4 + H7 routes — three additions over
v1.4.22 baseline.

Tests +9 unit (aggregator Coach-bucket slicing, admin section
render-paths) plus an integration round-trip for the route.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Marc-André BombeckandClaude Opus 4.7 committed 2026-05-11 01:51:27 +02:00
1 parent 1faee95c84
commit 05c7f1474d
16 files changed
+1225 -4

No files matched your search

+79
View File
@@ -149,6 +149,49 @@ components:
- data
- error
type: object
CoachMessageFeedbackRequest:
description: Per-message helpful/unhelpful feedback (v1.4.23 H7). Optional `reason` is free-form prose, capped at 200 chars.
properties:
rating:
enum:
- helpful
- unhelpful
type: string
reason:
maxLength: 200
minLength: 1
type: string
required:
- rating
type: object
CoachMessageFeedbackResponse:
additionalProperties: false
properties:
data:
additionalProperties: false
properties:
createdAt:
format: date-time
pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$
type: string
id:
type: string
required:
- id
- createdAt
type: object
error:
type: "null"
meta:
additionalProperties: false
properties:
requestId:
type: string
type: object
required:
- data
- error
type: object
CreateMeasurementResponse:
additionalProperties: false
properties:
@@ -857,6 +900,42 @@ paths:
summary: Register native device + APNs token
tags:
- Devices
/api/insights/chat/messages/{id}/feedback:
post:
description: Persists a helpful/unhelpful rating for a single Coach reply. Reuses the v1.4.16 RecommendationFeedback
table via the polymorphic `targetType` column. The aggregator buckets ratings by (promptVersion, tone,
verbosity).
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/CoachMessageFeedbackRequest"
required: true
responses:
"201":
content:
application/json:
schema:
$ref: "#/components/schemas/CoachMessageFeedbackResponse"
description: Feedback saved.
"401": *a1
"404":
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
description: Message not found or not owned by the caller.
"409":
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorEnvelope"
description: Caller has already rated this message text.
"422": *a2
"429": *a3
summary: Rate a Coach assistant message
tags:
- Insights
/api/insights/comprehensive:
get:
description: Full Insights surface — daily briefing, recommendations with rationale, optional weekly report + storyboard
+23 -1
View File
@@ -919,7 +919,10 @@
"settingsEvidenceHint": "Die Belege-Box unter jeder Antwort automatisch öffnen.",
"settingsSave": "Speichern",
"settingsSaved": "Gespeichert.",
"settingsCancel": "Abbrechen"
"settingsCancel": "Abbrechen",
"feedbackHelpful": "Hilfreich",
"feedbackUnhelpful": "Nicht ganz",
"feedbackThanks": "Danke für das Signal."
},
"viewAll": "Alle ansehen",
"previewEmpty": "Noch keine Empfehlungen.",
@@ -1778,6 +1781,21 @@
"colNotHelpful": "Nicht hilfreich",
"colHelpfulRate": "Hilfreich-Quote"
},
"coachFeedback": {
"title": "Coach-Feedback",
"loading": "Coach-Feedback-Zusammenfassung wird geladen...",
"loadError": "Coach-Feedback-Zusammenfassung konnte nicht geladen werden. Bitte die Seite neu laden.",
"empty": "Noch kein Coach-Feedback. Der Aggregator läuft täglich um 04:00 — sobald Nutzer Coach-Antworten bewerten, erscheinen hier die Hilfreich-Quoten.",
"windowLabel": "Letzte {days} Tage",
"generatedAtLabel": "Erstellt",
"colPromptVersion": "Prompt-Version",
"colTone": "Tonfall",
"colVerbosity": "Ausführlichkeit",
"colHelpful": "Hilfreich",
"colNotHelpful": "Nicht hilfreich",
"colN": "n",
"colHelpfulRate": "Hilfreich-Quote"
},
"workerStatus": "Job-Worker",
"workerRunning": "Aktiv",
"workerStopped": "Gestoppt",
@@ -2024,6 +2042,10 @@
"title": "KI-Qualität",
"subtitle": "Hilfreichkeit nach Schweregrad und Provider — Nutzer-Feedback der letzten 30 Tage."
},
"coach-feedback": {
"title": "Coach-Feedback",
"subtitle": "Hilfreich-Quote pro Coach-Prompt-Version, Tonfall und Ausführlichkeit der letzten 30 Tage."
},
"feedback": {
"title": "Feedback",
"subtitle": "Feedback, Bug-Reports und Feature-Wünsche der Nutzer sichten."
+23 -1
View File
@@ -919,7 +919,10 @@
"settingsEvidenceHint": "Open the disclosure under every reply automatically.",
"settingsSave": "Save",
"settingsSaved": "Saved.",
"settingsCancel": "Cancel"
"settingsCancel": "Cancel",
"feedbackHelpful": "Helpful",
"feedbackUnhelpful": "Not quite",
"feedbackThanks": "Thanks for the signal."
},
"viewAll": "View all",
"previewEmpty": "No recommendations yet.",
@@ -1778,6 +1781,21 @@
"colNotHelpful": "Not helpful",
"colHelpfulRate": "Helpful rate"
},
"coachFeedback": {
"title": "Coach Feedback",
"loading": "Loading Coach feedback summary...",
"loadError": "Failed to load Coach feedback summary. Try refreshing the page.",
"empty": "No Coach feedback yet. The aggregator runs daily at 04:00 — once users rate Coach replies, helpful-rate bands appear here.",
"windowLabel": "Last {days} days",
"generatedAtLabel": "Generated",
"colPromptVersion": "Prompt version",
"colTone": "Tone",
"colVerbosity": "Verbosity",
"colHelpful": "Helpful",
"colNotHelpful": "Not helpful",
"colN": "n",
"colHelpfulRate": "Helpful rate"
},
"workerStatus": "Job Worker",
"workerRunning": "Running",
"workerStopped": "Stopped",
@@ -2024,6 +2042,10 @@
"title": "AI Quality",
"subtitle": "Helpfulness scores grouped by severity and provider, from the last 30 days of user feedback."
},
"coach-feedback": {
"title": "Coach Feedback",
"subtitle": "Helpful-rate per Coach prompt version, tone, and verbosity from the last 30 days."
},
"feedback": {
"title": "Feedback",
"subtitle": "Triage user feedback, bug reports, and feature requests."
@@ -0,0 +1,18 @@
-- v1.4.23 H7 — polymorphic feedback target.
--
-- Reuses the v1.4.16 B5e RecommendationFeedback table for Coach
-- assistant-message thumbs so the aggregator only has to slice one
-- table. `target_type` defaults to "recommendation" so every
-- pre-v1.4.23 row reads as the existing surface; new Coach rows write
-- "coach" + the message id in the existing `recommendation_id` slot.
-- `reason` is optional free-form prose (capped to 200 chars by the
-- route).
--
-- Index on (target_type, created_at) lets the admin aggregator scan
-- per-surface buckets without a full table sweep.
ALTER TABLE "recommendation_feedback"
ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'recommendation',
ADD COLUMN IF NOT EXISTS "reason" TEXT;
CREATE INDEX IF NOT EXISTS "recommendation_feedback_target_type_created_at_idx"
ON "recommendation_feedback" ("target_type", "created_at");
+13
View File
@@ -913,11 +913,24 @@ model RecommendationFeedback {
promptVersion String @map("prompt_version")
createdAt DateTime @default(now()) @map("created_at")
// v1.4.23 H7 — polymorphic target so Coach replies share the table.
// "recommendation" (default) is the v1.4.16 B5e shape; "coach" is a
// Coach assistant message. The aggregator slices on targetType to
// bucket each surface independently. Default value lets every
// pre-v1.4.23 row read as "recommendation" without a backfill.
targetType String @default("recommendation") @map("target_type")
// Optional free-form reason — populated when the user clicks
// "unhelpful" and selects a category from the small chip strip.
// Capped at 200 chars upstream so the table can't bloat from a
// pasted essay.
reason String? @map("reason")
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([userId, recommendationId, recommendationText], map: "rec_feedback_user_rec_text_key")
@@index([userId, createdAt])
@@index([recommendationSeverity, helpful, createdAt])
@@index([targetType, createdAt])
@@map("recommendation_feedback")
}
+10
View File
@@ -17,6 +17,7 @@ import type { JSX } from "react";
import { useAuth } from "@/hooks/use-auth";
import { useTranslations } from "@/lib/i18n/context";
import { AiQualitySection } from "@/components/admin/ai-quality-section";
import { CoachFeedbackSection } from "@/components/admin/coach-feedback-section";
import { ApiTokenOverviewSection } from "@/components/admin/api-token-overview-section";
import { AppLogPreviewSection } from "@/components/admin/app-log-preview-section";
import { BackupsSection } from "@/components/admin/backups-section";
@@ -93,6 +94,15 @@ export function AdminSectionRenderer({
<AiQualitySection />
</SectionFrame>
);
case "coach-feedback":
return (
<SectionFrame
title={t("admin.section.coach-feedback.title")}
subtitle={t("admin.section.coach-feedback.subtitle")}
>
<CoachFeedbackSection />
</SectionFrame>
);
case "feedback":
return (
<SectionFrame
@@ -0,0 +1,174 @@
/**
* v1.4.23 H7 — Coach assistant-message helpful/unhelpful feedback.
*
* POST /api/insights/chat/messages/:id/feedback
* Body: { rating: "helpful" | "unhelpful", reason?: string }
*
* Reuses the v1.4.16 B5e `RecommendationFeedback` table via the
* polymorphic `targetType` field added in migration
* `0040_recommendation_feedback_target_type`. Each Coach feedback row
* snapshots the user's active prompt-tuning prefs (tone + verbosity)
* into `metricSourceType` so the daily aggregator can bucket
* helpful-rate per (promptVersion, tone, verbosity) without a join.
*
* The route enforces ownership: the message id must belong to a
* conversation the authenticated user owns. Cross-account submissions
* return 404 (never 403 — same lesson as the conversation route, no
* cross-user existence leak).
*
* Idempotency: the existing unique
* `(userId, recommendationId, recommendationText)` index already
* dedupes a replay submission for the same message + same prose
* snapshot. A re-rating after the user changes their mind requires the
* UI to delete + re-create — out of scope for v1.4.23 H7.
*/
import type { NextRequest } from "next/server";
import { z } from "zod/v4";
import { apiHandler, requireAuth, HttpError } from "@/lib/api-handler";
import { apiError, apiSuccess, getClientIp, safeJson } from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { prisma } from "@/lib/db";
import { Prisma } from "@/generated/prisma/client";
import { resolveCoachFeedbackAttribution } from "@/lib/ai/feedback-attribution";
import { decrypt } from "@/lib/crypto";
import { annotate } from "@/lib/logging/context";
import { checkRateLimit } from "@/lib/rate-limit";
const coachMessageFeedbackSchema = z.object({
rating: z.enum(["helpful", "unhelpful"]),
reason: z.string().min(1).max(200).optional(),
});
interface RouteContext {
params: Promise<{ id: string }>;
}
async function handlePost(request: NextRequest, ctx: RouteContext) {
const { user } = await requireAuth();
const { id: messageId } = await ctx.params;
// Per-user rate limit — same shape as the recommendation feedback
// route (60/h). Keeps a chatty client from drowning the bucketing
// signal.
const rl = await checkRateLimit(
`coach-feedback:${user.id}`,
60,
60 * 60 * 1000,
);
if (!rl.allowed) {
annotate({
action: { name: "insights.coach.message.feedback" },
meta: { outcome: "rate_limited" },
});
return apiError("Too many feedback submissions, try again later", 429);
}
const { data: rawBody, error } = await safeJson<unknown>(request);
if (error) return error;
const parsed = coachMessageFeedbackSchema.safeParse(rawBody);
if (!parsed.success) {
annotate({
action: { name: "insights.coach.message.feedback" },
meta: { outcome: "validation_failed" },
});
throw new HttpError(422, "feedback.body.invalid");
}
const body = parsed.data;
// Verify ownership + pull the prose snapshot. The encrypted-content
// model stores the message body via AES-256-GCM; decrypt to use as
// the `recommendationText` slot so the per-(user, message, text)
// dedup remains stable across re-rates.
const message = await prisma.coachMessage.findFirst({
where: {
id: messageId,
role: "assistant",
conversation: { userId: user.id },
},
select: { id: true, encryptedContent: true },
});
if (!message) {
annotate({
action: { name: "insights.coach.message.feedback" },
meta: { outcome: "not_found" },
});
throw new HttpError(404, "coach.message.notFound");
}
let proseSnapshot = "";
try {
const ciphertext = Buffer.from(message.encryptedContent).toString("utf8");
proseSnapshot = decrypt(ciphertext);
} catch {
// Decryption failure shouldn't block the feedback row — fall back
// to a stable id-keyed marker so the dedup still works.
proseSnapshot = `<undecryptable:${messageId}>`;
}
// Cap the snapshot at the same byte budget the rec-feedback table
// already accepts (a Coach reply is usually ~400-1500 chars; clip
// hard at 4 KB so a runaway provider can't bloat the table).
if (proseSnapshot.length > 4096) {
proseSnapshot = proseSnapshot.slice(0, 4096);
}
const attribution = await resolveCoachFeedbackAttribution(user.id, messageId);
try {
const created = await prisma.recommendationFeedback.create({
data: {
userId: user.id,
recommendationId: messageId,
recommendationText: proseSnapshot,
recommendationSeverity: "coach",
metricSourceType: attribution.metricSourceType,
metricSourceTimeRange: "single_message",
helpful: body.rating === "helpful",
providerType: attribution.providerType,
promptVersion: attribution.promptVersion,
targetType: "coach",
reason: body.reason ?? null,
},
select: { id: true, createdAt: true },
});
await auditLog("insights.coach.message.feedback", {
userId: user.id,
ipAddress: getClientIp(request),
details: {
feedbackId: created.id,
messageId,
rating: body.rating,
providerType: attribution.providerType,
promptVersion: attribution.promptVersion,
metricSourceType: attribution.metricSourceType,
},
});
annotate({
action: { name: "insights.coach.message.feedback" },
meta: {
outcome: "created",
rating: body.rating,
providerType: attribution.providerType,
promptVersion: attribution.promptVersion,
},
});
return apiSuccess({ id: created.id, createdAt: created.createdAt }, 201);
} catch (err) {
if (
err instanceof Prisma.PrismaClientKnownRequestError &&
err.code === "P2002"
) {
annotate({
action: { name: "insights.coach.message.feedback" },
meta: { outcome: "already_rated" },
});
return apiError("already_rated", 409);
}
throw err;
}
}
export const POST = apiHandler(handlePost);
@@ -0,0 +1,110 @@
import { describe, expect, it, vi } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
/**
* v1.4.23 H7 — admin Coach feedback section.
*
* Three render paths matter:
* 1. Loading — spinner + "Loading Coach feedback summary..."
* 2. Empty — quiet message when the aggregator hasn't run or no
* Coach rows exist in the rolling window.
* 3. Populated — table renders one row per (promptVersion, tone,
* verbosity) bucket with helpful-rate column tinted by band.
*/
const useQueryMock = vi.fn();
vi.mock("@tanstack/react-query", () => ({
useQuery: (...args: unknown[]) => useQueryMock(...args),
}));
import { I18nProvider } from "@/lib/i18n/context";
import { CoachFeedbackSection } from "../coach-feedback-section";
function render() {
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<CoachFeedbackSection />
</I18nProvider>,
);
}
describe("CoachFeedbackSection", () => {
it("renders the loading state when the query is pending", () => {
useQueryMock.mockReturnValue({
isLoading: true,
isError: false,
data: null,
});
const html = render();
expect(html).toContain("Loading Coach feedback summary");
expect(html).toContain("animate-spin");
});
it("renders the empty state when the summary has no Coach buckets", () => {
useQueryMock.mockReturnValue({
isLoading: false,
isError: false,
data: {
generatedAt: "2026-05-10T04:00:00Z",
windowDays: 30,
coachBuckets: [],
},
});
const html = render();
expect(html).toContain("No Coach feedback yet");
expect(html).toContain("Coach Feedback");
});
it("renders one row per bucket with helpful-rate tint", () => {
useQueryMock.mockReturnValue({
isLoading: false,
isError: false,
data: {
generatedAt: "2026-05-10T04:00:00Z",
windowDays: 30,
coachBuckets: [
{
promptVersion: "4.23.0",
tone: "warm",
verbosity: "default",
helpful: 12,
notHelpful: 3,
total: 15,
helpfulRate: 0.8,
},
{
promptVersion: "4.23.0",
tone: "concise",
verbosity: "brief",
helpful: 4,
notHelpful: 4,
total: 8,
helpfulRate: 0.5,
},
],
},
});
const html = render();
// Two bucket rows in the table.
const matches = (html.match(/coach-feedback-bucket/g) ?? []).length;
expect(matches).toBe(2);
// Helpful-rate column carries the band tint.
expect(html).toContain("text-dracula-green"); // 80% bucket
expect(html).toContain("text-dracula-yellow"); // 50% bucket
// Numbers visible in the markup.
expect(html).toContain("80%");
expect(html).toContain("50%");
});
it("renders the load-error path when the fetch failed", () => {
useQueryMock.mockReturnValue({
isLoading: false,
isError: true,
data: null,
});
const html = render();
expect(html).toContain("Failed to load Coach feedback summary");
expect(html).toContain('role="alert"');
});
});
@@ -0,0 +1,188 @@
"use client";
/**
* `<CoachFeedbackSection>` — admin view onto the v1.4.23 H7 Coach
* helpful/unhelpful aggregate. Reads the same feedback-aggregator row
* that the AI Quality view consumes; this section just slices the
* Coach-only buckets so the operator can see prompt-version × helpful-
* rate × n at a glance.
*
* First useful question this answers in the v1.4.23 first-week window:
* "Is the v1.4.22 Coach prose rewrite (PROMPT_VERSION 4.22.0 → 4.23.x)
* landing well, or did the warm tone overshoot?" If the helpful-rate
* for the new prompt version drops below 50 % within the first 100
* ratings, v1.4.24 walks the persona back.
*
* Empty state when the aggregator hasn't run or no Coach feedback has
* landed in the rolling window. Cross-user aggregation only — the
* admin view never exposes per-user prose.
*/
import { useQuery } from "@tanstack/react-query";
import { Loader2, Sparkles } from "lucide-react";
import { useTranslations, useFormatters } from "@/lib/i18n/context";
interface CoachFeedbackBucket {
promptVersion: string;
tone: string;
verbosity: string;
helpful: number;
notHelpful: number;
total: number;
helpfulRate: number;
}
interface FeedbackSummary {
generatedAt: string;
windowDays: number;
coachBuckets?: CoachFeedbackBucket[];
}
interface CoachFeedbackResponse {
data: { summary: FeedbackSummary | null } | null;
error?: string | null;
}
function helpfulRateColour(rate: number): string {
if (rate >= 0.8) return "text-dracula-green";
if (rate >= 0.5) return "text-dracula-yellow";
return "text-dracula-orange";
}
export function CoachFeedbackSection() {
const { t } = useTranslations();
const fmt = useFormatters();
const query = useQuery({
queryKey: ["admin", "coach-feedback"],
queryFn: async () => {
const res = await fetch("/api/admin/ai-quality");
const json = (await res.json()) as CoachFeedbackResponse;
if (!res.ok) throw new Error(json.error ?? "coach_feedback_failed");
return json.data?.summary ?? null;
},
});
if (query.isLoading) {
return (
<div className="bg-card border-border flex items-center gap-2 rounded-xl border p-6">
<Loader2 className="h-4 w-4 animate-spin" />
<span className="text-muted-foreground text-sm">
{t("admin.coachFeedback.loading")}
</span>
</div>
);
}
if (query.isError) {
return (
<div
role="alert"
className="text-destructive bg-destructive/10 border-destructive/30 rounded-xl border p-4 text-sm"
>
{t("admin.coachFeedback.loadError")}
</div>
);
}
const summary = query.data;
const buckets = summary?.coachBuckets ?? [];
if (!summary || buckets.length === 0) {
return (
<div className="bg-card border-border rounded-xl border p-6">
<div className="flex items-center gap-2">
<Sparkles className="text-dracula-purple h-5 w-5" />
<div className="text-lg font-semibold">
{t("admin.coachFeedback.title")}
</div>
</div>
<p className="text-muted-foreground mt-3 text-sm">
{t("admin.coachFeedback.empty")}
</p>
</div>
);
}
return (
<div className="bg-card border-border space-y-4 rounded-xl border p-6">
<div className="flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
<Sparkles className="text-dracula-purple h-5 w-5" />
<div className="text-lg font-semibold">
{t("admin.coachFeedback.title")}
</div>
</div>
<p className="text-muted-foreground text-xs">
{t("admin.coachFeedback.windowLabel", {
days: String(summary.windowDays),
})}
{" · "}
{t("admin.coachFeedback.generatedAtLabel")}{" "}
{fmt.dateTime(summary.generatedAt)}
</p>
</div>
<div className="overflow-x-auto">
<table className="w-full text-sm" data-slot="coach-feedback-table">
<thead>
<tr className="text-muted-foreground border-border border-b text-left text-xs tracking-wider uppercase">
<th className="py-2 pr-3 font-medium">
{t("admin.coachFeedback.colPromptVersion")}
</th>
<th className="py-2 pr-3 font-medium">
{t("admin.coachFeedback.colTone")}
</th>
<th className="py-2 pr-3 font-medium">
{t("admin.coachFeedback.colVerbosity")}
</th>
<th className="py-2 pr-3 font-medium tabular-nums">
{t("admin.coachFeedback.colHelpful")}
</th>
<th className="py-2 pr-3 font-medium tabular-nums">
{t("admin.coachFeedback.colNotHelpful")}
</th>
<th className="py-2 pr-3 font-medium tabular-nums">
{t("admin.coachFeedback.colN")}
</th>
<th className="py-2 font-medium tabular-nums">
{t("admin.coachFeedback.colHelpfulRate")}
</th>
</tr>
</thead>
<tbody>
{buckets.map((bucket, idx) => {
const ratePct = Math.round(bucket.helpfulRate * 100);
return (
<tr
key={`${bucket.promptVersion}-${bucket.tone}-${bucket.verbosity}-${idx}`}
className="border-border/40 border-b last:border-0"
data-slot="coach-feedback-bucket"
>
<td className="text-muted-foreground py-2 pr-3 text-xs">
{bucket.promptVersion}
</td>
<td className="py-2 pr-3">{bucket.tone}</td>
<td className="py-2 pr-3">{bucket.verbosity}</td>
<td className="py-2 pr-3 tabular-nums">{bucket.helpful}</td>
<td className="py-2 pr-3 tabular-nums">
{bucket.notHelpful}
</td>
<td className="py-2 pr-3 tabular-nums">{bucket.total}</td>
<td
className={`py-2 font-semibold tabular-nums ${helpfulRateColour(
bucket.helpfulRate,
)}`}
>
{ratePct}%
</td>
</tr>
);
})}
</tbody>
</table>
</div>
</div>
);
}
+1
View File
@@ -15,6 +15,7 @@ export const ADMIN_SECTION_SLUGS = [
"services",
"integrations",
"ai-quality",
"coach-feedback",
"feedback",
"reminders",
"users",
@@ -1,8 +1,15 @@
"use client";
import { useEffect, useRef, useState } from "react";
import { useQuery } from "@tanstack/react-query";
import { Bot, ChevronRight, Sparkles, User } from "lucide-react";
import { useMutation, useQuery } from "@tanstack/react-query";
import {
Bot,
ChevronRight,
Sparkles,
ThumbsDown,
ThumbsUp,
User,
} from "lucide-react";
import { cn } from "@/lib/utils";
import { useTranslations } from "@/lib/i18n/context";
@@ -180,6 +187,7 @@ export function MessageThread({
metricSource={m.metricSource}
providerType={m.providerType}
evidenceDefaultOpen={evidenceDefaultOpen}
messageId={m.id}
/>
);
})}
@@ -228,6 +236,12 @@ interface ChatBubbleProps {
errorCode?: string | null;
/** v1.4.23 H4 — when true the evidence `<details>` mounts open. */
evidenceDefaultOpen?: boolean;
/**
* v1.4.23 H7 — present only on persisted assistant messages.
* Streaming bubbles (no message id yet) skip the thumbs row so the
* user can't rate before the message lands on disk.
*/
messageId?: string;
}
function ChatBubble({
@@ -238,6 +252,7 @@ function ChatBubble({
inProgress,
errorCode,
evidenceDefaultOpen,
messageId,
}: ChatBubbleProps) {
const { t } = useTranslations();
const { user } = useAuth();
@@ -383,7 +398,87 @@ function ChatBubble({
</ul>
</details>
)}
{/* v1.4.23 H7 — per-message thumbs feedback. Only persisted
assistant messages get the row (skipped for refusals,
errors, in-flight stream bubbles). The aggregator buckets
the rating by (promptVersion, tone, verbosity). */}
{messageId &&
!inProgress &&
!errorCode &&
providerType !== "refusal" && (
<CoachMessageFeedback messageId={messageId} />
)}
</div>
</div>
);
}
interface CoachMessageFeedbackProps {
messageId: string;
}
function CoachMessageFeedback({ messageId }: CoachMessageFeedbackProps) {
const { t } = useTranslations();
const [submittedRating, setSubmittedRating] = useState<
"helpful" | "unhelpful" | null
>(null);
const submit = useMutation({
mutationFn: async (rating: "helpful" | "unhelpful") => {
const res = await fetch(
`/api/insights/chat/messages/${messageId}/feedback`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ rating }),
},
);
// Treat 409 (already_rated) as a successful no-op so the user
// never sees an error toast for double-clicking the same chip.
if (!res.ok && res.status !== 409) {
throw new Error("coach-feedback.failed");
}
return rating;
},
onSuccess: (rating) => setSubmittedRating(rating),
});
if (submittedRating) {
return (
<p
data-slot="coach-message-feedback-thanks"
className="text-muted-foreground text-[10px]"
>
{t("insights.coach.feedbackThanks")}
</p>
);
}
return (
<div
data-slot="coach-message-feedback"
className="flex items-center gap-2"
>
<button
type="button"
data-slot="coach-message-feedback-helpful"
onClick={() => submit.mutate("helpful")}
disabled={submit.isPending}
className="text-muted-foreground hover:text-dracula-green focus-visible:ring-ring/50 inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[11px] outline-none focus-visible:ring-2 disabled:opacity-50"
>
<ThumbsUp className="size-3" aria-hidden="true" />
{t("insights.coach.feedbackHelpful")}
</button>
<button
type="button"
data-slot="coach-message-feedback-unhelpful"
onClick={() => submit.mutate("unhelpful")}
disabled={submit.isPending}
className="text-muted-foreground hover:text-dracula-orange focus-visible:ring-ring/50 inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[11px] outline-none focus-visible:ring-2 disabled:opacity-50"
>
<ThumbsDown className="size-3" aria-hidden="true" />
{t("insights.coach.feedbackUnhelpful")}
</button>
</div>
);
}
+62
View File
@@ -81,3 +81,65 @@ export async function resolveFeedbackAttribution(
promptVersion: PROMPT_VERSION,
};
}
/**
* v1.4.23 H7 — attribution shape for a Coach assistant-message
* thumbs row. The polymorphic `RecommendationFeedback` table reuses
* its existing slots for Coach rows:
* - `recommendationId` carries the Coach message id
* - `recommendationText` carries the assistant prose snapshot
* - `recommendationSeverity` is always "coach" so dashboards can
* filter / group on the literal
* - `metricSourceType` encodes the user's active prefs at the
* moment of the rating: `coach:tone=warm:verbosity=default`. The
* aggregator buckets on this field so a future audit can slice
* helpful-rate per (tone, verbosity) without a join.
* - `metricSourceTimeRange` is always "single_message"
* - `target_type` is "coach"
*/
export interface CoachFeedbackAttribution extends FeedbackAttribution {
metricSourceType: string;
}
export function buildCoachMetricSourceType(
tone: string,
verbosity: string,
): string {
return `coach:tone=${tone}:verbosity=${verbosity}`;
}
/**
* Resolve the (providerType, promptVersion, metricSourceType) tuple
* for a Coach feedback row. Reads the user's latest persisted
* Coach assistant message to recover the providerType + promptVersion
* stamped on it, and queries the user's active `coachPrefsJson` for
* the tone/verbosity slice. Falls back to safe defaults when either
* lookup misses.
*/
export async function resolveCoachFeedbackAttribution(
userId: string,
messageId: string,
): Promise<CoachFeedbackAttribution> {
const [message, userRow] = await Promise.all([
prisma.coachMessage.findFirst({
where: { id: messageId, conversation: { userId } },
select: { providerType: true, promptVersion: true },
}),
prisma.user.findUnique({
where: { id: userId },
select: { coachPrefsJson: true },
}),
]);
// Lazy-import to avoid circular imports — coach-prefs validation
// doesn't depend on attribution but the inverse would create a cycle.
const { parseCoachPrefs } = await import("@/lib/validations/coach-prefs");
const prefs = parseCoachPrefs(userRow?.coachPrefsJson);
return {
providerType:
message?.providerType ?? FEEDBACK_ATTRIBUTION_FALLBACK.providerType,
promptVersion: message?.promptVersion ?? PROMPT_VERSION,
metricSourceType: buildCoachMetricSourceType(prefs.tone, prefs.verbosity),
};
}
@@ -2,6 +2,7 @@ import { describe, it, expect, vi } from "vitest";
import type { PrismaClient } from "@/generated/prisma/client";
import {
aggregateRecommendationFeedback,
buildCoachFeedbackBuckets,
buildFeedbackBuckets,
DEFAULT_FEEDBACK_AGGREGATION_WINDOW_DAYS,
} from "../feedback-aggregator";
@@ -12,6 +13,7 @@ interface FeedbackRow {
providerType: string;
promptVersion: string;
helpful: boolean;
targetType?: string;
}
function makePrismaMock(rows: FeedbackRow[]) {
@@ -165,5 +167,98 @@ describe("aggregateRecommendationFeedback", () => {
expect(upsertCall.update.adminAiInsightsFeedbackSummary.buckets).toEqual(
[],
);
// v1.4.23 H7 — coachBuckets always present, empty by default.
expect(
upsertCall.update.adminAiInsightsFeedbackSummary.coachBuckets,
).toEqual([]);
});
});
/**
* v1.4.23 H7 — Coach-only bucket view. The aggregator slices the
* unified RecommendationFeedback rows on `targetType === "coach"` and
* groups by (promptVersion, tone, verbosity).
*/
describe("buildCoachFeedbackBuckets", () => {
it("ignores non-Coach rows", () => {
const rows: FeedbackRow[] = [
{
recommendationSeverity: "important",
metricSourceType: "bloodPressure",
providerType: "codex",
promptVersion: "4.16.0",
helpful: true,
targetType: "recommendation",
},
];
expect(buildCoachFeedbackBuckets(rows)).toEqual([]);
});
it("groups Coach rows by (promptVersion, tone, verbosity)", () => {
const rows: FeedbackRow[] = [
{
recommendationSeverity: "coach",
metricSourceType: "coach:tone=warm:verbosity=default",
providerType: "codex",
promptVersion: "4.23.0",
helpful: true,
targetType: "coach",
},
{
recommendationSeverity: "coach",
metricSourceType: "coach:tone=warm:verbosity=default",
providerType: "codex",
promptVersion: "4.23.0",
helpful: true,
targetType: "coach",
},
{
recommendationSeverity: "coach",
metricSourceType: "coach:tone=warm:verbosity=default",
providerType: "codex",
promptVersion: "4.23.0",
helpful: false,
targetType: "coach",
},
{
recommendationSeverity: "coach",
metricSourceType: "coach:tone=concise:verbosity=brief",
providerType: "codex",
promptVersion: "4.23.0",
helpful: true,
targetType: "coach",
},
];
const buckets = buildCoachFeedbackBuckets(rows);
expect(buckets).toHaveLength(2);
const warm = buckets.find(
(b) => b.tone === "warm" && b.verbosity === "default",
);
expect(warm).toBeDefined();
expect(warm?.helpful).toBe(2);
expect(warm?.notHelpful).toBe(1);
expect(warm?.helpfulRate).toBeCloseTo(0.67, 2);
const concise = buckets.find(
(b) => b.tone === "concise" && b.verbosity === "brief",
);
expect(concise?.total).toBe(1);
expect(concise?.helpfulRate).toBe(1);
});
it("falls back to tone=unknown when metricSourceType doesn't match the encoded shape", () => {
const rows: FeedbackRow[] = [
{
recommendationSeverity: "coach",
metricSourceType: "weird-shape",
providerType: "codex",
promptVersion: "4.23.0",
helpful: false,
targetType: "coach",
},
];
const buckets = buildCoachFeedbackBuckets(rows);
expect(buckets).toHaveLength(1);
expect(buckets[0].tone).toBe("unknown");
expect(buckets[0].verbosity).toBe("unknown");
});
});
+105
View File
@@ -35,12 +35,41 @@ export interface FeedbackBucket {
total: number;
/** Fraction in [0..1], rounded to 2 decimals. Always 0 when total=0. */
helpfulRate: number;
/**
* v1.4.23 H7 — surface origin so the admin dashboard can group
* recommendation rows separately from Coach assistant-message rows
* (the existing buckets blended both before this field landed).
*/
targetType: string;
}
/**
* v1.4.23 H7 — slim view onto the Coach-only rows so the admin
* coach-feedback dashboard can render the first-week aggregate without
* re-deriving the slice from the recommendation buckets.
*/
export interface CoachFeedbackBucket {
promptVersion: string;
/** Encoded as `tone=warm|neutral|concise` */
tone: string;
/** Encoded as `verbosity=brief|default|detailed` */
verbosity: string;
helpful: number;
notHelpful: number;
total: number;
helpfulRate: number;
}
export interface FeedbackAggregationSummary {
generatedAt: string;
windowDays: number;
buckets: FeedbackBucket[];
/**
* v1.4.23 H7 — Coach assistant-message buckets sliced by
* (promptVersion, tone, verbosity). Empty array when the
* aggregation window has no Coach feedback rows.
*/
coachBuckets: CoachFeedbackBucket[];
}
interface MinimalFeedbackRow {
@@ -49,6 +78,13 @@ interface MinimalFeedbackRow {
providerType: string;
promptVersion: string;
helpful: boolean;
/**
* v1.4.23 H7 — present on every row from the new schema. Defaults to
* "recommendation" for legacy rows so the bucketing still works
* during the upgrade window when the migration has run but the route
* hasn't started writing the field yet.
*/
targetType?: string;
}
/**
@@ -67,7 +103,9 @@ export function buildFeedbackBuckets(
): FeedbackBucket[] {
const map = new Map<string, FeedbackBucket>();
for (const row of rows) {
const targetType = row.targetType ?? "recommendation";
const key = [
targetType,
row.recommendationSeverity,
row.metricSourceType,
row.providerType,
@@ -88,6 +126,7 @@ export function buildFeedbackBuckets(
notHelpful: row.helpful ? 0 : 1,
total: 1,
helpfulRate: 0,
targetType,
});
}
}
@@ -100,6 +139,7 @@ export function buildFeedbackBuckets(
buckets.sort((a, b) => {
return (
a.targetType.localeCompare(b.targetType) ||
a.severity.localeCompare(b.severity) ||
a.metricSourceType.localeCompare(b.metricSourceType) ||
a.providerType.localeCompare(b.providerType) ||
@@ -110,6 +150,69 @@ export function buildFeedbackBuckets(
return buckets;
}
/**
* v1.4.23 H7 — derive the Coach-only bucket view from the unified
* recommendation feedback rows. The Coach route writes the user's
* active prefs into `metricSourceType` as
* `coach:tone=<x>:verbosity=<y>`; this helper parses that back out so
* the admin dashboard can group on the typed dimensions without
* re-running a regex per row.
*
* Rows whose `targetType !== "coach"` are ignored. Rows whose
* `metricSourceType` does not match the encoded shape land under
* `tone=unknown:verbosity=unknown` so an early-write pre-migration row
* still contributes to a bucket rather than silently dropping.
*/
export function buildCoachFeedbackBuckets(
rows: MinimalFeedbackRow[],
): CoachFeedbackBucket[] {
const map = new Map<string, CoachFeedbackBucket>();
for (const row of rows) {
if ((row.targetType ?? "recommendation") !== "coach") continue;
const { tone, verbosity } = parseCoachMetricSource(row.metricSourceType);
const key = `${row.promptVersion}|${tone}|${verbosity}`;
const existing = map.get(key);
if (existing) {
if (row.helpful) existing.helpful += 1;
else existing.notHelpful += 1;
existing.total += 1;
} else {
map.set(key, {
promptVersion: row.promptVersion,
tone,
verbosity,
helpful: row.helpful ? 1 : 0,
notHelpful: row.helpful ? 0 : 1,
total: 1,
helpfulRate: 0,
});
}
}
const buckets = [...map.values()];
for (const b of buckets) {
b.helpfulRate =
b.total === 0 ? 0 : Math.round((b.helpful / b.total) * 100) / 100;
}
buckets.sort(
(a, b) =>
a.promptVersion.localeCompare(b.promptVersion) ||
a.tone.localeCompare(b.tone) ||
a.verbosity.localeCompare(b.verbosity),
);
return buckets;
}
const COACH_METRIC_SOURCE_RE = /^coach:tone=([^:]+):verbosity=([^:]+)$/;
function parseCoachMetricSource(metricSourceType: string): {
tone: string;
verbosity: string;
} {
const match = metricSourceType.match(COACH_METRIC_SOURCE_RE);
if (!match) return { tone: "unknown", verbosity: "unknown" };
return { tone: match[1], verbosity: match[2] };
}
export interface AggregateOptions {
/** Override "now" for deterministic tests. Defaults to `new Date()`. */
now?: Date;
@@ -139,6 +242,7 @@ export async function aggregateRecommendationFeedback(
providerType: true,
promptVersion: true,
helpful: true,
targetType: true,
},
});
@@ -146,6 +250,7 @@ export async function aggregateRecommendationFeedback(
generatedAt: now.toISOString(),
windowDays,
buckets: buildFeedbackBuckets(rows),
coachBuckets: buildCoachFeedbackBuckets(rows),
};
// The singleton AppSettings row is auto-created on first install
+50
View File
@@ -103,6 +103,17 @@ coachPrefsSchema.meta({
"Per-user Coach prompt-tuning preferences (v1.4.23 H4). All fields default to the legacy v1.4.22 behaviour when omitted.",
});
const coachMessageFeedbackBody = z
.object({
rating: z.enum(["helpful", "unhelpful"]),
reason: z.string().min(1).max(200).optional(),
})
.meta({
id: "CoachMessageFeedbackRequest",
description:
"Per-message helpful/unhelpful feedback (v1.4.23 H7). Optional `reason` is free-form prose, capped at 200 chars.",
});
// ── Sub-schemas owned here (route-specific shapes) ───────────────────
const passkeyLoginVerifyRequest = z
@@ -526,6 +537,45 @@ export const openApiPaths: NonNullable<ZodOpenApiObject["paths"]> = {
},
},
},
"/api/insights/chat/messages/{id}/feedback": {
post: {
tags: ["Insights"],
summary: "Rate a Coach assistant message",
description:
"Persists a helpful/unhelpful rating for a single Coach reply. Reuses the v1.4.16 RecommendationFeedback table via the polymorphic `targetType` column. The aggregator buckets ratings by (promptVersion, tone, verbosity).",
requestBody: {
required: true,
content: {
"application/json": { schema: coachMessageFeedbackBody },
},
},
responses: {
"201": {
description: "Feedback saved.",
content: {
"application/json": {
schema: dataEnvelope(
z.object({
id: z.string(),
createdAt: z.iso.datetime({ offset: true }),
}),
"CoachMessageFeedbackResponse",
),
},
},
},
"404": {
description: "Message not found or not owned by the caller.",
content: { "application/json": { schema: errorEnvelope } },
},
"409": {
description: "Caller has already rated this message text.",
content: { "application/json": { schema: errorEnvelope } },
},
...stdResponses,
},
},
},
"/api/insights/comprehensive": {
get: {
tags: ["Insights"],
+177
View File
@@ -0,0 +1,177 @@
/**
* v1.4.23 H7 — Coach assistant-message thumbs feedback route.
*
* Round-trip: seed a session + a coach conversation + an assistant
* message, POST a thumbs rating, assert a `RecommendationFeedback` row
* landed with `targetType="coach"` and the encoded prefs metric source.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { encrypt } from "@/lib/crypto";
import { cookieJar, headerJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => {
cookieJar.set(name, value);
},
delete: (name: string) => {
cookieJar.delete(name);
},
})),
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
cookieJar.clear();
headerJar.clear();
});
afterEach(() => {
vi.restoreAllMocks();
});
async function seedSession(username: string) {
const prisma = getPrismaClient();
const user = await prisma.user.create({
data: { username, email: `${username}@example.test` },
});
const session = await prisma.session.create({
data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) },
});
cookieJar.set("healthlog_session", session.id);
return user;
}
async function seedAssistantMessage(userId: string, content: string) {
const prisma = getPrismaClient();
const conversation = await prisma.coachConversation.create({
data: { userId, title: "Test conversation" },
});
const ciphertext = encrypt(content);
return prisma.coachMessage.create({
data: {
conversationId: conversation.id,
role: "assistant",
encryptedContent: Buffer.from(ciphertext, "utf8"),
providerType: "codex",
promptVersion: "4.23.0",
},
});
}
describe("POST /api/insights/chat/messages/:id/feedback", () => {
it("persists a thumbs-up with target_type=coach and the encoded metric source", async () => {
const prisma = getPrismaClient();
const user = await seedSession("coach-feedback-h7");
const message = await seedAssistantMessage(user.id, "Your last week...");
const { POST } = await import(
"@/app/api/insights/chat/messages/[id]/feedback/route"
);
const res = await (
POST as (
req: Request,
ctx: { params: Promise<{ id: string }> },
) => Promise<Response>
)(
new Request(
`http://localhost/api/insights/chat/messages/${message.id}/feedback`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ rating: "helpful" }),
},
),
{ params: Promise.resolve({ id: message.id }) },
);
expect(res.status).toBe(201);
const rows = await prisma.recommendationFeedback.findMany({
where: { userId: user.id },
});
expect(rows).toHaveLength(1);
expect(rows[0].targetType).toBe("coach");
expect(rows[0].helpful).toBe(true);
expect(rows[0].recommendationSeverity).toBe("coach");
// Default prefs encode as warm/default — the user has no
// coachPrefsJson row so the route falls back to defaults.
expect(rows[0].metricSourceType).toBe("coach:tone=warm:verbosity=default");
expect(rows[0].providerType).toBe("codex");
expect(rows[0].promptVersion).toBe("4.23.0");
});
it("returns 404 when the message belongs to another user", async () => {
const otherUser = await getPrismaClient().user.create({
data: { username: "other-coach-h7" },
});
const otherMessage = await seedAssistantMessage(otherUser.id, "private");
await seedSession("attacker-h7");
const { POST } = await import(
"@/app/api/insights/chat/messages/[id]/feedback/route"
);
const res = await (
POST as (
req: Request,
ctx: { params: Promise<{ id: string }> },
) => Promise<Response>
)(
new Request(
`http://localhost/api/insights/chat/messages/${otherMessage.id}/feedback`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ rating: "helpful" }),
},
),
{ params: Promise.resolve({ id: otherMessage.id }) },
);
expect(res.status).toBe(404);
});
it("rejects an unknown rating value with 422", async () => {
const user = await seedSession("coach-feedback-422");
const message = await seedAssistantMessage(user.id, "Reply.");
const { POST } = await import(
"@/app/api/insights/chat/messages/[id]/feedback/route"
);
const res = await (
POST as (
req: Request,
ctx: { params: Promise<{ id: string }> },
) => Promise<Response>
)(
new Request(
`http://localhost/api/insights/chat/messages/${message.id}/feedback`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ rating: "meh" }),
},
),
{ params: Promise.resolve({ id: message.id }) },
);
expect(res.status).toBe(422);
});
});