Stop the Coach inventing a /learn link

The Coach may point at a published /learn guide, but only a slug in the
catalog. That was a prompt instruction with no enforcement — a model could
hallucinate /learn/lower-your-cortisol and ship a dead link. Add a deterministic
post-filter on the assembled reply that keeps a published slug verbatim and
strips any reference whose slug is not in the catalog, tidying the prose around
the removed link. Makes the catalog's impossible-by-construction claim an
enforced guarantee.
This commit is contained in:
Marc-André Bombeck committed 2026-06-26 13:38:31 +02:00
1 parent 976ecf8a75
commit d5cb1e4ef8
3 files changed
+186

No files matched your search

+23
View File
@@ -90,6 +90,7 @@ import {
findUnverifiedCoachNumbers,
stripUnverifiedNumbers,
} from "@/lib/ai/coach/coach-prose-grounding";
import { scrubUnknownLearnLinks } from "@/lib/ai/coach/learn-link-guard";
import { parseSuggestReminder } from "@/lib/ai/coach/suggest-reminder";
import { gateSuggestion } from "@/lib/ai/coach/suggest-gate";
import {
@@ -768,6 +769,28 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch
}
}
// v1.21.0 (NEW-C C-3) — Learn-link post-filter. The prompt instructs the
// model to only link a published `/learn/<slug>`, but that is guidance, not
// enforcement: a fabricated `/learn/<invented-slug>` would otherwise ship as
// a dead link. Scrub any reference whose slug is not in the catalog (a real
// one is kept verbatim). A blocked turn carries canned fallback prose with no
// links, so skip it.
if (!outbound.block && replyText.includes("/learn/")) {
const scrubbed = scrubUnknownLearnLinks(replyText);
if (scrubbed.dropped.length > 0) {
replyText = scrubbed.text;
annotate({
action: { name: "coach.learn.link_dropped" },
meta: {
dropped: scrubbed.dropped.length,
// Truncated slug tokens for ops triage — no user content.
slugs: scrubbed.dropped.slice(0, 6),
promptVersion: PROMPT_VERSION,
},
});
}
}
let surfacedSuggestion: CoachSuggestion | null = null;
if (!outbound.block && suggestParse.cadence) {
const decision = await gateSuggestion({
@@ -0,0 +1,63 @@
/**
* v1.21.0 (NEW-C C-3) — the Coach Learn-link post-filter. A published
* `/learn/<slug>` is kept verbatim; a fabricated one (a slug not in the
* catalog) is neutralised, so the catalog's "impossible by construction" claim
* becomes an enforced guarantee rather than a prompt promise.
*/
import { describe, expect, it } from "vitest";
import { scrubUnknownLearnLinks } from "@/lib/ai/coach/learn-link-guard";
import { LEARN_GUIDES } from "@/lib/ai/coach/learn-catalog";
const REAL = LEARN_GUIDES.find((g) => g.slug === "resting-heart-rate")!;
describe("scrubUnknownLearnLinks", () => {
it("returns the input unchanged when there is no /learn reference", () => {
const reply = "Your resting heart rate is settling nicely this week.";
const out = scrubUnknownLearnLinks(reply);
expect(out.text).toBe(reply);
expect(out.dropped).toEqual([]);
});
it("keeps a published /learn link verbatim (absolute URL)", () => {
const reply = `Worth a read — more on this: ${REAL.url}.`;
const out = scrubUnknownLearnLinks(reply);
expect(out.text).toContain(REAL.url);
expect(out.dropped).toEqual([]);
});
it("drops a fabricated /learn slug and reports it", () => {
const reply =
"That can drift with stress. More on this: https://healthlog.dev/learn/lower-your-cortisol-fast — give it a look.";
const out = scrubUnknownLearnLinks(reply);
expect(out.dropped).toContain("lower-your-cortisol-fast");
expect(out.text).not.toContain("lower-your-cortisol-fast");
expect(out.text).not.toContain("/learn/");
// The surrounding sentence survives.
expect(out.text).toContain("That can drift with stress");
expect(out.text).toContain("give it a look");
});
it("drops a fabricated relative /learn link", () => {
const reply = "See /learn/made-up-topic for the details.";
const out = scrubUnknownLearnLinks(reply);
expect(out.dropped).toContain("made-up-topic");
expect(out.text).not.toContain("/learn/");
});
it("keeps the real link and drops the fake one in the same reply", () => {
const reply = `Two reads: ${REAL.url} and https://healthlog.dev/learn/fictional-guide.`;
const out = scrubUnknownLearnLinks(reply);
expect(out.text).toContain(REAL.url);
expect(out.text).not.toContain("fictional-guide");
expect(out.dropped).toEqual(["fictional-guide"]);
});
it("is case-insensitive on the slug match", () => {
// A real slug in mixed case still resolves as published (kept).
const reply = "Read: https://healthlog.dev/learn/Resting-Heart-Rate.";
const out = scrubUnknownLearnLinks(reply);
expect(out.dropped).toEqual([]);
expect(out.text).toContain("Resting-Heart-Rate");
});
});
+100
View File
@@ -0,0 +1,100 @@
/**
* v1.21.0 (NEW-C C-3) — Coach Learn-link post-filter.
*
* The Coach MAY point the user at a public `/learn/<slug>` guide, but only a
* slug that appears in `learn-catalog.ts`. The system prompt instructs this
* ("Only link a URL from this list — never invent a /learn URL"), but that is a
* prompt instruction, not enforcement: a model could still hallucinate
* `/learn/lower-your-cortisol` and ship a dead/wrong link.
*
* This deterministic post-filter runs on the assembled reply before
* persistence and streaming. It scans for every `/learn/<slug>` reference
* (relative or the absolute `healthlog.dev/learn/<slug>` form) and NEUTRALISES
* any whose slug is not a published catalog entry — stripping the fabricated
* URL out of the prose while leaving the surrounding sentence intact. A real
* slug is kept verbatim. This converts the catalog's "impossible by
* construction" claim from a prompt promise into an enforced guarantee.
*
* Cheap + read-only: a single regex pass over the reply, the slug set sourced
* from the read-only `LEARN_GUIDES` catalog. No allocation on the common path
* (a reply with no `/learn` mention returns the input unchanged).
*/
import { LEARN_GUIDES } from "./learn-catalog";
/** The set of published `/learn` slugs, lower-cased for case-insensitive match. */
const KNOWN_SLUGS: ReadonlySet<string> = new Set(
LEARN_GUIDES.map((g) => g.slug.toLowerCase()),
);
/**
* Match a `/learn/<slug>` reference, with or without the host. Captures the
* slug so the caller can validate it. The slug is the URL-path segment after
* `/learn/` — letters, digits, and hyphens (the catalog's slug grammar). A
* trailing slash or punctuation ends the slug.
*
* Examples matched:
* https://healthlog.dev/learn/resting-heart-rate
* healthlog.dev/learn/made-up-slug
* /learn/another-fabrication
*/
const LEARN_LINK_RE =
/(?:https?:\/\/)?(?:[a-z0-9.-]*healthlog\.dev)?\/learn\/([a-z0-9-]+)\/?/gi;
export interface LearnLinkScrubResult {
/** The reply with every unknown `/learn/<slug>` reference removed. */
text: string;
/** The unknown slugs that were neutralised (lower-cased, deduped). */
dropped: string[];
}
/**
* Strip every `/learn/<slug>` reference whose slug is not in the published
* catalog. A known slug is preserved verbatim; an unknown one (a fabrication)
* is removed along with a common lead-in fragment ("more on this: ", "see "),
* so the prose does not read as a broken link. Returns the input unchanged when
* the reply contains no `/learn` reference.
*/
export function scrubUnknownLearnLinks(reply: string): LearnLinkScrubResult {
if (!reply || !reply.includes("/learn/")) {
return { text: reply, dropped: [] };
}
const dropped = new Set<string>();
// First pass: collect the unknown slugs (so the result is reportable) and
// build the cleaned string in one replace.
const text = reply.replace(LEARN_LINK_RE, (match, slug: string) => {
if (KNOWN_SLUGS.has(slug.toLowerCase())) {
// Published guide — keep the link exactly as written.
return match;
}
dropped.add(slug.toLowerCase());
// Drop the fabricated URL. Leave the surrounding prose; a dangling
// "more on this:" lead-in is cleaned by the post-pass below.
return "";
});
if (dropped.size === 0) {
// Every /learn link was valid — return the original (the replace above is a
// no-op clone, but be explicit about the contract).
return { text: reply, dropped: [] };
}
// Tidy the prose around a removed link: collapse a now-empty "more on this:"
// / "see:" lead-in and any double spaces / stranded punctuation the removal
// left behind. Conservative — only the well-known lead-in phrases the prompt
// example seeds, plus whitespace normalisation.
const tidied = text
// Drop a now-empty "more on this:" / "read more:" / "see:" lead-in (EN+DE).
.replace(
/\b(?:more on this|read more|see|learn more|mehr dazu|mehr hier)\s*:?\s*(?=[\s).,;!?]|$)/gi,
"",
)
// Collapse a doubled space + tidy a space-before-punctuation the gap left.
.replace(/[ \t]{2,}/g, " ")
.replace(/\s+([).,;!?])/g, "$1")
// Drop an empty parenthesis pair left by a parenthesised link.
.replace(/\(\s*\)/g, "")
.trim();
return { text: tidied, dropped: [...dropped] };
}