test(insights): pin the period-narrative and coach output-language contract

Three layers. The German and English narrative prompts are frozen by content
hash, normalised for the prompt-version string that a prompt change is
required to move — any other drift fails. The four riding locales must name
their language, carry the verbatim reply-language directive from their own
safety-contract file as the last instruction, and contain no German-body
sentinel. A generator-level test drives a French run end to end and asserts
the row is keyed fr, the cache action is locale-scoped, and the system prompt
reaches the provider in English with the French directive.

Alongside it a source-level ban on the exact shape the bug took
(`=== "en" ? "en" : "de"`, `?? "de"`) across the coach and narrative
surfaces, with a self-check so a typo in the pattern cannot turn the guard
into a silent pass, and coach-memory recall cases for all four locales.
This commit is contained in:
Marc-André Bombeck committed 2026-07-18 23:32:49 +02:00
1 parent e4ec0eec49
commit 93738ae968
4 files changed
+307 -1

No files matched your search

@@ -0,0 +1,76 @@
/**
* Regression-class ban for the German-default locale binary.
*
* The bug this pins had exactly one shape: `locale === "en" ? "en" : "de"`
* (and its `=== "de" ? ... : ...` inversions with a `de` fallback). Every
* occurrence sent French, Spanish, Italian and Polish accounts down the German
* branch. This walks the source of the surfaces that carried it and fails on
* the shape itself, so a future edit cannot quietly reintroduce it.
*
* Scoped to the files this change owns — the coach memory refresh, the coach
* snapshot's narrative recall, the narrative route + generator + warm, and the
* dashboard briefing recall. Blunt by design: it matches text, not behaviour,
* which is precisely what makes it survive a refactor of the logic around it.
*/
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
const ROOT = resolve(__dirname, "../../../..");
const GUARDED_FILES = [
"app/api/insights/chat/route.ts",
"app/api/insights/narrative/route.ts",
"lib/ai/coach/snapshot.ts",
"lib/ai/coach/memory-snapshot.ts",
"lib/ai/coach/coach-memory-refresh-worker.ts",
"lib/dashboard/snapshot.ts",
"lib/insights/narrative/period-narrative-generate.ts",
"lib/jobs/period-narrative-warm.ts",
"lib/jobs/period-narrative-shared.ts",
];
/**
* A ternary whose FALSE branch is the German literal — the de-default shape.
* Deliberately narrow: it must not flag `locale === "de" ? deBody : enBody`,
* which is the correct polarity (German only for German readers).
*/
const DE_DEFAULT_TERNARY = /\?\s*"[a-z]{2}"\s*:\s*"de"/;
/** `payload.locale ?? "de"` and friends — a German default by omission. */
const DE_DEFAULT_NULLISH = /\?\?\s*"de"/;
/**
* Read a guarded file with comments stripped.
*
* The comments at these call sites deliberately QUOTE the banned shape to
* explain what was fixed; scanning them would make the guard self-tripping.
* Only executable source is checked.
*/
function readCode(rel: string): string {
return readFileSync(resolve(ROOT, rel), "utf8")
.replace(/\/\*[\s\S]*?\*\//g, "")
.replace(/^\s*\/\/.*$/gm, "");
}
describe("no German-default locale binary", () => {
it.each(GUARDED_FILES)("%s carries no de-default ternary", (rel) => {
expect(readCode(rel)).not.toMatch(DE_DEFAULT_TERNARY);
});
it.each(GUARDED_FILES)("%s carries no de-default fallback", (rel) => {
expect(readCode(rel)).not.toMatch(DE_DEFAULT_NULLISH);
});
it("recognises the shape it bans", () => {
// Guard the guard: the patterns must actually match the bug they describe,
// so a typo cannot turn this file into a silent pass.
expect(`locale === "en" ? "en" : "de"`).toMatch(DE_DEFAULT_TERNARY);
expect(`payload.locale ?? "de"`).toMatch(DE_DEFAULT_NULLISH);
// ...and must NOT match the correct polarity.
expect(`instructionLocale(locale)`).not.toMatch(DE_DEFAULT_TERNARY);
expect(`locale === "de" ? DE_BODY : EN_BODY`).not.toMatch(
DE_DEFAULT_TERNARY,
);
});
});
@@ -168,4 +168,19 @@ describe("buildCoachMemoryBlock", () => {
await buildCoachMemoryBlock(USER, PROFILE, NOW, "en");
expect(readPeriodNarrative).toHaveBeenCalledWith(USER, "month", "en");
});
// The narrative row is keyed by the full UI locale union. A French account's
// recall must read the `fr` row — narrowing it to `de` (the old behaviour)
// recalled a German narrative, and narrowing it to `en` would read a row
// that account never has.
it.each(["fr", "es", "it", "pl"] as const)(
"reads the %s narrative row without narrowing the locale",
async (locale) => {
readPeriodNarrative.mockResolvedValue(null);
buildPeriodNarrativeContext.mockResolvedValue(readyContext([]));
await buildCoachMemoryBlock(USER, PROFILE, NOW, locale);
expect(readPeriodNarrative).toHaveBeenCalledWith(USER, "month", locale);
},
);
});
@@ -115,7 +115,7 @@ describe("buildNarrativeUserPrompt", () => {
expect(prompt).toContain("WEIGHT");
expect(prompt).toContain("ACTIVITY_STEPS ~ SLEEP_DURATION");
expect(prompt).toContain("12 pairs tested");
expect(NARRATIVE_PROMPT_VERSION).toBe("1.13.0");
expect(NARRATIVE_PROMPT_VERSION).toBe("1.14.0");
});
});
@@ -0,0 +1,215 @@
/**
* Output-language contract for the PERIOD NARRATIVE surface.
*
* The bug this pins: the read route narrowed every non-English locale to `de`,
* so a French, Spanish, Italian or Polish reader received a German
* retrospective composed from the German instruction body. The fix keeps the
* two reviewed bodies (de, en) and lets the other four ride the English body
* with their language named and their OWN reply-language directive appended.
*
* Three things are asserted here, in ascending strength:
* 1. the four locales name their language and carry the verbatim directive
* from the translator-facing `safety-contracts.<locale>.yaml`;
* 2. no non-German prompt contains a German-body sentinel — the "nothing
* silently falls back to German" assertion this whole change exists for;
* 3. the German and English prompts are byte-identical to their pre-change
* form, modulo the prompt-version string, pinned by content hash.
*/
import { describe, it, expect, vi } from "vitest";
vi.mock("@/lib/crypto", () => ({
encrypt: (s: string) => `enc:${s}`,
decrypt: (s: string) => s.replace(/^enc:/, ""),
}));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/db", () => ({ prisma: {} }));
import { createHash } from "node:crypto";
import {
buildNarrativeSystemPrompt,
generatePeriodNarrative,
NARRATIVE_PROMPT_VERSION,
} from "@/lib/insights/narrative/period-narrative-generate";
import { loadSafetyContracts } from "@/lib/ai/prompts/safety-contracts";
import { locales, type Locale } from "@/lib/i18n/config";
import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative";
/** The four locales that ride the English body. */
const RIDING_LOCALES: Locale[] = ["fr", "es", "it", "pl"];
const LANGUAGE_NAMES: Record<string, string> = {
fr: "French",
es: "Spanish",
it: "Italian",
pl: "Polish",
};
/**
* Sentinels that only ever appear in the GERMAN instruction body. A hit on a
* non-German prompt means that locale fell back to German — the exact bug.
*/
const GERMAN_SENTINELS = [
"AUSGABEFORMAT",
"Antworte",
"auf Deutsch",
"Feste Regeln",
"Prompt-Version",
];
/**
* sha256 of the composed prompt with the version string normalised away.
*
* The version is deliberately excluded: bumping `NARRATIVE_PROMPT_VERSION` is
* a required part of any prompt change and is the ONE byte the de/en prompts
* are allowed to move by. Everything else is frozen.
*/
function promptHash(locale: "de" | "en"): string {
const normalised = buildNarrativeSystemPrompt(locale)
.split(NARRATIVE_PROMPT_VERSION)
.join("VERSION");
return createHash("sha256").update(normalised, "utf8").digest("hex");
}
/**
* Hashes captured from the prompts as they stood at NARRATIVE_PROMPT_VERSION
* 1.13.0, before the output-language change. If either moves, the de or en
* prompt drifted — which this change promised would not happen.
*/
const FROZEN_PROMPT_HASHES: Record<"de" | "en", string> = {
de: "be667f16beeb44f51a81fedd929716b2249579fe914b45bb8cba0cd9e64a8c6d",
en: "bb0b080e21a2b9729bdf0047ae221c326ec29ecd74d0999f19ea8deabcbf078a",
};
describe("period-narrative output language", () => {
it("keeps the German and English prompts byte-identical modulo the version", () => {
expect(promptHash("de")).toBe(FROZEN_PROMPT_HASHES.de);
expect(promptHash("en")).toBe(FROZEN_PROMPT_HASHES.en);
});
it("appends no directive to the two reviewed bodies", () => {
expect(buildNarrativeSystemPrompt("de")).not.toContain("OUTPUT LANGUAGE:");
expect(buildNarrativeSystemPrompt("en")).not.toContain("OUTPUT LANGUAGE:");
// The English body must not grow a language rule for its own locale.
expect(buildNarrativeSystemPrompt("en")).not.toContain(
"Write the summary in English.",
);
});
it.each(RIDING_LOCALES)(
"names the language and carries the verbatim directive for %s",
(locale) => {
const prompt = buildNarrativeSystemPrompt(locale);
const directive = loadSafetyContracts(locale).reply_language_directive;
expect(prompt).toContain(
`Write the summary in ${LANGUAGE_NAMES[locale]}.`,
);
expect(prompt).toContain(`OUTPUT LANGUAGE: ${directive}`);
// The directive is the LAST instruction the model reads.
expect(prompt.trimEnd().endsWith(directive.trimEnd())).toBe(true);
// The scaffolding is the English body, not a translation of it.
expect(prompt).toContain("Hard rules:");
},
);
it("specifically composes the French prompt in English with the French directive", () => {
const prompt = buildNarrativeSystemPrompt("fr");
expect(prompt).toContain("Write the summary in French.");
expect(prompt).toContain(
loadSafetyContracts("fr").reply_language_directive,
);
expect(prompt).toContain(
"You summarise one person's health-tracking PERIOD",
);
});
it.each(locales.filter((l) => l !== "de"))(
"leaks no German instruction body into the %s prompt",
(locale) => {
const prompt = buildNarrativeSystemPrompt(locale);
for (const sentinel of GERMAN_SENTINELS) {
expect(prompt).not.toContain(sentinel);
}
},
);
it("keeps the German prompt German", () => {
const prompt = buildNarrativeSystemPrompt("de");
expect(prompt).toContain("Feste Regeln:");
expect(prompt).toContain("Prompt-Version:");
});
});
describe("period-narrative locale storage", () => {
function readyContext(): PeriodNarrativeContext {
return {
status: "ready",
period: "week",
metricDeltas: [
{
type: "WEIGHT",
unit: "kg",
current: 80,
prior: 81,
delta: -1,
deltaPercent: -1.2,
currentDays: 6,
priorDays: 7,
},
],
bandTransitions: [],
drivers: [],
coincidentFlags: [],
pairsTested: 12,
fdrQ: 0.1,
provenance: {
metrics: ["WEIGHT"],
window: {
from: "2026-05-01T00:00:00.000Z",
to: "2026-05-15T00:00:00.000Z",
},
computedAt: "2026-05-15T05:00:00.000Z",
},
};
}
it("keys the row by the reader's own locale and prompts in their language", async () => {
const upsert = vi.fn(async (args: { create: { locale: string } }) => args);
const prisma = {
user: { findUnique: vi.fn(async () => ({ timezone: "Europe/Berlin" })) },
insightNarrative: { findUnique: vi.fn(async () => null), upsert },
};
let capturedSystemPrompt = "";
const runCompletion = vi.fn(async (args: { systemPrompt: string }) => {
capturedSystemPrompt = args.systemPrompt;
return {
kind: "ok" as const,
content: "Votre semaine.",
providerType: "mock",
};
});
const outcome = await generatePeriodNarrative("user-1", {
period: "week",
locale: "fr",
force: true,
buildContext: vi.fn(async () => readyContext()) as never,
runCompletion: runCompletion as never,
prisma: prisma as never,
});
expect(outcome).toEqual({ status: "generated", providerType: "mock" });
// The row is stored under `fr` — NOT collapsed into the de or en row, so a
// French retrospective is never served to a German or English reader.
expect(upsert.mock.calls[0]?.[0]).toMatchObject({
create: { locale: "fr" },
where: { userId_period_locale: { locale: "fr" } },
});
expect(capturedSystemPrompt).toContain("Write the summary in French.");
expect(capturedSystemPrompt).not.toContain("AUSGABEFORMAT");
// The provider cache action is locale-scoped too.
expect(runCompletion.mock.calls[0]?.[0]).toMatchObject({
cacheAction: "insights.narrative.week.fr",
});
});
});