test(ai): pin the output-language contract across all six locales

The parity test joined raw fragments, which no longer matches now that
the English output clause carries a language token; it substitutes the
expected name from a literal map instead of from the module under test.

Adds the contract test: which body each locale composes, that its own
language is named, that its own translated directive is present and
last, that no German reaches a non-German prompt, and that no two
locales collapse to the same prompt. What it cannot show - whether the
prose reads well in the four languages - is stated in the file.
This commit is contained in:
Marc-André Bombeck committed 2026-07-18 23:14:45 +02:00
1 parent ec4e3adfdc
commit 2032b74fc6
2 files changed
+167 -3

No files matched your search

@@ -40,13 +40,28 @@ describe("base-system assessment prompt — locale skeleton parity", () => {
});
it("the composed prompt equals the fragments joined by a blank line", () => {
// The English output clause carries a language token so a locale riding
// the English body can name its own language. For these two locales the
// substitution is their own language name — written as a literal here, not
// read from the module under test, so the assertion stays independent.
const EXPECTED_LANGUAGE_NAME = { en: "English", de: "German" } as const;
for (const locale of ["en", "de"] as const) {
const expected = ASSESSMENT_SECTION_PAIRS.map((s) => s[locale]).join(
"\n\n",
);
const expected = ASSESSMENT_SECTION_PAIRS.map((s) =>
s[locale]
.split("{{OUTPUT_LANGUAGE}}")
.join(EXPECTED_LANGUAGE_NAME[locale]),
).join("\n\n");
expect(getBaseSystemPrompt(locale)).toBe(expected);
}
});
it("de and en carry no appended output-language directive", () => {
// Their bodies name the language natively; appending a directive would
// change two prompts that are deliberately byte-stable across this change.
for (const locale of ["en", "de"] as const) {
expect(getBaseSystemPrompt(locale)).not.toContain("OUTPUT LANGUAGE:");
}
});
});
describe("insight system prompt — locale skeleton parity", () => {
@@ -0,0 +1,149 @@
import { describe, expect, it } from "vitest";
import { locales, type Locale } from "@/lib/i18n/config";
import { getBaseSystemPrompt } from "@/lib/ai/prompts/base-system";
import {
instructionLocale,
outputLanguageDirective,
targetLanguageName,
withOutputLanguage,
} from "@/lib/ai/prompts/output-language";
import { loadSafetyContracts } from "@/lib/ai/prompts/safety-contracts";
/**
* The output-language contract.
*
* These tests exist because four of the six locales cannot be eyeballed at
* review time. They pin the mechanical guarantees — which body a locale
* composes, that its own language is named, that the locale's own directive is
* present and last, and that no German leaks into a non-German prompt.
*
* What they deliberately do NOT claim: that the model's prose is idiomatic or
* correct in French, Spanish, Italian or Polish. No offline test can show
* that. The precedent is the native insights/briefing prompts, which have
* carried the same directive in production, and the residual risk is covered
* by a production check, not by this file.
*/
/**
* A word that appears in the German instruction body and in no English one.
* If it shows up in a non-German prompt, the de-default collapse is back.
*/
const GERMAN_SENTINEL = /AUSGABEFORMAT|Antworte ausschließlich|Einschätzung/;
/** English names, written literally — never derived from the module under test. */
const EXPECTED_LANGUAGE_NAME: Record<Locale, string> = {
de: "German",
en: "English",
fr: "French",
es: "Spanish",
it: "Italian",
pl: "Polish",
};
/** Which body each locale is expected to compose. */
const EXPECTED_BODY: Record<Locale, "de" | "en"> = {
de: "de",
en: "en",
fr: "en",
es: "en",
it: "en",
pl: "en",
};
describe("output-language helper", () => {
it("routes only German to the German body — every other locale to English", () => {
for (const locale of locales) {
expect(instructionLocale(locale), `instruction body for ${locale}`).toBe(
EXPECTED_BODY[locale],
);
}
});
it("names each locale's language", () => {
for (const locale of locales) {
expect(targetLanguageName(locale)).toBe(EXPECTED_LANGUAGE_NAME[locale]);
}
});
it("emits the locale's OWN directive for the four riding the English body", () => {
for (const locale of ["fr", "es", "it", "pl"] as const) {
const directive = outputLanguageDirective(locale);
expect(directive, `directive for ${locale}`).toContain(
"OUTPUT LANGUAGE:",
);
// The directive text must be the locale's own translated clause, not a
// generic English one — that is what makes it authoritative to the model.
expect(directive).toContain(
loadSafetyContracts(locale).reply_language_directive,
);
}
});
it("emits no directive for de/en (their bodies name the language natively)", () => {
expect(outputLanguageDirective("de")).toBe("");
expect(outputLanguageDirective("en")).toBe("");
});
it("appends the directive last, blank-line separated", () => {
const composed = withOutputLanguage("BODY", "fr");
expect(composed.startsWith("BODY\n\n")).toBe(true);
expect(
composed.endsWith(loadSafetyContracts("fr").reply_language_directive),
).toBe(true);
});
it("leaves a de/en prompt untouched", () => {
expect(withOutputLanguage("BODY", "de")).toBe("BODY");
expect(withOutputLanguage("BODY", "en")).toBe("BODY");
});
});
describe("base system prompt — every locale", () => {
it("asks for the assessment in the reader's own language", () => {
for (const locale of locales) {
const prompt = getBaseSystemPrompt(locale);
if (locale === "de") {
// The German body states its language in German.
expect(prompt).toContain("auf Deutsch");
} else {
expect(
prompt,
`${locale} must name its own language in the output clause`,
).toContain(`complete assessment in ${EXPECTED_LANGUAGE_NAME[locale]}`);
}
}
});
it("never sends German instructions to a non-German locale", () => {
// This is the regression the whole change exists to prevent.
for (const locale of locales) {
if (locale === "de") continue;
expect(
GERMAN_SENTINEL.test(getBaseSystemPrompt(locale)),
`${locale} prompt contains German instruction text`,
).toBe(false);
}
});
it("ends the four non-de/en prompts with their own directive", () => {
for (const locale of ["fr", "es", "it", "pl"] as const) {
expect(getBaseSystemPrompt(locale).trimEnd()).toMatch(
/OUTPUT LANGUAGE: .+$/s,
);
}
});
it("produces a distinct prompt per locale (no silent collapse)", () => {
const seen = new Map<string, Locale>();
for (const locale of locales) {
const prompt = getBaseSystemPrompt(locale);
const clash = seen.get(prompt);
expect(
clash,
`${locale} produced a prompt identical to ${clash} — a locale collapsed`,
).toBeUndefined();
seen.set(prompt, locale);
}
});
});