From 2032b74fc61f5638b676ed7d5d3eeeaeae1a7de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 18 Jul 2026 23:14:45 +0200 Subject: [PATCH] test(ai): pin the output-language contract across all six locales The parity test joined raw fragments, which no longer matches now that the English output clause carries a language token; it substitutes the expected name from a literal map instead of from the module under test. Adds the contract test: which body each locale composes, that its own language is named, that its own translated directive is present and last, that no German reaches a non-German prompt, and that no two locales collapse to the same prompt. What it cannot show - whether the prose reads well in the four languages - is stated in the file. --- .../__tests__/locale-skeleton-parity.test.ts | 21 ++- .../output-language-contract.test.ts | 149 ++++++++++++++++++ 2 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 src/lib/ai/prompts/__tests__/output-language-contract.test.ts diff --git a/src/lib/ai/prompts/__tests__/locale-skeleton-parity.test.ts b/src/lib/ai/prompts/__tests__/locale-skeleton-parity.test.ts index 1d4c97382..5d81e269c 100644 --- a/src/lib/ai/prompts/__tests__/locale-skeleton-parity.test.ts +++ b/src/lib/ai/prompts/__tests__/locale-skeleton-parity.test.ts @@ -40,13 +40,28 @@ describe("base-system assessment prompt — locale skeleton parity", () => { }); it("the composed prompt equals the fragments joined by a blank line", () => { + // The English output clause carries a language token so a locale riding + // the English body can name its own language. For these two locales the + // substitution is their own language name — written as a literal here, not + // read from the module under test, so the assertion stays independent. + const EXPECTED_LANGUAGE_NAME = { en: "English", de: "German" } as const; for (const locale of ["en", "de"] as const) { - const expected = ASSESSMENT_SECTION_PAIRS.map((s) => s[locale]).join( - "\n\n", - ); + const expected = ASSESSMENT_SECTION_PAIRS.map((s) => + s[locale] + .split("{{OUTPUT_LANGUAGE}}") + .join(EXPECTED_LANGUAGE_NAME[locale]), + ).join("\n\n"); expect(getBaseSystemPrompt(locale)).toBe(expected); } }); + + it("de and en carry no appended output-language directive", () => { + // Their bodies name the language natively; appending a directive would + // change two prompts that are deliberately byte-stable across this change. + for (const locale of ["en", "de"] as const) { + expect(getBaseSystemPrompt(locale)).not.toContain("OUTPUT LANGUAGE:"); + } + }); }); describe("insight system prompt — locale skeleton parity", () => { diff --git a/src/lib/ai/prompts/__tests__/output-language-contract.test.ts b/src/lib/ai/prompts/__tests__/output-language-contract.test.ts new file mode 100644 index 000000000..4bfd05e25 --- /dev/null +++ b/src/lib/ai/prompts/__tests__/output-language-contract.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; + +import { locales, type Locale } from "@/lib/i18n/config"; +import { getBaseSystemPrompt } from "@/lib/ai/prompts/base-system"; +import { + instructionLocale, + outputLanguageDirective, + targetLanguageName, + withOutputLanguage, +} from "@/lib/ai/prompts/output-language"; +import { loadSafetyContracts } from "@/lib/ai/prompts/safety-contracts"; + +/** + * The output-language contract. + * + * These tests exist because four of the six locales cannot be eyeballed at + * review time. They pin the mechanical guarantees — which body a locale + * composes, that its own language is named, that the locale's own directive is + * present and last, and that no German leaks into a non-German prompt. + * + * What they deliberately do NOT claim: that the model's prose is idiomatic or + * correct in French, Spanish, Italian or Polish. No offline test can show + * that. The precedent is the native insights/briefing prompts, which have + * carried the same directive in production, and the residual risk is covered + * by a production check, not by this file. + */ + +/** + * A word that appears in the German instruction body and in no English one. + * If it shows up in a non-German prompt, the de-default collapse is back. + */ +const GERMAN_SENTINEL = /AUSGABEFORMAT|Antworte ausschließlich|Einschätzung/; + +/** English names, written literally — never derived from the module under test. */ +const EXPECTED_LANGUAGE_NAME: Record = { + de: "German", + en: "English", + fr: "French", + es: "Spanish", + it: "Italian", + pl: "Polish", +}; + +/** Which body each locale is expected to compose. */ +const EXPECTED_BODY: Record = { + de: "de", + en: "en", + fr: "en", + es: "en", + it: "en", + pl: "en", +}; + +describe("output-language helper", () => { + it("routes only German to the German body — every other locale to English", () => { + for (const locale of locales) { + expect(instructionLocale(locale), `instruction body for ${locale}`).toBe( + EXPECTED_BODY[locale], + ); + } + }); + + it("names each locale's language", () => { + for (const locale of locales) { + expect(targetLanguageName(locale)).toBe(EXPECTED_LANGUAGE_NAME[locale]); + } + }); + + it("emits the locale's OWN directive for the four riding the English body", () => { + for (const locale of ["fr", "es", "it", "pl"] as const) { + const directive = outputLanguageDirective(locale); + expect(directive, `directive for ${locale}`).toContain( + "OUTPUT LANGUAGE:", + ); + // The directive text must be the locale's own translated clause, not a + // generic English one — that is what makes it authoritative to the model. + expect(directive).toContain( + loadSafetyContracts(locale).reply_language_directive, + ); + } + }); + + it("emits no directive for de/en (their bodies name the language natively)", () => { + expect(outputLanguageDirective("de")).toBe(""); + expect(outputLanguageDirective("en")).toBe(""); + }); + + it("appends the directive last, blank-line separated", () => { + const composed = withOutputLanguage("BODY", "fr"); + expect(composed.startsWith("BODY\n\n")).toBe(true); + expect( + composed.endsWith(loadSafetyContracts("fr").reply_language_directive), + ).toBe(true); + }); + + it("leaves a de/en prompt untouched", () => { + expect(withOutputLanguage("BODY", "de")).toBe("BODY"); + expect(withOutputLanguage("BODY", "en")).toBe("BODY"); + }); +}); + +describe("base system prompt — every locale", () => { + it("asks for the assessment in the reader's own language", () => { + for (const locale of locales) { + const prompt = getBaseSystemPrompt(locale); + if (locale === "de") { + // The German body states its language in German. + expect(prompt).toContain("auf Deutsch"); + } else { + expect( + prompt, + `${locale} must name its own language in the output clause`, + ).toContain(`complete assessment in ${EXPECTED_LANGUAGE_NAME[locale]}`); + } + } + }); + + it("never sends German instructions to a non-German locale", () => { + // This is the regression the whole change exists to prevent. + for (const locale of locales) { + if (locale === "de") continue; + expect( + GERMAN_SENTINEL.test(getBaseSystemPrompt(locale)), + `${locale} prompt contains German instruction text`, + ).toBe(false); + } + }); + + it("ends the four non-de/en prompts with their own directive", () => { + for (const locale of ["fr", "es", "it", "pl"] as const) { + expect(getBaseSystemPrompt(locale).trimEnd()).toMatch( + /OUTPUT LANGUAGE: .+$/s, + ); + } + }); + + it("produces a distinct prompt per locale (no silent collapse)", () => { + const seen = new Map(); + for (const locale of locales) { + const prompt = getBaseSystemPrompt(locale); + const clash = seen.get(prompt); + expect( + clash, + `${locale} produced a prompt identical to ${clash} — a locale collapsed`, + ).toBeUndefined(); + seen.set(prompt, locale); + } + }); +});