mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 21:10:59 +00:00
test(ai): pin the output-language contract across all six locales
The parity test joined raw fragments, which no longer matches now that the English output clause carries a language token; it substitutes the expected name from a literal map instead of from the module under test. Adds the contract test: which body each locale composes, that its own language is named, that its own translated directive is present and last, that no German reaches a non-German prompt, and that no two locales collapse to the same prompt. What it cannot show - whether the prose reads well in the four languages - is stated in the file.
This commit is contained in:
1 parent
ec4e3adfdc
commit
2032b74fc6
2 files changed
+167
-3
No files matched your search
@@ -40,13 +40,28 @@ describe("base-system assessment prompt — locale skeleton parity", () => {
|
||||
});
|
||||
|
||||
it("the composed prompt equals the fragments joined by a blank line", () => {
|
||||
// The English output clause carries a language token so a locale riding
|
||||
// the English body can name its own language. For these two locales the
|
||||
// substitution is their own language name — written as a literal here, not
|
||||
// read from the module under test, so the assertion stays independent.
|
||||
const EXPECTED_LANGUAGE_NAME = { en: "English", de: "German" } as const;
|
||||
for (const locale of ["en", "de"] as const) {
|
||||
const expected = ASSESSMENT_SECTION_PAIRS.map((s) => s[locale]).join(
|
||||
"\n\n",
|
||||
);
|
||||
const expected = ASSESSMENT_SECTION_PAIRS.map((s) =>
|
||||
s[locale]
|
||||
.split("{{OUTPUT_LANGUAGE}}")
|
||||
.join(EXPECTED_LANGUAGE_NAME[locale]),
|
||||
).join("\n\n");
|
||||
expect(getBaseSystemPrompt(locale)).toBe(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("de and en carry no appended output-language directive", () => {
|
||||
// Their bodies name the language natively; appending a directive would
|
||||
// change two prompts that are deliberately byte-stable across this change.
|
||||
for (const locale of ["en", "de"] as const) {
|
||||
expect(getBaseSystemPrompt(locale)).not.toContain("OUTPUT LANGUAGE:");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("insight system prompt — locale skeleton parity", () => {
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { locales, type Locale } from "@/lib/i18n/config";
|
||||
import { getBaseSystemPrompt } from "@/lib/ai/prompts/base-system";
|
||||
import {
|
||||
instructionLocale,
|
||||
outputLanguageDirective,
|
||||
targetLanguageName,
|
||||
withOutputLanguage,
|
||||
} from "@/lib/ai/prompts/output-language";
|
||||
import { loadSafetyContracts } from "@/lib/ai/prompts/safety-contracts";
|
||||
|
||||
/**
|
||||
* The output-language contract.
|
||||
*
|
||||
* These tests exist because four of the six locales cannot be eyeballed at
|
||||
* review time. They pin the mechanical guarantees — which body a locale
|
||||
* composes, that its own language is named, that the locale's own directive is
|
||||
* present and last, and that no German leaks into a non-German prompt.
|
||||
*
|
||||
* What they deliberately do NOT claim: that the model's prose is idiomatic or
|
||||
* correct in French, Spanish, Italian or Polish. No offline test can show
|
||||
* that. The precedent is the native insights/briefing prompts, which have
|
||||
* carried the same directive in production, and the residual risk is covered
|
||||
* by a production check, not by this file.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A word that appears in the German instruction body and in no English one.
|
||||
* If it shows up in a non-German prompt, the de-default collapse is back.
|
||||
*/
|
||||
const GERMAN_SENTINEL = /AUSGABEFORMAT|Antworte ausschließlich|Einschätzung/;
|
||||
|
||||
/** English names, written literally — never derived from the module under test. */
|
||||
const EXPECTED_LANGUAGE_NAME: Record<Locale, string> = {
|
||||
de: "German",
|
||||
en: "English",
|
||||
fr: "French",
|
||||
es: "Spanish",
|
||||
it: "Italian",
|
||||
pl: "Polish",
|
||||
};
|
||||
|
||||
/** Which body each locale is expected to compose. */
|
||||
const EXPECTED_BODY: Record<Locale, "de" | "en"> = {
|
||||
de: "de",
|
||||
en: "en",
|
||||
fr: "en",
|
||||
es: "en",
|
||||
it: "en",
|
||||
pl: "en",
|
||||
};
|
||||
|
||||
describe("output-language helper", () => {
|
||||
it("routes only German to the German body — every other locale to English", () => {
|
||||
for (const locale of locales) {
|
||||
expect(instructionLocale(locale), `instruction body for ${locale}`).toBe(
|
||||
EXPECTED_BODY[locale],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("names each locale's language", () => {
|
||||
for (const locale of locales) {
|
||||
expect(targetLanguageName(locale)).toBe(EXPECTED_LANGUAGE_NAME[locale]);
|
||||
}
|
||||
});
|
||||
|
||||
it("emits the locale's OWN directive for the four riding the English body", () => {
|
||||
for (const locale of ["fr", "es", "it", "pl"] as const) {
|
||||
const directive = outputLanguageDirective(locale);
|
||||
expect(directive, `directive for ${locale}`).toContain(
|
||||
"OUTPUT LANGUAGE:",
|
||||
);
|
||||
// The directive text must be the locale's own translated clause, not a
|
||||
// generic English one — that is what makes it authoritative to the model.
|
||||
expect(directive).toContain(
|
||||
loadSafetyContracts(locale).reply_language_directive,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("emits no directive for de/en (their bodies name the language natively)", () => {
|
||||
expect(outputLanguageDirective("de")).toBe("");
|
||||
expect(outputLanguageDirective("en")).toBe("");
|
||||
});
|
||||
|
||||
it("appends the directive last, blank-line separated", () => {
|
||||
const composed = withOutputLanguage("BODY", "fr");
|
||||
expect(composed.startsWith("BODY\n\n")).toBe(true);
|
||||
expect(
|
||||
composed.endsWith(loadSafetyContracts("fr").reply_language_directive),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves a de/en prompt untouched", () => {
|
||||
expect(withOutputLanguage("BODY", "de")).toBe("BODY");
|
||||
expect(withOutputLanguage("BODY", "en")).toBe("BODY");
|
||||
});
|
||||
});
|
||||
|
||||
describe("base system prompt — every locale", () => {
|
||||
it("asks for the assessment in the reader's own language", () => {
|
||||
for (const locale of locales) {
|
||||
const prompt = getBaseSystemPrompt(locale);
|
||||
if (locale === "de") {
|
||||
// The German body states its language in German.
|
||||
expect(prompt).toContain("auf Deutsch");
|
||||
} else {
|
||||
expect(
|
||||
prompt,
|
||||
`${locale} must name its own language in the output clause`,
|
||||
).toContain(`complete assessment in ${EXPECTED_LANGUAGE_NAME[locale]}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("never sends German instructions to a non-German locale", () => {
|
||||
// This is the regression the whole change exists to prevent.
|
||||
for (const locale of locales) {
|
||||
if (locale === "de") continue;
|
||||
expect(
|
||||
GERMAN_SENTINEL.test(getBaseSystemPrompt(locale)),
|
||||
`${locale} prompt contains German instruction text`,
|
||||
).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("ends the four non-de/en prompts with their own directive", () => {
|
||||
for (const locale of ["fr", "es", "it", "pl"] as const) {
|
||||
expect(getBaseSystemPrompt(locale).trimEnd()).toMatch(
|
||||
/OUTPUT LANGUAGE: .+$/s,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("produces a distinct prompt per locale (no silent collapse)", () => {
|
||||
const seen = new Map<string, Locale>();
|
||||
for (const locale of locales) {
|
||||
const prompt = getBaseSystemPrompt(locale);
|
||||
const clash = seen.get(prompt);
|
||||
expect(
|
||||
clash,
|
||||
`${locale} produced a prompt identical to ${clash} — a locale collapsed`,
|
||||
).toBeUndefined();
|
||||
seen.set(prompt, locale);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user