mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 20:42:43 +00:00
test(insights): pin the period-narrative and coach output-language contract
Three layers. The German and English narrative prompts are frozen by content hash, normalised for the prompt-version string that a prompt change is required to move — any other drift fails. The four riding locales must name their language, carry the verbatim reply-language directive from their own safety-contract file as the last instruction, and contain no German-body sentinel. A generator-level test drives a French run end to end and asserts the row is keyed fr, the cache action is locale-scoped, and the system prompt reaches the provider in English with the French directive. Alongside it a source-level ban on the exact shape the bug took (`=== "en" ? "en" : "de"`, `?? "de"`) across the coach and narrative surfaces, with a self-check so a typo in the pattern cannot turn the guard into a silent pass, and coach-memory recall cases for all four locales.
This commit is contained in:
1 parent
e4ec0eec49
commit
93738ae968
4 files changed
+307
-1
No files matched your search
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* Regression-class ban for the German-default locale binary.
|
||||
*
|
||||
* The bug this pins had exactly one shape: `locale === "en" ? "en" : "de"`
|
||||
* (and its `=== "de" ? ... : ...` inversions with a `de` fallback). Every
|
||||
* occurrence sent French, Spanish, Italian and Polish accounts down the German
|
||||
* branch. This walks the source of the surfaces that carried it and fails on
|
||||
* the shape itself, so a future edit cannot quietly reintroduce it.
|
||||
*
|
||||
* Scoped to the files this change owns — the coach memory refresh, the coach
|
||||
* snapshot's narrative recall, the narrative route + generator + warm, and the
|
||||
* dashboard briefing recall. Blunt by design: it matches text, not behaviour,
|
||||
* which is precisely what makes it survive a refactor of the logic around it.
|
||||
*/
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
const ROOT = resolve(__dirname, "../../../..");
|
||||
|
||||
const GUARDED_FILES = [
|
||||
"app/api/insights/chat/route.ts",
|
||||
"app/api/insights/narrative/route.ts",
|
||||
"lib/ai/coach/snapshot.ts",
|
||||
"lib/ai/coach/memory-snapshot.ts",
|
||||
"lib/ai/coach/coach-memory-refresh-worker.ts",
|
||||
"lib/dashboard/snapshot.ts",
|
||||
"lib/insights/narrative/period-narrative-generate.ts",
|
||||
"lib/jobs/period-narrative-warm.ts",
|
||||
"lib/jobs/period-narrative-shared.ts",
|
||||
];
|
||||
|
||||
/**
|
||||
* A ternary whose FALSE branch is the German literal — the de-default shape.
|
||||
* Deliberately narrow: it must not flag `locale === "de" ? deBody : enBody`,
|
||||
* which is the correct polarity (German only for German readers).
|
||||
*/
|
||||
const DE_DEFAULT_TERNARY = /\?\s*"[a-z]{2}"\s*:\s*"de"/;
|
||||
|
||||
/** `payload.locale ?? "de"` and friends — a German default by omission. */
|
||||
const DE_DEFAULT_NULLISH = /\?\?\s*"de"/;
|
||||
|
||||
/**
|
||||
* Read a guarded file with comments stripped.
|
||||
*
|
||||
* The comments at these call sites deliberately QUOTE the banned shape to
|
||||
* explain what was fixed; scanning them would make the guard self-tripping.
|
||||
* Only executable source is checked.
|
||||
*/
|
||||
function readCode(rel: string): string {
|
||||
return readFileSync(resolve(ROOT, rel), "utf8")
|
||||
.replace(/\/\*[\s\S]*?\*\//g, "")
|
||||
.replace(/^\s*\/\/.*$/gm, "");
|
||||
}
|
||||
|
||||
describe("no German-default locale binary", () => {
|
||||
it.each(GUARDED_FILES)("%s carries no de-default ternary", (rel) => {
|
||||
expect(readCode(rel)).not.toMatch(DE_DEFAULT_TERNARY);
|
||||
});
|
||||
|
||||
it.each(GUARDED_FILES)("%s carries no de-default fallback", (rel) => {
|
||||
expect(readCode(rel)).not.toMatch(DE_DEFAULT_NULLISH);
|
||||
});
|
||||
|
||||
it("recognises the shape it bans", () => {
|
||||
// Guard the guard: the patterns must actually match the bug they describe,
|
||||
// so a typo cannot turn this file into a silent pass.
|
||||
expect(`locale === "en" ? "en" : "de"`).toMatch(DE_DEFAULT_TERNARY);
|
||||
expect(`payload.locale ?? "de"`).toMatch(DE_DEFAULT_NULLISH);
|
||||
// ...and must NOT match the correct polarity.
|
||||
expect(`instructionLocale(locale)`).not.toMatch(DE_DEFAULT_TERNARY);
|
||||
expect(`locale === "de" ? DE_BODY : EN_BODY`).not.toMatch(
|
||||
DE_DEFAULT_TERNARY,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -168,4 +168,19 @@ describe("buildCoachMemoryBlock", () => {
|
||||
await buildCoachMemoryBlock(USER, PROFILE, NOW, "en");
|
||||
expect(readPeriodNarrative).toHaveBeenCalledWith(USER, "month", "en");
|
||||
});
|
||||
|
||||
// The narrative row is keyed by the full UI locale union. A French account's
|
||||
// recall must read the `fr` row — narrowing it to `de` (the old behaviour)
|
||||
// recalled a German narrative, and narrowing it to `en` would read a row
|
||||
// that account never has.
|
||||
it.each(["fr", "es", "it", "pl"] as const)(
|
||||
"reads the %s narrative row without narrowing the locale",
|
||||
async (locale) => {
|
||||
readPeriodNarrative.mockResolvedValue(null);
|
||||
buildPeriodNarrativeContext.mockResolvedValue(readyContext([]));
|
||||
|
||||
await buildCoachMemoryBlock(USER, PROFILE, NOW, locale);
|
||||
expect(readPeriodNarrative).toHaveBeenCalledWith(USER, "month", locale);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -115,7 +115,7 @@ describe("buildNarrativeUserPrompt", () => {
|
||||
expect(prompt).toContain("WEIGHT");
|
||||
expect(prompt).toContain("ACTIVITY_STEPS ~ SLEEP_DURATION");
|
||||
expect(prompt).toContain("12 pairs tested");
|
||||
expect(NARRATIVE_PROMPT_VERSION).toBe("1.13.0");
|
||||
expect(NARRATIVE_PROMPT_VERSION).toBe("1.14.0");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
/**
|
||||
* Output-language contract for the PERIOD NARRATIVE surface.
|
||||
*
|
||||
* The bug this pins: the read route narrowed every non-English locale to `de`,
|
||||
* so a French, Spanish, Italian or Polish reader received a German
|
||||
* retrospective composed from the German instruction body. The fix keeps the
|
||||
* two reviewed bodies (de, en) and lets the other four ride the English body
|
||||
* with their language named and their OWN reply-language directive appended.
|
||||
*
|
||||
* Three things are asserted here, in ascending strength:
|
||||
* 1. the four locales name their language and carry the verbatim directive
|
||||
* from the translator-facing `safety-contracts.<locale>.yaml`;
|
||||
* 2. no non-German prompt contains a German-body sentinel — the "nothing
|
||||
* silently falls back to German" assertion this whole change exists for;
|
||||
* 3. the German and English prompts are byte-identical to their pre-change
|
||||
* form, modulo the prompt-version string, pinned by content hash.
|
||||
*/
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({
|
||||
encrypt: (s: string) => `enc:${s}`,
|
||||
decrypt: (s: string) => s.replace(/^enc:/, ""),
|
||||
}));
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
vi.mock("@/lib/db", () => ({ prisma: {} }));
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
buildNarrativeSystemPrompt,
|
||||
generatePeriodNarrative,
|
||||
NARRATIVE_PROMPT_VERSION,
|
||||
} from "@/lib/insights/narrative/period-narrative-generate";
|
||||
import { loadSafetyContracts } from "@/lib/ai/prompts/safety-contracts";
|
||||
import { locales, type Locale } from "@/lib/i18n/config";
|
||||
import type { PeriodNarrativeContext } from "@/lib/insights/narrative/period-narrative";
|
||||
|
||||
/** The four locales that ride the English body. */
|
||||
const RIDING_LOCALES: Locale[] = ["fr", "es", "it", "pl"];
|
||||
|
||||
const LANGUAGE_NAMES: Record<string, string> = {
|
||||
fr: "French",
|
||||
es: "Spanish",
|
||||
it: "Italian",
|
||||
pl: "Polish",
|
||||
};
|
||||
|
||||
/**
|
||||
* Sentinels that only ever appear in the GERMAN instruction body. A hit on a
|
||||
* non-German prompt means that locale fell back to German — the exact bug.
|
||||
*/
|
||||
const GERMAN_SENTINELS = [
|
||||
"AUSGABEFORMAT",
|
||||
"Antworte",
|
||||
"auf Deutsch",
|
||||
"Feste Regeln",
|
||||
"Prompt-Version",
|
||||
];
|
||||
|
||||
/**
|
||||
* sha256 of the composed prompt with the version string normalised away.
|
||||
*
|
||||
* The version is deliberately excluded: bumping `NARRATIVE_PROMPT_VERSION` is
|
||||
* a required part of any prompt change and is the ONE byte the de/en prompts
|
||||
* are allowed to move by. Everything else is frozen.
|
||||
*/
|
||||
function promptHash(locale: "de" | "en"): string {
|
||||
const normalised = buildNarrativeSystemPrompt(locale)
|
||||
.split(NARRATIVE_PROMPT_VERSION)
|
||||
.join("VERSION");
|
||||
return createHash("sha256").update(normalised, "utf8").digest("hex");
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes captured from the prompts as they stood at NARRATIVE_PROMPT_VERSION
|
||||
* 1.13.0, before the output-language change. If either moves, the de or en
|
||||
* prompt drifted — which this change promised would not happen.
|
||||
*/
|
||||
const FROZEN_PROMPT_HASHES: Record<"de" | "en", string> = {
|
||||
de: "be667f16beeb44f51a81fedd929716b2249579fe914b45bb8cba0cd9e64a8c6d",
|
||||
en: "bb0b080e21a2b9729bdf0047ae221c326ec29ecd74d0999f19ea8deabcbf078a",
|
||||
};
|
||||
|
||||
describe("period-narrative output language", () => {
|
||||
it("keeps the German and English prompts byte-identical modulo the version", () => {
|
||||
expect(promptHash("de")).toBe(FROZEN_PROMPT_HASHES.de);
|
||||
expect(promptHash("en")).toBe(FROZEN_PROMPT_HASHES.en);
|
||||
});
|
||||
|
||||
it("appends no directive to the two reviewed bodies", () => {
|
||||
expect(buildNarrativeSystemPrompt("de")).not.toContain("OUTPUT LANGUAGE:");
|
||||
expect(buildNarrativeSystemPrompt("en")).not.toContain("OUTPUT LANGUAGE:");
|
||||
// The English body must not grow a language rule for its own locale.
|
||||
expect(buildNarrativeSystemPrompt("en")).not.toContain(
|
||||
"Write the summary in English.",
|
||||
);
|
||||
});
|
||||
|
||||
it.each(RIDING_LOCALES)(
|
||||
"names the language and carries the verbatim directive for %s",
|
||||
(locale) => {
|
||||
const prompt = buildNarrativeSystemPrompt(locale);
|
||||
const directive = loadSafetyContracts(locale).reply_language_directive;
|
||||
|
||||
expect(prompt).toContain(
|
||||
`Write the summary in ${LANGUAGE_NAMES[locale]}.`,
|
||||
);
|
||||
expect(prompt).toContain(`OUTPUT LANGUAGE: ${directive}`);
|
||||
// The directive is the LAST instruction the model reads.
|
||||
expect(prompt.trimEnd().endsWith(directive.trimEnd())).toBe(true);
|
||||
// The scaffolding is the English body, not a translation of it.
|
||||
expect(prompt).toContain("Hard rules:");
|
||||
},
|
||||
);
|
||||
|
||||
it("specifically composes the French prompt in English with the French directive", () => {
|
||||
const prompt = buildNarrativeSystemPrompt("fr");
|
||||
expect(prompt).toContain("Write the summary in French.");
|
||||
expect(prompt).toContain(
|
||||
loadSafetyContracts("fr").reply_language_directive,
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"You summarise one person's health-tracking PERIOD",
|
||||
);
|
||||
});
|
||||
|
||||
it.each(locales.filter((l) => l !== "de"))(
|
||||
"leaks no German instruction body into the %s prompt",
|
||||
(locale) => {
|
||||
const prompt = buildNarrativeSystemPrompt(locale);
|
||||
for (const sentinel of GERMAN_SENTINELS) {
|
||||
expect(prompt).not.toContain(sentinel);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the German prompt German", () => {
|
||||
const prompt = buildNarrativeSystemPrompt("de");
|
||||
expect(prompt).toContain("Feste Regeln:");
|
||||
expect(prompt).toContain("Prompt-Version:");
|
||||
});
|
||||
});
|
||||
|
||||
describe("period-narrative locale storage", () => {
|
||||
function readyContext(): PeriodNarrativeContext {
|
||||
return {
|
||||
status: "ready",
|
||||
period: "week",
|
||||
metricDeltas: [
|
||||
{
|
||||
type: "WEIGHT",
|
||||
unit: "kg",
|
||||
current: 80,
|
||||
prior: 81,
|
||||
delta: -1,
|
||||
deltaPercent: -1.2,
|
||||
currentDays: 6,
|
||||
priorDays: 7,
|
||||
},
|
||||
],
|
||||
bandTransitions: [],
|
||||
drivers: [],
|
||||
coincidentFlags: [],
|
||||
pairsTested: 12,
|
||||
fdrQ: 0.1,
|
||||
provenance: {
|
||||
metrics: ["WEIGHT"],
|
||||
window: {
|
||||
from: "2026-05-01T00:00:00.000Z",
|
||||
to: "2026-05-15T00:00:00.000Z",
|
||||
},
|
||||
computedAt: "2026-05-15T05:00:00.000Z",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("keys the row by the reader's own locale and prompts in their language", async () => {
|
||||
const upsert = vi.fn(async (args: { create: { locale: string } }) => args);
|
||||
const prisma = {
|
||||
user: { findUnique: vi.fn(async () => ({ timezone: "Europe/Berlin" })) },
|
||||
insightNarrative: { findUnique: vi.fn(async () => null), upsert },
|
||||
};
|
||||
let capturedSystemPrompt = "";
|
||||
const runCompletion = vi.fn(async (args: { systemPrompt: string }) => {
|
||||
capturedSystemPrompt = args.systemPrompt;
|
||||
return {
|
||||
kind: "ok" as const,
|
||||
content: "Votre semaine.",
|
||||
providerType: "mock",
|
||||
};
|
||||
});
|
||||
|
||||
const outcome = await generatePeriodNarrative("user-1", {
|
||||
period: "week",
|
||||
locale: "fr",
|
||||
force: true,
|
||||
buildContext: vi.fn(async () => readyContext()) as never,
|
||||
runCompletion: runCompletion as never,
|
||||
prisma: prisma as never,
|
||||
});
|
||||
|
||||
expect(outcome).toEqual({ status: "generated", providerType: "mock" });
|
||||
// The row is stored under `fr` — NOT collapsed into the de or en row, so a
|
||||
// French retrospective is never served to a German or English reader.
|
||||
expect(upsert.mock.calls[0]?.[0]).toMatchObject({
|
||||
create: { locale: "fr" },
|
||||
where: { userId_period_locale: { locale: "fr" } },
|
||||
});
|
||||
expect(capturedSystemPrompt).toContain("Write the summary in French.");
|
||||
expect(capturedSystemPrompt).not.toContain("AUSGABEFORMAT");
|
||||
// The provider cache action is locale-scoped too.
|
||||
expect(runCompletion.mock.calls[0]?.[0]).toMatchObject({
|
||||
cacheAction: "insights.narrative.week.fr",
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user