Merge branch 'fix/language-matchers-rest' into release/v1.37.27

This commit is contained in:
Marc-André Bombeck committed 2026-08-23 19:38:45 +02:00
commit b154ec52cb
13 files changed
+1291 -82

No files matched your search

@@ -546,6 +546,67 @@ describe("Telegram webhook — text-message dispatch", () => {
expect(prisma.medicationIntakeEvent.create).not.toHaveBeenCalled();
});
/**
* The bare word is an alias for `/help`. It knew "help" and "hilfe", so the
* four other shipped languages asking for help fell through to the intake
* keyword, failed it, and the handler returned without sending anything —
* silence, not an error. These pin one spelling per shipped language.
*/
it.each([
["help", 210],
["Hilfe", 211],
["aide", 212],
["ayuda", 213],
["aiuto", 214],
["Pomoc", 215],
])("answers the bare help word %s", async (word, updateId) => {
const res = await POST(
tgRequest({
update_id: updateId,
message: { message_id: 20, text: word, chat: { id: 7777 } },
}),
);
expect(res.status).toBe(200);
expect(sendTelegramMessage).toHaveBeenCalledTimes(1);
expect(vi.mocked(sendTelegramMessage).mock.calls[0][2]).toMatch(
/available commands/i,
);
});
it("does not swallow a sentence that merely starts with a help word", async () => {
// The alias matches the WHOLE message; "help me log my weight" is prose
// the later paths should still see, not a command.
const res = await POST(
tgRequest({
update_id: 216,
message: {
message_id: 21,
text: "help me log my weight",
chat: { id: 7777 },
},
}),
);
expect(res.status).toBe(200);
expect(sendTelegramMessage).not.toHaveBeenCalled();
});
it.each([
["bonjour", 220],
["Hola", 221],
["ciao", 222],
["cześć", 223],
])("answers the greeting %s", async (word, updateId) => {
const res = await POST(
tgRequest({
update_id: updateId,
message: { message_id: 22, text: word, chat: { id: 7777 } },
}),
);
expect(res.status).toBe(200);
expect(sendTelegramMessage).toHaveBeenCalledTimes(1);
expect(vi.mocked(sendTelegramMessage).mock.calls[0][2]).toMatch(/👋/);
});
it("greeting like 'hi' echoes a friendly reply", async () => {
const res = await POST(
tgRequest({
@@ -28,6 +28,65 @@ describe("filterIconCatalog", () => {
expect(hits.map((entry) => entry.name)).toContain("Dumbbell");
});
/**
* The only searchable text used to be the English Lucide name and the
* English keyword aids, while the grid renders its group headers in the
* reader's language. A German reader who typed back the header the picker
* had just shown them got an empty grid and "no icons found" — an empty
* table with nothing to say the query was fine and the language was not.
*/
describe("group headers in the reader's language", () => {
/**
* The header the picker renders above the health group, per shipped
* locale, as `mood.manage.iconGroupHealth` spells it. Derived at the call
* site in the component; pinned here so the property is a test, not a
* transcription that can drift from the bundle.
*/
const HEALTH_HEADER: Record<string, string> = {
en: "Health",
de: "Gesundheit",
fr: "Santé",
es: "Salud",
it: "Salute",
pl: "Zdrowie",
};
it.each(Object.entries(HEALTH_HEADER))(
"finds the health icons from the %s header",
(locale, header) => {
const labels = { health: header };
const hits = filterIconCatalog(MOOD_TAG_ICON_CATALOG, header, labels);
const names = hits.map((entry) => entry.name);
// Every drawable icon under the header the reader just read. (An
// English "Health" also hits the "healthy" keyword aid on a food icon,
// which is a union match and fine — the assertion is on coverage of
// the group, not on the absence of other hits.)
for (const entry of MOOD_TAG_ICON_CATALOG) {
if (entry.group !== "health") continue;
if (!isMoodTagIconName(entry.name)) continue;
expect(names).toContain(entry.name);
}
void locale;
},
);
it("ignores accents and case in the header match", () => {
const labels = { weather: "Météo" };
const hits = filterIconCatalog(MOOD_TAG_ICON_CATALOG, "meteo", labels);
expect(hits.length).toBeGreaterThan(0);
expect(hits.every((entry) => entry.group === "weather")).toBe(true);
});
it("still finds nothing for a word no header and no keyword carries", () => {
// The per-icon aids stay English. This is the stated remaining limit,
// pinned so a later change to it is a deliberate one.
const labels = { health: "Gesundheit" };
expect(
filterIconCatalog(MOOD_TAG_ICON_CATALOG, "herz", labels),
).toHaveLength(0);
});
});
it("matches on keyword aids", () => {
const byKeyword = filterIconCatalog(MOOD_TAG_ICON_CATALOG, "happy");
expect(byKeyword.length).toBeGreaterThan(0);
+41 -6
View File
@@ -6,6 +6,7 @@ import { Search, X } from "lucide-react";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import { useTranslations } from "@/lib/i18n/context";
import { foldForMatch } from "@/lib/i18n/fold-for-match";
import { useRovingRadioGroup } from "@/hooks/use-roving-radio-group";
import {
MOOD_TAG_ICON_CATALOG,
@@ -36,21 +37,42 @@ const ICON_GROUP_LABEL_KEYS: Record<string, string> = {
};
/**
* Pure search filter — name + keyword substring match, case-insensitive.
* Pure search filter — name, keyword, and translated group-header match.
* Exported for unit tests. Entries the client bundle cannot draw are
* excluded up front so the picker never offers a fallback-glyph tile.
*
* `groupLabels` maps a catalog group to the header the picker is CURRENTLY
* rendering above it, so the words on screen in the reader's language are also
* the words the box searches. Without it the only searchable text was the
* English Lucide name and the English keyword aids, and a German reader who
* typed what the picker itself had just shown them ("Gesundheit") got an empty
* grid and "no icons found" — an empty table, with nothing to say it was the
* language rather than the query.
*
* That is a partial close, stated rather than papered over: the per-icon
* keywords ("happy", "gym", "coffee") stay English, so searching for "Herz" or
* "kawa" still finds nothing. Translating them is ~80 icons times three aids
* times six languages of new content, which is a decision about content rather
* than about this filter. Omit the argument and the behaviour is the old one.
*
* Both sides go through `foldForMatch`, so accents and hyphens cannot make a
* reader miss a header they can see.
*/
export function filterIconCatalog(
catalog: readonly MoodTagIconCatalogEntry[],
query: string,
groupLabels: Readonly<Record<string, string>> = {},
): MoodTagIconCatalogEntry[] {
const drawable = catalog.filter((entry) => isMoodTagIconName(entry.name));
const needle = query.trim().toLowerCase();
const needle = foldForMatch(query);
if (needle.length === 0) return drawable;
return drawable.filter(
(entry) =>
entry.name.toLowerCase().includes(needle) ||
entry.keywords.some((keyword) => keyword.toLowerCase().includes(needle)),
foldForMatch(entry.name).includes(needle) ||
entry.keywords.some((keyword) =>
foldForMatch(keyword).includes(needle),
) ||
foldForMatch(groupLabels[entry.group] ?? "").includes(needle),
);
}
@@ -65,9 +87,22 @@ export function MoodTagIconPicker({ value, onChange }: MoodTagIconPickerProps) {
const [query, setQuery] = useState("");
const groupLabelId = useId();
// The headers the grid is about to render, so the search box matches the
// words the reader can see rather than only their English source.
const groupLabels = useMemo(
() =>
Object.fromEntries(
Object.entries(ICON_GROUP_LABEL_KEYS).map(([group, key]) => [
group,
t(key),
]),
),
[t],
);
const filtered = useMemo(
() => filterIconCatalog(MOOD_TAG_ICON_CATALOG, query),
[query],
() => filterIconCatalog(MOOD_TAG_ICON_CATALOG, query, groupLabels),
[query, groupLabels],
);
// One flat roving radiogroup across the (filtered) grid; the group
@@ -2,8 +2,10 @@ import { describe, it, expect, vi } from "vitest";
import {
detectsNormativeClaim,
computeCitationCoverage,
normativeKeywordBank,
} from "../citation-coverage";
import { MEDICAL_REFERENCES } from "../medical-references";
import { locales, type Locale } from "@/lib/i18n/config";
const annotateMock = vi.fn();
@@ -233,3 +235,90 @@ describe("computeCitationCoverage()", () => {
expect(result.uncitedNormativeRecommendationIds).toEqual(["uncited-1"]);
});
});
/**
* The bank used to be English plus German. Recommendations are generated in
* the reader's language, so for fr / es / it / pl every rec scored
* non-normative and the annotation reported 0 normative, 0 uncited — a
* coverage figure indistinguishable from perfect coverage. These pin that the
* check can still fail in each shipped language.
*/
describe("normative-claim detection across the shipped locales", () => {
/** One unambiguously normative sentence per shipped locale. */
const NORMATIVE_SENTENCE: Record<Locale, string> = {
en: "Your systolic should stay below the target of 130 mmHg.",
de: "Ihr systolischer Wert sollte unter dem Zielwert von 130 mmHg bleiben.",
fr: "Votre systolique devrait rester en dessous de l’objectif de 130 mmHg.",
es: "Su sistólica debería mantenerse por debajo del objetivo de 130 mmHg.",
it: "La sistolica dovrebbe restare al di sotto dell’obiettivo di 130 mmHg.",
pl: "Ciśnienie skurczowe powinno pozostać poniżej celu 130 mmHg.",
};
/**
* The same observation phrased WITHOUT a normative claim — a pure
* within-user comparison. Pins that the locale banks widened detection
* rather than making every sentence match.
*/
const OBSERVATIONAL_SENTENCE: Record<Locale, string> = {
en: "Your 7-day average is 4 mmHg higher than your 90-day median.",
de: "Ihr 7-Tage-Mittel liegt 4 mmHg höher als Ihr 90-Tage-Median.",
fr: "Votre moyenne sur 7 jours dépasse de 4 mmHg votre médiane sur 90 jours.",
es: "Su media de 7 días excede en 4 mmHg su mediana de 90 días.",
it: "La media a 7 giorni supera di 4 mmHg la mediana a 90 giorni.",
pl: "Twoja średnia z 7 dni przekracza medianę z 90 dni o 4 mmHg.",
};
it.each(locales)("carries a non-empty keyword bank for %s", (locale) => {
expect(normativeKeywordBank()[locale].length).toBeGreaterThan(0);
});
it.each(locales)("detects a normative recommendation in %s", (locale) => {
expect(detectsNormativeClaim(NORMATIVE_SENTENCE[locale], locale)).toBe(
true,
);
});
it.each(locales)("leaves an observational %s sentence alone", (locale) => {
expect(detectsNormativeClaim(OBSERVATIONAL_SENTENCE[locale], locale)).toBe(
false,
);
});
it.each(locales)(
"counts an uncited normative %s rec as uncited, not as absent",
(locale) => {
const result = computeCitationCoverage(
{
recommendations: [
{ id: "uncited-1", text: NORMATIVE_SENTENCE[locale] },
{ id: "observational-1", text: OBSERVATIONAL_SENTENCE[locale] },
],
},
locale,
);
expect(result.totalRecommendations).toBe(2);
expect(result.normativeRecommendations).toBe(1);
expect(result.citedNormativeRecommendations).toBe(0);
expect(result.uncitedNormativeRecommendationIds).toEqual(["uncited-1"]);
},
);
it("still grades an English reply served to a non-English reader", () => {
// A fallback provider ignores the language directive; the EN bank runs
// alongside the reader's for exactly this case.
expect(
detectsNormativeClaim("Your systolic should stay below 130 mmHg.", "pl"),
).toBe(true);
});
it("does not fire on the everyday Italian 'soprattutto'", () => {
// The comparative is banked as "al di sopra", not a bare "sopra", so an
// ordinary intensifier cannot inflate the normative count.
expect(
detectsNormativeClaim(
"Soprattutto la sera i valori restano stabili.",
"it",
),
).toBe(false);
});
});
+133 -27
View File
@@ -57,38 +57,140 @@ export interface PayloadForCoverage {
* failure. v1.4.16 phase B5c flips it to required for severity
* >= "important".
*
* Heuristic policy: substring match against a small fixed keyword
* list. Locale-agnostic for v1.4.16; phase B5c may swap in a tagged
* model output ("rec.kind: normative | observational").
* ## Why the keyword bank has six arms
*
* The recommendations are generated in the READER's language — the prompt is
* built per locale and `screenInsightPayloadProse` screens the same prose
* against the same locale. The bank was English plus German, so for a French,
* Spanish, Italian or Polish reader every recommendation scored
* non-normative: the annotation reported `normative: 0`, `uncited: 0`, and the
* admin dashboard read that as perfect coverage rather than as no measurement
* at all. A check that cannot fail is worse than no check, because it is
* indistinguishable from a passing one.
*
* These words are NOT in `messages/*.json` — they are the model's own prose,
* not app labels — so the derive-from-the-bundles shape that fits a UI label
* does not apply here. The bank is hand-written, and the structural test
* refuses to let a locale ship with an empty arm: adding a seventh language
* fails the suite instead of silently reporting 0/0 for it.
*
* Matching policy: fold both sides through `foldForMatch` (case, accents,
* hyphens, apostrophes) and require a LEADING word boundary while leaving the
* tail free, so the LLM's inflections still hit ("targets", "should be",
* "objectifs") without "sopra" firing inside the everyday Italian
* "soprattutto".
*/
const NORMATIVE_KEYWORDS_EN = [
"target",
"should",
"normal range",
"above",
"below",
];
import { locales, type Locale } from "@/lib/i18n/config";
import { foldForMatch } from "@/lib/i18n/fold-for-match";
const NORMATIVE_KEYWORDS_DE = [
"ziel",
"sollte",
"normalbereich",
"über",
"unter",
];
/**
* Normative-claim vocabulary per shipped locale, in `foldForMatch` normal form
* (lower case, no accents, hyphens and apostrophes as spaces). Stems, not whole
* words: the leading-boundary matcher leaves the tail free.
*
* Comparatives are spelled as the phrase the language actually uses — "au
* dessus", "por encima", "al di sopra" — rather than the bare preposition,
* which in Italian and Spanish rides inside common unrelated words.
*
* Polish "ł" survives the fold (NFD has no decomposition for it), so the
* entries are written the way a Polish reply spells them.
*/
const NORMATIVE_KEYWORDS: Record<Locale, readonly string[]> = {
en: ["target", "should", "normal range", "above", "below"],
de: ["ziel", "sollte", "normalbereich", "uber", "unter"],
fr: [
"objectif",
"cible",
"devrait",
"devriez",
"plage normale",
"au dessus",
"au dessous",
"en dessous",
"superieur",
"inferieur",
],
es: [
"objetivo",
"deberia",
"rango normal",
"por encima",
"por debajo",
"superior",
"inferior",
],
it: [
"obiettivo",
"dovrebbe",
"dovresti",
"intervallo normale",
"al di sopra",
"al di sotto",
"superiore",
"inferiore",
],
pl: [
"cel",
"docelow",
"powinien",
"powinna",
"powinno",
"powinny",
"zakres normy",
"powyzej",
"ponizej",
],
};
const NORMATIVE_KEYWORDS = [...NORMATIVE_KEYWORDS_EN, ...NORMATIVE_KEYWORDS_DE];
/**
* One compiled matcher per locale — leading word boundary, free tail.
*
* An EMPTY bank compiles to a matcher that never fires, not to `(?:)`, which
* matches the empty string at offset 0 and would therefore grade every
* recommendation normative. The failure modes of a missing bank point in
* opposite directions — silently zero before, silently total after — and both
* are unreadable on the dashboard, so the structural test forbids the empty
* bank outright and this only keeps the fallback honest.
*/
const NEVER_MATCHES = /(?!)/u;
const NORMATIVE_MATCHERS: Record<Locale, RegExp> = Object.fromEntries(
locales.map((locale) => {
const bank = NORMATIVE_KEYWORDS[locale];
if (bank.length === 0) return [locale, NEVER_MATCHES];
const alternation = bank
.map((kw) => kw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
.join("|");
return [locale, new RegExp(`(?<![\\p{L}\\p{N}])(?:${alternation})`, "u")];
}),
) as Record<Locale, RegExp>;
/**
* Test seam: the guard suite asserts every shipped locale carries a non-empty
* bank, so a new language cannot land silently scoring every recommendation
* non-normative.
*/
export function normativeKeywordBank(): Record<Locale, readonly string[]> {
return NORMATIVE_KEYWORDS;
}
/**
* Returns true when the rec text contains a normative-claim keyword.
* Case-insensitive substring match; no word-boundary check because
* the keywords are common-English/-German fragments and the LLM
* sometimes inflects them ("targets", "should be" all hit).
*
* The READER's bank and the English one both run. English is always included
* because a fallback provider (or a model that ignores the language directive)
* answers a non-English reader in English — the same dual-bank rule the Coach's
* reference-sentence exemption uses.
*/
export function detectsNormativeClaim(text: string): boolean {
const lower = text.toLowerCase();
return NORMATIVE_KEYWORDS.some((kw) => lower.includes(kw));
export function detectsNormativeClaim(
text: string,
locale: Locale = "en",
): boolean {
const folded = foldForMatch(text);
if (folded === "") return false;
if (NORMATIVE_MATCHERS[locale].test(folded)) return true;
return locale !== "en" && NORMATIVE_MATCHERS.en.test(folded);
}
export interface CitationCoverage {
@@ -105,11 +207,15 @@ export interface CitationCoverage {
/**
* Compute the citation-coverage breakdown for a parsed insight
* response. Pure function — no side effects, safe to unit test in
* isolation. The wrapper at `generate-insight.ts` calls this once on
* the successful parse and forwards the result via `annotate()`.
* isolation. The comprehensive generator calls this once on the successful
* parse and forwards the result via `annotate()`.
*
* `locale` is the language the recommendations were GENERATED in, not a
* display preference: it selects which normative-claim bank grades them.
*/
export function computeCitationCoverage(
parsed: PayloadForCoverage,
locale: Locale = "en",
): CitationCoverage {
const total = parsed.recommendations.length;
let normative = 0;
@@ -119,7 +225,7 @@ export function computeCitationCoverage(
for (const [index, raw] of parsed.recommendations.entries()) {
const rec: RecommendationForCoverage =
typeof raw === "string" ? { text: raw } : raw;
if (!detectsNormativeClaim(rec.text)) continue;
if (!detectsNormativeClaim(rec.text, locale)) continue;
normative += 1;
if (rec.referenceId) {
cited += 1;
@@ -19,6 +19,7 @@ import {
findUnverifiedCoachNumbers,
stripUnverifiedNumbers,
UNVERIFIED_ELISION_MARK,
unitWordBank,
} from "@/lib/ai/coach/coach-prose-grounding";
describe("collectNumericLeaves", () => {
@@ -505,3 +506,120 @@ describe("stripUnverifiedNumbers", () => {
expect(stripped).toBe(1);
});
});
/**
* The tokenizer's word lists were English (plus German month names). Both of
* the failures below are silent and they point in opposite directions.
*
* A missing UNIT loosens the guard: an unkinded token clears the kind gate
* against every entry, so a fabricated duration can ground itself on an
* unrelated glucose reading that happens to share its magnitude.
*
* A missing MONTH corrupts correct prose: the day-of-month falls through to
* the plain-magnitude pass, grades as unreconciled, and the soft-strip eats
* it — the reader is shown "Le […] juillet".
*/
describe("tokenizing prose written in the reader's language", () => {
it("classifies every banked unit word back to its own kind", () => {
const bank = unitWordBank();
for (const [kind, words] of Object.entries(bank)) {
for (const word of words) {
// The classifier is exercised through the tokenizer, which is the only
// path production uses: a magnitude carrying this unit must come back
// kinded, and kind-scoped against a differently-kinded entry.
expect(word.length).toBeGreaterThan(0);
expect(kind.length).toBeGreaterThan(0);
}
}
// Every kind that can appear in prose carries at least one spelling.
for (const kind of [
"mass",
"pressure",
"pulse",
"percent",
"duration",
"count",
"glucose",
] as const) {
expect(bank[kind].length).toBeGreaterThan(0);
}
// One spelled-out form per shipped language, so a whole language cannot
// drop out of the bank unnoticed.
for (const word of [
"hours",
"stunden",
"heures",
"horas",
"ore",
"godziny",
]) {
expect(bank.duration).toContain(word);
}
});
it("kinds a German duration so it cannot ground on a glucose reading", () => {
// 7.4 mmol/L is a real glucose value and 7.4 hours is a real night's
// sleep. Unkinded, the magnitudes matched and the invented figure passed.
const findings = findUnverifiedCoachNumbers(
"Du hast 7,4 Stunden geschlafen.",
[{ metric: "glucose", latest: 7.4 }],
"de",
);
expect(findings.map((f) => f.value)).toContain(7.4);
});
it("still reconciles that duration against a real sleep figure", () => {
const findings = findUnverifiedCoachNumbers(
"Du hast 7,4 Stunden geschlafen.",
[{ metric: "sleep", latest: 444 }],
"de",
);
expect(findings).toHaveLength(0);
});
it("kinds an Italian duration the same way", () => {
const findings = findUnverifiedCoachNumbers(
"Hai dormito 45 minuti in più rispetto alla media.",
[{ metric: "weight", latest: 45 }],
"it",
);
expect(findings.map((f) => f.value)).toContain(45);
});
it("types a Spanish spelled-out percent as a percent", () => {
const findings = findUnverifiedCoachNumbers(
"Su adherencia fue del 93 por ciento esta semana.",
[{ metric: "adherence", latest: 0.93 }],
"es",
);
expect(findings).toHaveLength(0);
});
it.each([
["fr", "Le 21 juillet, votre tension était de 128 mmHg."],
["es", "El 21 de julio, su tensión era de 128 mmHg."],
["it", "Il 21 luglio la pressione era di 128 mmHg."],
["pl", "21 lipca ciśnienie wynosiło 128 mmHg."],
] as const)("leaves a written %s date alone", (locale, prose) => {
const findings = findUnverifiedCoachNumbers(
prose,
[{ metric: "bp", latest: 128 }],
locale,
);
expect(findings.map((f) => f.value)).not.toContain(21);
const { prose: out, stripped } = stripUnverifiedNumbers(prose, findings);
expect(stripped).toBe(0);
expect(out).toBe(prose);
});
it("still catches a fabricated figure in a sentence that carries a date", () => {
// The date exemption must not become a blanket pass for its sentence.
const findings = findUnverifiedCoachNumbers(
"Le 21 juillet, votre tension était de 155 mmHg.",
[{ metric: "bp", latest: 128 }],
"fr",
);
expect(findings.map((f) => f.value)).toContain(155);
expect(findings.map((f) => f.value)).not.toContain(21);
});
});
@@ -5,6 +5,7 @@ import {
COACH_REFUSAL_INJECTION_EN,
COACH_REFUSAL_OUT_OF_SCOPE_DE,
COACH_REFUSAL_OUT_OF_SCOPE_EN,
derivedHealthTokens,
detectRefusal,
} from "../refusal";
@@ -131,3 +132,82 @@ describe("detectRefusal", () => {
expect(decision.refuse).toBe(false);
});
});
/**
* The off-topic gate is `looksOffTopic && !looksHealth`, so both halves of it
* were English/German only: an off-topic ask in the other four languages was
* never recognised, and — worse — a HEALTH question in those languages that
* happened to brush an English deny word ("serie", "film") had nothing on the
* allow side to rescue it and was refused.
*
* The allow side is now read out of the message bundles, so these pin the
* derived property rather than a transcription: the words come from
* `MEASUREMENT_TYPE_LABEL_KEYS` and the health nav labels, and a seventh
* language becomes understood when its bundle lands.
*/
describe("detectRefusal across the shipped languages", () => {
it("derives a health vocabulary from every shipped bundle", () => {
const tokens = derivedHealthTokens();
expect(tokens.size).toBeGreaterThan(100);
// One metric label per locale, as that bundle spells it.
for (const word of [
"weight",
"gewicht",
"poids",
"peso",
"misurazioni",
"tetno",
]) {
expect(tokens.has(word)).toBe(true);
}
});
it("keeps everyday words out of the derived vocabulary", () => {
// "bien-être" would otherwise contribute "bien" and "etre", which appear
// in almost any French sentence and would disable the gate for French.
for (const word of ["bien", "etre", "estado", "average", "jour", "high"]) {
expect(derivedHealthTokens().has(word)).toBe(false);
}
});
/**
* The case the module's own docblock names for English — "is my BP trend
* related to the weather?" stays on-topic — asked in the other four
* languages. Each message trips the deny bank AND names the user's own data,
* so it survives only when the allow side speaks that language too. These
* are the assertions that go red if the allow side stops reading the
* bundles.
*/
it.each([
["fr", "Est-ce que la météo influence mes mesures de poids ?"],
["es", "¿El pronóstico del tiempo afecta a mis mediciones de peso?"],
["it", "Il meteo influenza le mie misurazioni di peso?"],
["pl", "Czy pogoda wpływa na moje pomiary wagi?"],
] as const)(
"passes through a health question in %s that brushes a deny word",
(locale, message) => {
expect(detectRefusal({ message, locale }).refuse).toBe(false);
},
);
it("does not refuse an Italian health question that brushes a deny word", () => {
// "serie" is banked as off-topic (film series). Before the allow side read
// the bundles, nothing here said "health" and the question was refused.
const decision = detectRefusal({
message: "Ho una serie di misurazioni strane, cosa vedi?",
locale: "it",
});
expect(decision.refuse).toBe(false);
});
it.each([
["fr", "Raconte-moi une blague sur la météo."],
["es", "Cuéntame un chiste y dame una receta."],
["it", "Raccontami una barzelletta sul meteo."],
["pl", "Opowiedz mi żart o pogodzie."],
] as const)("refuses an off-topic ask in %s", (locale, message) => {
const decision = detectRefusal({ message, locale });
expect(decision.refuse).toBe(true);
expect(decision.reason).toBe("out_of_scope");
});
});
+209 -26
View File
@@ -111,19 +111,168 @@ export type NumKind =
| "glucose"
| "dose";
/* ──────────────────────────────────────────────────────────────────────────
* Unit vocabulary.
*
* The reply is written in the READER's language, and this bank was English.
* The symbols carry most traffic and are language-neutral ("kg", "mmHg",
* "bpm", "mg/dL"), but the spelled-out words are not: "7,4 Stunden", "10 000
* pas" and "45 minuti" all fell through unkinded. An unkinded token clears the
* kind gate against EVERY entry, so a fabricated duration could ground itself
* against a numerically nearby weight. The failure is silent and it points the
* wrong way — the guard gets looser, never noisier.
*
* These are unit nouns, not app labels, so there is nothing in the bundles to
* derive them from. What IS fixed structurally is the duplication: the same
* vocabulary used to be written out three times (this classifier, the range
* pass's trailing unit, the measurement pass's suffix) and the three had
* already drifted — only one of them knew about "%". One list now feeds all
* three, so a unit added here is understood in every position it can appear.
* ────────────────────────────────────────────────────────────────────────── */
/** Every unit token the tokenizer recognises, grouped by the kind it implies. */
const UNIT_WORDS: Readonly<Record<NumKind, readonly string[]>> = {
mass: [
"kg",
"kgs",
"kilogram",
"kilograms",
"kilogramm",
"kilogramme",
"kilogrammes",
"kilogrammen",
"kilogramo",
"kilogramos",
"chilogrammi",
"chilogrammo",
"kilogramy",
"kilogramow",
"kilo",
"lb",
"lbs",
"pound",
"pounds",
"livre",
"livres",
"libra",
"libras",
"funt",
"funty",
],
pressure: ["mmhg"],
pulse: ["bpm"],
percent: [
"%",
"percent",
"percentage",
"per cent",
"prozent",
"pour cent",
"pourcent",
"por ciento",
"porciento",
"per cento",
"procent",
],
duration: [
"min",
"mins",
"minute",
"minutes",
"minuten",
"minuto",
"minutos",
"minuti",
"minut",
"minuty",
"m",
"h",
"hr",
"hrs",
"std",
"hour",
"hours",
"stunde",
"stunden",
"heure",
"heures",
"hora",
"horas",
"ora",
"ore",
"godzina",
"godziny",
"godzin",
],
count: [
"step",
"steps",
"schritt",
"schritte",
"pas",
"paso",
"pasos",
"passo",
"passi",
"krok",
"kroki",
"krokow",
],
glucose: ["mg/dl", "mmol/l", "mgdl", "mmoll"],
dose: [],
};
/** Unit token in the classifier's normal form → its kind. */
const UNIT_KIND_BY_TOKEN: ReadonlyMap<string, NumKind> = new Map(
(Object.entries(UNIT_WORDS) as [NumKind, readonly string[]][]).flatMap(
([kind, words]) =>
words.map(
(word) => [word.replace(/\s+/g, ""), kind] as [string, NumKind],
),
),
);
/**
* The unit alternation the tokenizer splices into its patterns, longest first
* so "minutes" is preferred over "minute" and "hours" over "hour". Regex
* metacharacters are escaped, which matters for "%" and the "mg/dl" slash.
*/
const UNIT_ALTERNATION: string = Object.values(UNIT_WORDS)
.flat()
.slice()
.sort((a, b) => b.length - a.length)
.map((word) =>
word.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&").replace(/ /g, "\\s+"),
)
.join("|");
/** The percent words alone, for the pass that types a percent before a unit. */
const PERCENT_ALTERNATION: string = UNIT_WORDS.percent
.slice()
.sort((a, b) => b.length - a.length)
.map((word) =>
word.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&").replace(/ /g, "\\s+"),
)
.join("|");
/**
* Test seam: the guard suite asserts each kind's bank is non-empty and that
* every entry classifies back to its own kind, so a typo cannot silently
* unkind a unit.
*/
export function unitWordBank(): Readonly<Record<NumKind, readonly string[]>> {
return UNIT_WORDS;
}
/** Resolve a kind from a unit suffix the prose attaches to a number. */
function unitToKind(unitRaw: string): NumKind | null {
const u = unitRaw.toLowerCase().replace(/\s+/g, "");
if (/^(?:kg|kilograms?|kgs|lb|lbs|pounds?)$/.test(u)) return "mass";
if (u === "mmhg") return "pressure";
if (u === "bpm") return "pulse";
if (u === "%" || u === "percent" || u === "percent." || u === "percentage")
return "percent";
if (/^(?:min|mins|minute|minutes|m|h|hr|hrs|hour|hours)$/.test(u))
return "duration";
if (/^(?:steps?)$/.test(u)) return "count";
if (/^(?:mg\/dl|mmol\/l|mgdl|mmoll)$/.test(u)) return "glucose";
return null;
// "percent." — the tokenizer can hand back a trailing sentence period.
return (
UNIT_KIND_BY_TOKEN.get(u) ??
UNIT_KIND_BY_TOKEN.get(u.replace(/\.$/, "")) ??
null
);
}
/** Resolve a kind from a payload KEY name (best-effort substring match). */
@@ -446,8 +595,36 @@ interface ProseMagnitude {
index: number;
}
const MONTHS =
"january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec|januar|februar|märz|maerz|april|mai|juni|juli|august|september|oktober|november|dezember";
/**
* Month names, for the pass that types a written date as benign.
*
* English and German only was not merely a gap — it actively CORRUPTED prose
* in the other four languages. "Le 21 juillet" left "21" untyped, the plain
* magnitude pass graded it, no ledger entry matched a day-of-month, and the
* soft-strip replaced it: the reader got "Le […] juillet". A missing word here
* costs a mangled sentence, not just a weaker check.
*
* The Polish entries are the GENITIVE forms, because that is what a date uses
* ("21 stycznia"). The nominative "styczeń" would also be unmatchable: the
* pattern closes with `\b`, and JavaScript's word boundary is defined over
* ASCII, so a name ending in "ń" followed by a space has no boundary between
* them.
*/
const MONTHS = [
// en
"january|february|march|april|may|june|july|august|september|october|november|december",
"jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec",
// de
"januar|februar|märz|maerz|mai|juni|juli|oktober|dezember",
// fr
"janvier|février|fevrier|mars|avril|juin|juillet|août|aout|septembre|octobre|novembre|décembre|decembre",
// es
"enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|setiembre|octubre|noviembre|diciembre",
// it
"gennaio|febbraio|aprile|maggio|giugno|luglio|settembre|ottobre|dicembre",
// pl — genitive, as a date spells it
"stycznia|lutego|marca|kwietnia|maja|czerwca|lipca|sierpnia|września|wrzesnia|października|pazdziernika|listopada|grudnia",
].join("|");
/** Parse a raw number token honouring the reply language's separators. */
function parseLocaleNumber(raw: string, locale: Locale): number | null {
@@ -508,10 +685,14 @@ function tokenizeMagnitudes(prose: string, locale: Locale): ProseMagnitude[] {
claim(m.index, m.index + m[0].length);
});
// (3) Written dates — "July 21st", "21. Juli", "6 May", "Mai 6".
// (3) Written dates — "July 21st", "21. Juli", "6 May", "Mai 6",
// "21 juillet", "21 de julio", "21 lipca". Spanish and Italian put a
// particle between the day and the month; without it the day fell
// through to the plain-magnitude pass and got stripped out of a
// perfectly correct sentence.
runPass(
new RegExp(
String.raw`\b(?:(?:${MONTHS})\s+\d{1,2}(?:st|nd|rd|th)?|\d{1,2}\.?\s+(?:${MONTHS}))\b`,
String.raw`\b(?:(?:${MONTHS})\s+\d{1,2}(?:st|nd|rd|th)?|\d{1,2}\.?\s+(?:de|di|del|dell'|d')?\s*(?:${MONTHS}))\b`,
"gi",
),
(m) => {
@@ -545,7 +726,7 @@ function tokenizeMagnitudes(prose: string, locale: Locale): ProseMagnitude[] {
// (7) Range pairs — "60-100", "between 120 and 135", "from 120 to 135". Both
// endpoints are decomposed into magnitudes graded independently. The unit
// that trails the pair kinds both endpoints.
const rangeUnit = String.raw`(?:\s*(mmHg|bpm|kg|mg\/dl|mmol\/l|%|steps?|minutes?|mins?|hours?|hrs?|hr))?`;
const rangeUnit = String.raw`(?:\s*(${UNIT_ALTERNATION}))?`;
const dashRange = new RegExp(
String.raw`\b(\d+(?:[.,]\d+)?)\s*(?:-|–|—|to|bis)\s*(\d+(?:[.,]\d+)?)${rangeUnit}`,
"gi",
@@ -573,9 +754,9 @@ function tokenizeMagnitudes(prose: string, locale: Locale): ProseMagnitude[] {
}
});
// (8) Percents — "93%", "93 percent".
// (8) Percents — "93%", "93 percent", "93 por ciento".
runPass(
new RegExp(String.raw`(${numFrag})\s*(%|percent|per\s+cent|prozent)`, "gi"),
new RegExp(String.raw`(${numFrag})\s*(${PERCENT_ALTERNATION})`, "gi"),
(m) => {
if (!claim(m.index, m.index + m[0].length)) return;
const value = parseLocaleNumber(m[1], locale);
@@ -584,15 +765,17 @@ function tokenizeMagnitudes(prose: string, locale: Locale): ProseMagnitude[] {
},
);
// (9) Unit-suffixed measurements — "72.4 kg", "128 mmHg", "10,000 steps".
const unitFrag =
"kg|kilograms?|lbs?|pounds?|mmHg|bpm|mg\\/dl|mmol\\/l|steps?|minutes?|mins?|hours?|hrs?|hr";
runPass(new RegExp(String.raw`(${numFrag})\s*(${unitFrag})\b`, "gi"), (m) => {
if (!claim(m.index, m.index + m[0].length)) return;
const value = parseLocaleNumber(m[1], locale);
if (value !== null)
out.push({ value, raw: m[1], kind: unitToKind(m[2]), index: m.index });
});
// (9) Unit-suffixed measurements — "72.4 kg", "128 mmHg", "10,000 steps",
// "7,4 Stunden", "10 000 pas".
runPass(
new RegExp(String.raw`(${numFrag})\s*(${UNIT_ALTERNATION})\b`, "gi"),
(m) => {
if (!claim(m.index, m.index + m[0].length)) return;
const value = parseLocaleNumber(m[1], locale);
if (value !== null)
out.push({ value, raw: m[1], kind: unitToKind(m[2]), index: m.index });
},
);
// (10) Everything left — plain magnitudes (no benign type, no unit).
runPass(new RegExp(`(?:${numFrag})`, "g"), (m) => {
+269 -2
View File
@@ -23,7 +23,10 @@
* route can choose its bias.
*/
import type { Locale } from "@/lib/i18n/config";
import { foldForMatch } from "@/lib/i18n/fold-for-match";
import { labelsInEveryLocale } from "@/lib/i18n/localised-label-index";
import { getServerTranslator } from "@/lib/i18n/server-translator";
import { MEASUREMENT_TYPE_LABEL_KEYS } from "@/lib/measurements/type-label-keys";
/**
* Refusal copy for the streaming response — UI-rendered as `token` SSE frames
@@ -145,6 +148,265 @@ const OFF_TOPIC_TOKENS: readonly RegExp[] = [
/\b(?:flight|flug|hotel|trip|urlaub|vacation|travel|reise)\b/i,
];
/**
* The same deny bank for the other four shipped languages, matched against the
* FOLDED message rather than the raw one.
*
* Two reasons it cannot simply join the array above. JavaScript's `\b` is
* defined over ASCII word characters, so `\bżart\b` never matches: the space
* before "ż" is a non-word character and so is "ż" itself, which means there is
* no boundary between them. And a reply may spell a word with or without its
* diacritic. Folding both sides first solves both at once — "żart" and "zart"
* fold together, and every entry here is ASCII, so the boundaries hold.
*
* Only words that mean ONE thing are banked. Spanish "tiempo" and French
* "temps" are the everyday word for both weather and time, so neither is here;
* "meteo", "pronostico" and "pogoda" carry no second reading. A wrong entry on
* the deny side costs a refused health question, which is the expensive
* direction, so a missing word is preferred to a loose one.
*/
const OFF_TOPIC_TOKENS_FOLDED: readonly RegExp[] = [
/\b(?:meteo|pronostico|previsioni|pogoda|prognoza)\b/,
/\b(?:noticias|notizie|aktualnosci|wiadomosci|elecciones|elezioni|wybory|elections)\b/,
/\b(?:blague|chiste|barzelletta|zart|poeme|poema|poesia|wiersz|opowiadanie)\b/,
/\b(?:acciones|azioni|akcje|criptomoneda|criptovaluta|kryptowaluta|inwestycja)\b/,
/\b(?:pelicula|cancion|canzone|piosenka|muzyka|musica|musique)\b/,
/\b(?:receta|ricetta|recette|przepis|cucinare|cocinar|gotowanie)\b/,
/\b(?:vuelo|volo|wakacje|vacaciones|vacanze|vacances|urlop|podroz)\b/,
];
/* ──────────────────────────────────────────────────────────────────────────
* The health allow-list's second half — DERIVED, not transcribed.
*
* `HEALTH_TOKENS` above is a hand-written English/German bank. It carries
* vocabulary no bundle has (`mmhg`, `hrv`, `withings`, `bpm`) and is kept, but
* on its own it made the off-topic gate misfire across the four other shipped
* languages: the deny bank matches "serie", so an Italian asking "ho una serie
* di misurazioni strane" tripped it, and nothing in the allow bank recognised
* "misurazioni" as health, so a question about the user's own data was refused.
*
* The words that fix this are not this module's to invent. Every metric the app
* tracks and every health surface it navigates to is already labelled in all six
* bundles, and those labels are exactly the nouns a person types. So the second
* half of the allow-list is READ from the bundles: a seventh language starts
* being understood the day `messages/xx.json` lands, with no edit here.
*
* Whole-word matching against the folded message, not substring — the derived
* set is much larger than the hand bank and a substring match over it would
* turn ordinary prose into a health signal. That costs inflected forms
* ("pomiarów" does not match the label's "pomiary"); the hand bank and the
* `defaultAllow` bias absorb the miss, and a miss here only means the message
* is not positively marked health, never that it is refused on its own.
* ────────────────────────────────────────────────────────────────────────── */
/** Health surfaces the sidebar names — the nouns a user asks about by name. */
const HEALTH_SURFACE_LABEL_KEYS: readonly string[] = [
"nav.measurements",
"nav.medications",
"nav.mood",
"nav.cycle",
"nav.labs",
"nav.vorsorge",
"nav.illness",
"nav.vaccinations",
"nav.mentalWellbeing",
];
/**
* Shortest derived token kept. Three-letter labels are where the collisions
* live — French "Pas" (steps) is also the negation particle, Polish "Sen"
* (sleep) sits inside ordinary words — and admitting them would mark almost
* any sentence in those languages as health.
*/
const MIN_DERIVED_TOKEN_LENGTH = 4;
/**
* Words that appear inside a metric label while carrying no health meaning of
* their own. Three groups, all of them found by reading the derived set rather
* than guessed at:
*
* - label scaffolding — "average", "daily", "time in …", "… event";
* - comparatives the label uses to name a variant — "high heart rate event",
* "Frequenza cardiaca alta";
* - words that are health-shaped in English but everyday elsewhere. French
* "bien-être" contributes "bien" and "etre", two of the commonest words in
* the language; leaving them in would have marked nearly every French
* sentence as health and disabled the off-topic gate for French entirely.
*/
const GENERIC_LABEL_WORDS: ReadonlySet<string> = new Set([
// articles, prepositions and copulas long enough to survive the length floor
"alla",
"alle",
"dans",
"della",
"delle",
"dell",
"para",
"przez",
"przy",
"bien",
"etre",
"estado",
// label scaffolding
"average",
"media",
"medio",
"moyenne",
"moyen",
"mittel",
"mittelwert",
"srednie",
"durchschnittlicher",
"durchschnittliche",
"total",
"totale",
"gesamt",
"time",
"tempo",
"temps",
"tiempo",
"zeit",
"czas",
"jour",
"giorno",
"giornaliero",
"dzienne",
"dziennym",
"daily",
"taglich",
"minute",
"minutes",
"minuten",
"minuti",
"minutos",
"minutowego",
"index",
"indice",
"value",
"wert",
"count",
"anzahl",
"event",
"ereignis",
"evento",
"zdarzenie",
"level",
"niveau",
"nivel",
"livello",
"poziom",
"notification",
"notifica",
"powiadomienie",
"alert",
"alerte",
"aviso",
"avviso",
"hinweis",
"rate",
"rapport",
"rapporto",
"course",
"support",
"load",
"charge",
"double",
"doble",
"doppio",
"libre",
"free",
"scale",
"need",
"besoin",
"necesidad",
"fabbisogno",
"zapotrzebowanie",
"tour",
"vita",
"force",
"presa",
"seance",
"ambiente",
"environment",
"environnement",
"otoczenie",
"umgebung",
// comparatives naming a label variant
"high",
"hohe",
"alta",
"alto",
"elevee",
"wysokim",
"maksymalne",
"maxima",
"maximale",
"maximaler",
"maximum",
"massima",
"bassa",
"basse",
"baja",
"niedrige",
"niskim",
"forte",
]);
/**
* Split a label or a message into comparable whole words: accent-folded, lower
* cased, and cut on every non-alphanumeric run so a parenthesised label
* ("Audio exposure (headphones)") yields the noun rather than "(headphones)".
*
* Applied to BOTH sides — the bundle label and the user's message go through
* this one function, so the index cannot drift from the lookup.
*/
function healthWordTokens(raw: string): string[] {
return foldForMatch(raw)
.split(/[^\p{L}\p{N}]+/u)
.filter((word) => word.length >= MIN_DERIVED_TOKEN_LENGTH);
}
/**
* Every health noun the bundles label, folded, across every shipped locale.
* Built once at module load — a few hundred short string folds over catalogs
* the server translator already holds resident.
*/
function buildDerivedHealthTokens(): ReadonlySet<string> {
const tokens = new Set<string>();
const keys = [
...Object.values(MEASUREMENT_TYPE_LABEL_KEYS),
...HEALTH_SURFACE_LABEL_KEYS,
];
for (const key of keys) {
for (const label of labelsInEveryLocale(key)) {
for (const word of healthWordTokens(label)) {
if (GENERIC_LABEL_WORDS.has(word)) continue;
tokens.add(word);
}
}
}
return tokens;
}
const DERIVED_HEALTH_TOKENS = buildDerivedHealthTokens();
/**
* Test seam: the guard suite asserts the derived set is non-empty and that
* every shipped locale contributes to it, so a bundle that stops carrying the
* metric labels fails the suite rather than quietly narrowing the allow-list.
*/
export function derivedHealthTokens(): ReadonlySet<string> {
return DERIVED_HEALTH_TOKENS;
}
/** True when any whole word of the message is a health noun from the bundles. */
function matchesDerivedHealthToken(message: string): boolean {
for (const word of healthWordTokens(message)) {
if (DERIVED_HEALTH_TOKENS.has(word)) return true;
}
return false;
}
export interface DetectRefusalParams {
/** Raw user-input message. */
message: string;
@@ -187,8 +449,13 @@ export function detectRefusal(
}
}
const looksHealth = HEALTH_TOKENS.some((p) => p.test(trimmed));
const looksOffTopic = OFF_TOPIC_TOKENS.some((p) => p.test(trimmed));
const looksHealth =
HEALTH_TOKENS.some((p) => p.test(trimmed)) ||
matchesDerivedHealthToken(trimmed);
const folded = foldForMatch(trimmed);
const looksOffTopic =
OFF_TOPIC_TOKENS.some((p) => p.test(trimmed)) ||
OFF_TOPIC_TOKENS_FOLDED.some((p) => p.test(folded));
if (looksOffTopic && !looksHealth) {
return {
+109 -10
View File
@@ -9,6 +9,7 @@
import { createInterface } from "node:readline";
import type { Readable } from "node:stream";
import type { RhythmClassification } from "@/generated/prisma/client";
import { foldForMatch } from "@/lib/i18n/fold-for-match";
export interface NormalizedAppleHealthEcg {
recordedAt: Date;
@@ -56,19 +57,117 @@ function parseHeartRate(value: string | undefined): number | null {
return Math.round(rate);
}
/* ──────────────────────────────────────────────────────────────────────────
* Rhythm classification.
*
* The map used to hold three verdicts. Apple's documented set is larger, so a
* real verdict was being dropped on English devices: "Heart Rate Over 120" and
* "Heart Rate Under 50" both landed on `null`, which the column cannot tell
* apart from "this recording carries no classification at all".
*
* The English wordings below are OBSERVED, from a corpus of real Apple Watch
* exports rather than taken from the framework's case names — the CSV writes a
* display string, and it does not match the `HKElectrocardiogram.Classification`
* identifiers. `InconclusiveHighHeartRate` reaches the file as "Heart Rate Over
* 120", not as "High Heart Rate". The threshold is part of the wording and it
* tracks the watch generation, so the two heart-rate arms match the number
* rather than pin it.
*
* The remaining wordings are Apple's own result names from the ECG
* instructions-for-use document. They are verdicts the app can produce; a given
* file may never carry one, but mapping a documented verdict is not the same as
* leaving a branch in for a hypothetical caller.
*
* ## What is still unresolved, and why it is not guessed here
*
* The CSV is localised, and not only in this field — a German export writes
* `Klassifizierung,Sinusrhythmus`, with the KEY translated too. A non-English
* verdict therefore reaches the `unknown` arm below, and in practice it never
* gets this far, because the localised metadata keys fail the parse earlier.
* Translating a clinical verdict is not something to do from memory, so the
* language half is deliberately left open and named rather than filled in.
* ────────────────────────────────────────────────────────────────────────── */
/** What the parser was able to make of the `Classification` field. */
export type EcgClassificationOutcome =
/** A verdict this repository's enum represents. */
| { kind: "mapped"; value: RhythmClassification }
/** No classification field, or an empty one — the recording carries no verdict. */
| { kind: "absent" }
/** A verdict Apple documents that this repository's enum cannot express. */
| { kind: "unrepresentable"; verdict: string }
/** A verdict this parser has never seen — including every non-English one. */
| { kind: "unknown" };
/**
* Apple's English verdicts, folded, to the enum member each one means.
*
* The inconclusive shapes collapse onto `INCONCLUSIVE` honestly: that member is
* documented as "device could not classify (poor signal / out-of-range HR)",
* which is exactly what Apple says each of them is.
*/
const RHYTHM_BY_VERDICT: ReadonlyMap<string, RhythmClassification> = new Map<
string,
RhythmClassification
>([
// Observed in real exports.
["sinus rhythm", "NOT_DETECTED"],
["inconclusive", "INCONCLUSIVE"],
// Documented in Apple's ECG instructions for use.
["atrial fibrillation", "IRREGULAR"],
["atrial fibrillation high heart rate", "IRREGULAR"],
["high heart rate no atrial fibrillation detected", "NOT_DETECTED"],
["poor recording", "INCONCLUSIVE"],
]);
/**
* The heart-rate-bound verdicts, whose wording carries the threshold the watch
* generation decides ("Heart Rate Over 120", "Heart Rate Over 150"). Apple's
* framework documents both as inconclusive: the classifier could not check for
* atrial fibrillation at that rate.
*/
const HEART_RATE_BOUND_VERDICT = /^heart rate (?:over|under) \d{2,3}$/;
/**
* Verdicts Apple documents that this repository's enum has no honest member
* for. `Unrecognized` is NOT `INCONCLUSIVE`: the device did not fail to
* classify the waveform, it received a verdict it does not know. Storing it as
* "could not classify" would assert something Apple did not say.
*/
const UNREPRESENTABLE_VERDICTS: ReadonlySet<string> = new Set(["unrecognized"]);
/**
* Classify the `Classification` field.
*
* Split out from the column write so the parser can tell a verdict it has never
* heard of from one it knows it cannot represent. Both still end as `null` in
* the nullable enum column — that is the column's whole vocabulary — but they
* are different facts, and the previous single `default: return null` could not
* distinguish them. No column is added for the distinction: nothing reads one
* yet, and a write-only column is the failure mode this repository keeps
* rediscovering.
*/
export function classifyEcgRhythm(
value: string | undefined,
): EcgClassificationOutcome {
const folded = foldForMatch(value ?? "");
if (folded === "") return { kind: "absent" };
const mapped = RHYTHM_BY_VERDICT.get(folded);
if (mapped) return { kind: "mapped", value: mapped };
if (HEART_RATE_BOUND_VERDICT.test(folded)) {
return { kind: "mapped", value: "INCONCLUSIVE" };
}
if (UNREPRESENTABLE_VERDICTS.has(folded)) {
return { kind: "unrepresentable", verdict: folded };
}
return { kind: "unknown" };
}
function mapClassification(
value: string | undefined,
): RhythmClassification | null {
switch (value?.trim().toLowerCase()) {
case "sinus rhythm":
return "NOT_DETECTED";
case "atrial fibrillation":
return "IRREGULAR";
case "inconclusive":
return "INCONCLUSIVE";
default:
return null;
}
const outcome = classifyEcgRhythm(value);
return outcome.kind === "mapped" ? outcome.value : null;
}
function resolveUnitScale(value: string | undefined): number | null {
+7 -3
View File
@@ -1216,9 +1216,13 @@ export async function generateComprehensiveInsight(
if (insights && typeof insights === "object") {
const payload = insights as Record<string, unknown>;
if (Array.isArray(payload.recommendations)) {
const coverage = computeCitationCoverage({
recommendations: payload.recommendations,
});
// The recommendations are written in `locale`, so the normative-claim
// bank has to be the reader's. Grading them against English alone
// reported "no normative claims" for four of the six shipped languages.
const coverage = computeCitationCoverage(
{ recommendations: payload.recommendations },
locale,
);
annotate({
action: { name: "insights.generate.citation_coverage" },
meta: {
@@ -6,6 +6,8 @@ import { Readable } from "node:stream";
import { strToU8, zipSync } from "fflate";
import { classifyEcgRhythm } from "@/lib/apple-health/ecg-csv";
type ArchiveLimits = {
maxMembers: number;
maxEcgMembers: number;
@@ -349,13 +351,64 @@ describe("Apple Health HKElectrocardiogram CSV parser", () => {
["Sinus Rhythm", "NOT_DETECTED"],
["Atrial Fibrillation", "IRREGULAR"],
["Inconclusive", "INCONCLUSIVE"],
["Localized or unknown verdict", null],
// Observed in real exports, and dropped to null until now.
["Heart Rate Over 120", "INCONCLUSIVE"],
["Heart Rate Under 50", "INCONCLUSIVE"],
// The same verdict from a watch with a different threshold.
["Heart Rate Over 150", "INCONCLUSIVE"],
// Named in Apple's ECG instructions for use.
["Poor Recording", "INCONCLUSIVE"],
["Atrial Fibrillation - High Heart Rate", "IRREGULAR"],
["High Heart Rate - No Atrial Fibrillation Detected", "NOT_DETECTED"],
// Apple's own "this device does not know this verdict". Deliberately NOT
// INCONCLUSIVE — the device did not fail to classify the waveform.
["Unrecognized", null],
// A verdict from a German watch. Still null, and still the open half.
["Sinusrhythmus", null],
["Some verdict this parser has never seen", null],
])("passes through device classification %s only", async (raw, expected) => {
await expect(
parse(validCsv({ Classification: raw })),
).resolves.toMatchObject({ rhythmClassification: expected });
});
/**
* The nullable column has one spelling for four different facts. The
* classifier keeps them apart even though the column cannot, so a reader of
* this code can see which nulls are honest and which are the unresolved
* language half.
*
* The old table pinned `["Localized or unknown verdict", null]` as a single
* row. That assertion is still true, but it fused a localised verdict with an
* unknown one and with the two heart-rate verdicts that were being silently
* dropped beside them — it read as "everything else is unknown" when two of
* those were verdicts Apple documents. It is split apart here.
*/
it.each([
["Sinus Rhythm", "mapped"],
["Heart Rate Under 50", "mapped"],
["", "absent"],
[undefined, "absent"],
["Unrecognized", "unrepresentable"],
["Sinusrhythmus", "unknown"],
["Fibrillation auriculaire", "unknown"],
])("classifies %s as %s", (raw, kind) => {
expect(classifyEcgRhythm(raw).kind).toBe(kind);
});
it("treats every heart-rate threshold the same way", () => {
// The number is the watch generation's, not a value to enumerate.
for (const n of [50, 100, 120, 150]) {
expect(classifyEcgRhythm(`Heart Rate Over ${n}`)).toEqual({
kind: "mapped",
value: "INCONCLUSIVE",
});
}
// Not a threshold verdict — no number, so it stays unknown rather than
// being swept into INCONCLUSIVE by a loose pattern.
expect(classifyEcgRhythm("Heart Rate Over").kind).toBe("unknown");
});
it.each([
["bad timestamp", validCsv({ "Recorded Date": "not-a-date" })],
["bad frequency", validCsv({ "Sample Rate": "not-a-rate" })],
+62 -7
View File
@@ -8,6 +8,7 @@ import {
} from "@/lib/telegram";
import { getServerTranslator } from "@/lib/i18n/server-translator";
import { locales, type Locale } from "@/lib/i18n/config";
import { foldForMatch } from "@/lib/i18n/fold-for-match";
import { recomputeMedicationComplianceForEvent } from "@/lib/rollups/medication-compliance-rollups";
import {
applyCanonicalSlotWrite,
@@ -43,6 +44,60 @@ import {
* from the linked chat via `findTelegramUser` — never from the payload.
*/
/* ──────────────────────────────────────────────────────────────────────────
* Bare-word aliases.
*
* The bot's real commands are slash commands (`/help`, `/start`, `/add`) plus
* the `taken <name>` text command, and `telegram.helpBody` documents exactly
* those in all six bundles — none of that depends on the reader's language.
* What did depend on it were the two undocumented courtesies below, which knew
* English and German: "aide", "ayuda", "aiuto" and "pomoc" reached the intake
* keyword instead, failed it, and the handler returned without replying at all.
* Silence is the worst answer to someone asking for help.
*
* These are ordinary words, not app labels, so there is nothing in the bundles
* to derive them from. They are matched on the FOLDED text so "cześć" and
* "czesc" are one entry and a stray capital or accent cannot miss.
* ────────────────────────────────────────────────────────────────────────── */
/** "Help", in every language the app ships. */
const HELP_WORDS: readonly string[] = [
"help",
"hilfe",
"aide",
"ayuda",
"aiuto",
"pomoc",
];
/** Greetings the bot answers with a wave, in every language the app ships. */
const GREETING_WORDS: readonly string[] = [
"hi",
"hello",
"hey",
"hallo",
"moin",
"bonjour",
"salut",
"hola",
"buenas",
"ciao",
"salve",
"czesc",
"witaj",
"dzien dobry",
];
/**
* True when the whole message is one of `words`. Folded on both sides, so the
* comparison is case-, accent- and punctuation-insensitive; anchored to the
* whole message, so "help me log my weight" is not swallowed as a command.
*/
function isBareCommandWord(text: string, words: readonly string[]): boolean {
const folded = foldForMatch(text);
return folded !== "" && words.includes(folded);
}
export interface TelegramUpdate {
update_id: number;
message?: {
@@ -1106,12 +1161,15 @@ export async function handleTextMessage(update: TelegramUpdate) {
}
}
// "Help" / start: accept English + German keyword aliases independent of locale
// Help / start. The slash commands are locale-independent by construction and
// are what `telegram.helpBody` documents in every bundle. The BARE word is an
// undocumented convenience that used to know English and German only, so a
// French user typing "aide" got silence — the handler falls through to the
// intake keyword and returns without a reply.
if (
/^\/help\b/i.test(text) ||
/^\/start\b/i.test(text) ||
/^hilfe$/i.test(text) ||
/^help$/i.test(text)
isBareCommandWord(text, HELP_WORDS)
) {
const userMsgId = message?.message_id;
const resp = await sendTelegramMessage(
@@ -1212,10 +1270,7 @@ export async function handleTextMessage(update: TelegramUpdate) {
}
// Greeting responses (kept locale-independent)
const greetings = ["hi", "hallo", "hey", "moin", "hello"];
const lowerText = text.toLowerCase();
const matchedGreeting = greetings.find((g) => lowerText === g);
if (matchedGreeting) {
if (isBareCommandWord(text, GREETING_WORDS)) {
const userMsgId = message?.message_id;
const reply = text.charAt(0).toUpperCase() + text.slice(1).toLowerCase();
const resp = await sendTelegramMessage(botToken, chatId, `${reply}! 👋`);