fix(01-20): classify safe assessment outcomes

- add a discriminated generated and fallback contract
- screen derived prose and expose retryable cache policy
This commit is contained in:
Marc-André Bombeck committed 2026-07-29 14:36:29 +02:00
1 parent 0ae9165aae
commit efe203212a
3 files changed
+226 -37

No files matched your search

@@ -31,10 +31,10 @@ import {
withOutputLanguage,
} from "@/lib/ai/prompts/output-language";
import {
extractAssessmentSummary,
finalizeStatusSummary,
normalizeLocale,
normalizeSummaryText,
persistStatusInsight,
type AssessmentStatus,
} from "@/lib/insights/status-shared";
import { runStatusCompletion } from "@/lib/insights/status-provider";
import {
@@ -227,11 +227,19 @@ export async function generateDerivedScoreAssessment(args: {
derived: Derived<unknown>;
locale: string | null | undefined;
now?: Date;
}): Promise<void> {
}): Promise<AssessmentStatus | null> {
const now = args.now ?? new Date();
const locale = normalizeLocale(args.locale);
if (!isAssessableDerivedScore(args.metric)) return;
if (args.derived.status !== "ok") return;
if (!isAssessableDerivedScore(args.metric)) return null;
if (args.derived.status !== "ok") return null;
const deterministic = resolveDeterministicAssessment(
args.metric,
args.derived,
instructionLocale(locale),
now,
);
if (!deterministic) return null;
// The deterministic signal labels ship de/en templates only; route them
// through the same instruction-body rule rather than a German default.
@@ -240,7 +248,7 @@ export async function generateDerivedScoreAssessment(args: {
args.derived.value,
instructionLocale(locale),
);
if (!signal) return;
if (!signal) return null;
const band = (args.derived.value as { band?: string }).band ?? "yellow";
const scope = derivedScoreScope(args.metric);
@@ -280,7 +288,13 @@ export async function generateDerivedScoreAssessment(args: {
action: { name: "insights.derived-assessment.input_unchanged" },
meta: { metric: args.metric },
});
return;
return {
kind: "generated",
text: unchangedInput.text,
hasProvider: true,
updatedAt: unchangedInput.updatedAt,
retryable: false,
};
}
const outcome = await runStatusCompletion({
@@ -308,23 +322,53 @@ export async function generateDerivedScoreAssessment(args: {
action: { name: "insights.derived-assessment.skip" },
meta: { metric: args.metric, reason: outcome.kind },
});
return;
if (outcome.kind === "none") {
return {
kind: "no-provider",
text: null,
hasProvider: false,
updatedAt: null,
retryable: false,
};
}
return {
kind: outcome.kind,
text: null,
hasProvider: true,
updatedAt: null,
retryable: true,
};
}
// A malformed / truncated / summary-less structured envelope must never
// persist as the derived-score assessment (the raw-JSON class of bug). On an
// unparseable envelope persist nothing — the route keeps serving the
// always-grounded deterministic fallback, so the field stays honest.
const extracted = extractAssessmentSummary(outcome.content);
if (extracted.kind === "unparseable") {
// Parse and screen through the same fail-closed chokepoint as every other
// assessment. The prior derived-only path parsed the envelope but skipped
// the causal/dose/fabrication screen, allowing blocked prose to persist.
const finalized = finalizeStatusSummary(outcome.content, locale);
if (!finalized.ok) {
annotate({
action: { name: "insights.derived-assessment.unparseable" },
meta: { metric: args.metric },
action: { name: "insights.derived-assessment.screened" },
meta: { metric: args.metric, reason: finalized.reason },
});
return;
return {
kind: "screened-fallback",
text: deterministic.text,
hasProvider: true,
updatedAt: null,
retryable: true,
reason: finalized.reason,
};
}
const text = finalized.text;
if (!text) {
return {
kind: "screened-fallback",
text: deterministic.text,
hasProvider: true,
updatedAt: null,
retryable: true,
reason: "unparseable_output",
};
}
const text = normalizeSummaryText(extracted.text);
if (!text) return;
// v1.22 (W6) — score number-grounding gate. The only numbers the prose may
// state are the score and its contributor values. On a miss we do NOT persist
@@ -341,10 +385,17 @@ export async function generateDerivedScoreAssessment(args: {
sample: ungrounded[0]?.source,
},
});
return;
return {
kind: "screened-fallback",
text: deterministic.text,
hasProvider: true,
updatedAt: null,
retryable: true,
reason: "ungrounded_output",
};
}
await persistStatusInsight({
const updatedAt = await persistStatusInsight({
userId: args.userId,
cacheAction,
todayKey,
@@ -361,4 +412,11 @@ export async function generateDerivedScoreAssessment(args: {
action: { name: "insights.derived-assessment.warmed" },
meta: { metric: args.metric, provider: outcome.providerType },
});
return {
kind: "generated",
text,
hasProvider: true,
updatedAt,
retryable: false,
};
}
+64 -10
View File
@@ -88,6 +88,14 @@ interface ParsedStatusCache {
timeout?: boolean;
/** v1.8.3 — ISO timestamp before which a timeout stub suppresses re-enqueue. */
retryAt?: string;
/** Explicit generated/negative cache classification for new rows. */
statusKind?: "generated" | "negative";
/** Whether the producer may be retried at the time this row is read. */
retryable?: boolean;
/** Generated rows expire once the caller's current day differs. */
expiresAfterDateKey?: string;
/** Stable non-sensitive negative-cache reason code. */
reason?: "timeout" | "error" | "screened";
/** v1.16.8 — fingerprint of the data snapshot the text was generated from. */
snapshotHash?: string;
/** v1.18.11 (P6) — cheap fingerprint of the salient inputs (count + newest). */
@@ -107,8 +115,11 @@ export function isTimeoutStub(parsed: {
}
export interface FreshStatusCacheHit {
kind: "generated";
text: string;
updatedAt: string;
retryable: false;
expiresAfterDateKey: string;
}
/**
@@ -145,8 +156,11 @@ export async function readFreshStatusText(args: {
return null;
}
return {
kind: "generated",
text: parsed.text,
updatedAt: latestCache.createdAt.toISOString(),
retryable: false,
expiresAfterDateKey: parsed.expiresAfterDateKey ?? parsed.dateKey,
};
} catch {
// Malformed cache payload — treat as a miss and regenerate.
@@ -230,7 +244,13 @@ export async function refreshUnchangedStatusInsight(args: {
action: { name: "insights.status.skipped_unchanged" },
meta: { cache_action: args.cacheAction },
});
return { text: parsed.text, updatedAt: created.createdAt.toISOString() };
return {
kind: "generated",
text: parsed.text,
updatedAt: created.createdAt.toISOString(),
retryable: false,
expiresAfterDateKey: args.todayKey,
};
}
/**
@@ -433,7 +453,13 @@ export async function gateUnchangedStatusInput(args: {
action: { name: "insights.status.skipped_unchanged" },
meta: { cache_action: args.cacheAction, gate: "input" },
});
return { text: parsed.text, updatedAt: created.createdAt.toISOString() };
return {
kind: "generated",
text: parsed.text,
updatedAt: created.createdAt.toISOString(),
retryable: false,
expiresAfterDateKey: args.todayKey,
};
}
export interface LastGoodStatusHit {
@@ -566,7 +592,11 @@ export async function resolveReadOnlyStatusMiss(args: {
// navigation while the provider is still degraded would be a storm. While
// the stub is fresh, stay in `preparing` without enqueuing; once it goes
// stale (or never existed) enqueue a fresh generation.
if (await hasFreshTimeoutStub({ userId: args.userId, cacheAction })) {
const negativeCache = await readStatusNegativeCache({
userId: args.userId,
cacheAction,
});
if (negativeCache && !negativeCache.retryable) {
annotate({
action: { name: "insights.status.preparing" },
meta: { metric: args.metric, suppressed_enqueue: true },
@@ -602,22 +632,46 @@ export async function resolveReadOnlyStatusMiss(args: {
* degraded. A stub without `retryAt` (legacy) is treated as stale so the
* resolver retries, matching the pre-v1.8.3 "transient miss" behaviour.
*/
async function hasFreshTimeoutStub(args: {
export interface StatusNegativeCache {
kind: "negative";
reason: "timeout" | "error" | "screened";
retryAt: string;
retryable: boolean;
}
/**
* Read the latest negative-cache policy without exposing provider errors or
* assessment content. `retryable` flips only when the explicit `retryAt`
* boundary has passed; legacy/malformed stubs are treated as absent so the
* next request may recover.
*/
export async function readStatusNegativeCache(args: {
userId: string;
cacheAction: string;
}): Promise<boolean> {
}): Promise<StatusNegativeCache | null> {
const latest = await prisma.auditLog.findFirst({
where: { userId: args.userId, action: args.cacheAction },
orderBy: { createdAt: "desc" },
select: { details: true },
});
if (!latest?.details) return false;
if (!latest?.details) return null;
try {
const parsed = JSON.parse(latest.details) as ParsedStatusCache;
if (!isTimeoutStub(parsed)) return false;
if (typeof parsed.retryAt !== "string") return false;
return new Date(parsed.retryAt).getTime() > Date.now();
if (!isTimeoutStub(parsed)) return null;
if (typeof parsed.retryAt !== "string") return null;
const retryAtMs = new Date(parsed.retryAt).getTime();
if (!Number.isFinite(retryAtMs)) return null;
const reason =
parsed.reason === "error" || parsed.reason === "screened"
? parsed.reason
: "timeout";
return {
kind: "negative",
reason,
retryAt: parsed.retryAt,
retryable: retryAtMs <= Date.now(),
};
} catch {
return false;
return null;
}
}
+83 -6
View File
@@ -37,6 +37,65 @@ export type SupportedLocale = Locale;
const SUPPORTED_LOCALES: ReadonlySet<string> = new Set(locales);
/**
* The presentation contract shared by assessment producers and consumers.
*
* `kind` is the only branch discriminator. In particular, provider
* availability is metadata: it must never hide safe deterministic text.
* Provider/error details deliberately do not belong in this envelope.
*/
export type AssessmentStatus =
| {
kind: "generated";
text: string;
hasProvider: true;
updatedAt: string | null;
retryable: false;
}
| {
kind: "screened-fallback";
text: string;
hasProvider: boolean;
updatedAt: null;
retryable: true;
reason: OutboundReason | "unparseable_output" | "ungrounded_output";
}
| {
kind: "no-provider";
text: null;
hasProvider: false;
updatedAt: null;
retryable: false;
}
| {
kind: "preparing";
text: null;
hasProvider: true;
updatedAt: null;
retryable: true;
}
| {
kind: "revalidating";
text: string;
hasProvider: true;
updatedAt: string | null;
retryable: true;
}
| {
kind: "timeout" | "error";
text: null;
hasProvider: true;
updatedAt: null;
retryable: true;
}
| {
kind: "exhausted";
text: null;
hasProvider: true;
updatedAt: null;
retryable: false;
};
/** Round to `digits` decimal places. */
export function round(value: number, digits = 1): number {
const factor = 10 ** digits;
@@ -305,6 +364,12 @@ export async function persistStatusInsight(args: {
providerType: args.providerType,
model: args.model,
tokensUsed: args.tokensUsed,
statusKind: "generated",
retryable: false,
// Freshness is defined by the user's calendar day. Keeping the
// boundary in the payload makes expiry explicit without inventing a
// UTC timestamp for a day key that belongs to another timezone.
expiresAfterDateKey: args.todayKey,
...(args.snapshotHash ? { snapshotHash: args.snapshotHash } : {}),
...(args.inputHash ? { inputHash: args.inputHash } : {}),
}),
@@ -320,8 +385,12 @@ export async function persistStatusInsight(args: {
* withholds.
*/
export type StatusSummaryOutcome =
| { ok: true; text: string }
| { ok: false; reason: OutboundReason | "unparseable_output" };
| { ok: true; kind: "generated"; text: string }
| {
ok: false;
kind: "screened-fallback";
reason: OutboundReason | "unparseable_output";
};
/**
* The single finalize step every per-metric / biomarker / derived-score status
@@ -359,13 +428,21 @@ export function finalizeStatusSummary(
// text to render. WITHHOLD — the caller serves its deterministic stub. Only
// the `unparseable` discriminator is logged, never the assessment content.
if (extracted.kind === "unparseable") {
return { ok: false, reason: "unparseable_output" };
return {
ok: false,
kind: "screened-fallback",
reason: "unparseable_output",
};
}
const text = normalizeSummaryText(extracted.text);
if (!text) return { ok: true, text: "" };
if (!text) return { ok: true, kind: "generated", text: "" };
const decision = screenModelOutput(text, locale, INSIGHTS_CONTRACTS);
if (decision.block && decision.reason) {
return { ok: false, reason: decision.reason };
return {
ok: false,
kind: "screened-fallback",
reason: decision.reason,
};
}
return { ok: true, text };
return { ok: true, kind: "generated", text };
}