mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
fix(01-20): classify safe assessment outcomes
- add a discriminated generated and fallback contract - screen derived prose and expose retryable cache policy
This commit is contained in:
1 parent
0ae9165aae
commit
efe203212a
3 files changed
+226
-37
No files matched your search
@@ -31,10 +31,10 @@ import {
|
||||
withOutputLanguage,
|
||||
} from "@/lib/ai/prompts/output-language";
|
||||
import {
|
||||
extractAssessmentSummary,
|
||||
finalizeStatusSummary,
|
||||
normalizeLocale,
|
||||
normalizeSummaryText,
|
||||
persistStatusInsight,
|
||||
type AssessmentStatus,
|
||||
} from "@/lib/insights/status-shared";
|
||||
import { runStatusCompletion } from "@/lib/insights/status-provider";
|
||||
import {
|
||||
@@ -227,11 +227,19 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
derived: Derived<unknown>;
|
||||
locale: string | null | undefined;
|
||||
now?: Date;
|
||||
}): Promise<void> {
|
||||
}): Promise<AssessmentStatus | null> {
|
||||
const now = args.now ?? new Date();
|
||||
const locale = normalizeLocale(args.locale);
|
||||
if (!isAssessableDerivedScore(args.metric)) return;
|
||||
if (args.derived.status !== "ok") return;
|
||||
if (!isAssessableDerivedScore(args.metric)) return null;
|
||||
if (args.derived.status !== "ok") return null;
|
||||
|
||||
const deterministic = resolveDeterministicAssessment(
|
||||
args.metric,
|
||||
args.derived,
|
||||
instructionLocale(locale),
|
||||
now,
|
||||
);
|
||||
if (!deterministic) return null;
|
||||
|
||||
// The deterministic signal labels ship de/en templates only; route them
|
||||
// through the same instruction-body rule rather than a German default.
|
||||
@@ -240,7 +248,7 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
args.derived.value,
|
||||
instructionLocale(locale),
|
||||
);
|
||||
if (!signal) return;
|
||||
if (!signal) return null;
|
||||
const band = (args.derived.value as { band?: string }).band ?? "yellow";
|
||||
|
||||
const scope = derivedScoreScope(args.metric);
|
||||
@@ -280,7 +288,13 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
action: { name: "insights.derived-assessment.input_unchanged" },
|
||||
meta: { metric: args.metric },
|
||||
});
|
||||
return;
|
||||
return {
|
||||
kind: "generated",
|
||||
text: unchangedInput.text,
|
||||
hasProvider: true,
|
||||
updatedAt: unchangedInput.updatedAt,
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
|
||||
const outcome = await runStatusCompletion({
|
||||
@@ -308,23 +322,53 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
action: { name: "insights.derived-assessment.skip" },
|
||||
meta: { metric: args.metric, reason: outcome.kind },
|
||||
});
|
||||
return;
|
||||
if (outcome.kind === "none") {
|
||||
return {
|
||||
kind: "no-provider",
|
||||
text: null,
|
||||
hasProvider: false,
|
||||
updatedAt: null,
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: outcome.kind,
|
||||
text: null,
|
||||
hasProvider: true,
|
||||
updatedAt: null,
|
||||
retryable: true,
|
||||
};
|
||||
}
|
||||
|
||||
// A malformed / truncated / summary-less structured envelope must never
|
||||
// persist as the derived-score assessment (the raw-JSON class of bug). On an
|
||||
// unparseable envelope persist nothing — the route keeps serving the
|
||||
// always-grounded deterministic fallback, so the field stays honest.
|
||||
const extracted = extractAssessmentSummary(outcome.content);
|
||||
if (extracted.kind === "unparseable") {
|
||||
// Parse and screen through the same fail-closed chokepoint as every other
|
||||
// assessment. The prior derived-only path parsed the envelope but skipped
|
||||
// the causal/dose/fabrication screen, allowing blocked prose to persist.
|
||||
const finalized = finalizeStatusSummary(outcome.content, locale);
|
||||
if (!finalized.ok) {
|
||||
annotate({
|
||||
action: { name: "insights.derived-assessment.unparseable" },
|
||||
meta: { metric: args.metric },
|
||||
action: { name: "insights.derived-assessment.screened" },
|
||||
meta: { metric: args.metric, reason: finalized.reason },
|
||||
});
|
||||
return;
|
||||
return {
|
||||
kind: "screened-fallback",
|
||||
text: deterministic.text,
|
||||
hasProvider: true,
|
||||
updatedAt: null,
|
||||
retryable: true,
|
||||
reason: finalized.reason,
|
||||
};
|
||||
}
|
||||
const text = finalized.text;
|
||||
if (!text) {
|
||||
return {
|
||||
kind: "screened-fallback",
|
||||
text: deterministic.text,
|
||||
hasProvider: true,
|
||||
updatedAt: null,
|
||||
retryable: true,
|
||||
reason: "unparseable_output",
|
||||
};
|
||||
}
|
||||
const text = normalizeSummaryText(extracted.text);
|
||||
if (!text) return;
|
||||
|
||||
// v1.22 (W6) — score number-grounding gate. The only numbers the prose may
|
||||
// state are the score and its contributor values. On a miss we do NOT persist
|
||||
@@ -341,10 +385,17 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
sample: ungrounded[0]?.source,
|
||||
},
|
||||
});
|
||||
return;
|
||||
return {
|
||||
kind: "screened-fallback",
|
||||
text: deterministic.text,
|
||||
hasProvider: true,
|
||||
updatedAt: null,
|
||||
retryable: true,
|
||||
reason: "ungrounded_output",
|
||||
};
|
||||
}
|
||||
|
||||
await persistStatusInsight({
|
||||
const updatedAt = await persistStatusInsight({
|
||||
userId: args.userId,
|
||||
cacheAction,
|
||||
todayKey,
|
||||
@@ -361,4 +412,11 @@ export async function generateDerivedScoreAssessment(args: {
|
||||
action: { name: "insights.derived-assessment.warmed" },
|
||||
meta: { metric: args.metric, provider: outcome.providerType },
|
||||
});
|
||||
return {
|
||||
kind: "generated",
|
||||
text,
|
||||
hasProvider: true,
|
||||
updatedAt,
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
@@ -88,6 +88,14 @@ interface ParsedStatusCache {
|
||||
timeout?: boolean;
|
||||
/** v1.8.3 — ISO timestamp before which a timeout stub suppresses re-enqueue. */
|
||||
retryAt?: string;
|
||||
/** Explicit generated/negative cache classification for new rows. */
|
||||
statusKind?: "generated" | "negative";
|
||||
/** Whether the producer may be retried at the time this row is read. */
|
||||
retryable?: boolean;
|
||||
/** Generated rows expire once the caller's current day differs. */
|
||||
expiresAfterDateKey?: string;
|
||||
/** Stable non-sensitive negative-cache reason code. */
|
||||
reason?: "timeout" | "error" | "screened";
|
||||
/** v1.16.8 — fingerprint of the data snapshot the text was generated from. */
|
||||
snapshotHash?: string;
|
||||
/** v1.18.11 (P6) — cheap fingerprint of the salient inputs (count + newest). */
|
||||
@@ -107,8 +115,11 @@ export function isTimeoutStub(parsed: {
|
||||
}
|
||||
|
||||
export interface FreshStatusCacheHit {
|
||||
kind: "generated";
|
||||
text: string;
|
||||
updatedAt: string;
|
||||
retryable: false;
|
||||
expiresAfterDateKey: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,8 +156,11 @@ export async function readFreshStatusText(args: {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
kind: "generated",
|
||||
text: parsed.text,
|
||||
updatedAt: latestCache.createdAt.toISOString(),
|
||||
retryable: false,
|
||||
expiresAfterDateKey: parsed.expiresAfterDateKey ?? parsed.dateKey,
|
||||
};
|
||||
} catch {
|
||||
// Malformed cache payload — treat as a miss and regenerate.
|
||||
@@ -230,7 +244,13 @@ export async function refreshUnchangedStatusInsight(args: {
|
||||
action: { name: "insights.status.skipped_unchanged" },
|
||||
meta: { cache_action: args.cacheAction },
|
||||
});
|
||||
return { text: parsed.text, updatedAt: created.createdAt.toISOString() };
|
||||
return {
|
||||
kind: "generated",
|
||||
text: parsed.text,
|
||||
updatedAt: created.createdAt.toISOString(),
|
||||
retryable: false,
|
||||
expiresAfterDateKey: args.todayKey,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -433,7 +453,13 @@ export async function gateUnchangedStatusInput(args: {
|
||||
action: { name: "insights.status.skipped_unchanged" },
|
||||
meta: { cache_action: args.cacheAction, gate: "input" },
|
||||
});
|
||||
return { text: parsed.text, updatedAt: created.createdAt.toISOString() };
|
||||
return {
|
||||
kind: "generated",
|
||||
text: parsed.text,
|
||||
updatedAt: created.createdAt.toISOString(),
|
||||
retryable: false,
|
||||
expiresAfterDateKey: args.todayKey,
|
||||
};
|
||||
}
|
||||
|
||||
export interface LastGoodStatusHit {
|
||||
@@ -566,7 +592,11 @@ export async function resolveReadOnlyStatusMiss(args: {
|
||||
// navigation while the provider is still degraded would be a storm. While
|
||||
// the stub is fresh, stay in `preparing` without enqueuing; once it goes
|
||||
// stale (or never existed) enqueue a fresh generation.
|
||||
if (await hasFreshTimeoutStub({ userId: args.userId, cacheAction })) {
|
||||
const negativeCache = await readStatusNegativeCache({
|
||||
userId: args.userId,
|
||||
cacheAction,
|
||||
});
|
||||
if (negativeCache && !negativeCache.retryable) {
|
||||
annotate({
|
||||
action: { name: "insights.status.preparing" },
|
||||
meta: { metric: args.metric, suppressed_enqueue: true },
|
||||
@@ -602,22 +632,46 @@ export async function resolveReadOnlyStatusMiss(args: {
|
||||
* degraded. A stub without `retryAt` (legacy) is treated as stale so the
|
||||
* resolver retries, matching the pre-v1.8.3 "transient miss" behaviour.
|
||||
*/
|
||||
async function hasFreshTimeoutStub(args: {
|
||||
export interface StatusNegativeCache {
|
||||
kind: "negative";
|
||||
reason: "timeout" | "error" | "screened";
|
||||
retryAt: string;
|
||||
retryable: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the latest negative-cache policy without exposing provider errors or
|
||||
* assessment content. `retryable` flips only when the explicit `retryAt`
|
||||
* boundary has passed; legacy/malformed stubs are treated as absent so the
|
||||
* next request may recover.
|
||||
*/
|
||||
export async function readStatusNegativeCache(args: {
|
||||
userId: string;
|
||||
cacheAction: string;
|
||||
}): Promise<boolean> {
|
||||
}): Promise<StatusNegativeCache | null> {
|
||||
const latest = await prisma.auditLog.findFirst({
|
||||
where: { userId: args.userId, action: args.cacheAction },
|
||||
orderBy: { createdAt: "desc" },
|
||||
select: { details: true },
|
||||
});
|
||||
if (!latest?.details) return false;
|
||||
if (!latest?.details) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(latest.details) as ParsedStatusCache;
|
||||
if (!isTimeoutStub(parsed)) return false;
|
||||
if (typeof parsed.retryAt !== "string") return false;
|
||||
return new Date(parsed.retryAt).getTime() > Date.now();
|
||||
if (!isTimeoutStub(parsed)) return null;
|
||||
if (typeof parsed.retryAt !== "string") return null;
|
||||
const retryAtMs = new Date(parsed.retryAt).getTime();
|
||||
if (!Number.isFinite(retryAtMs)) return null;
|
||||
const reason =
|
||||
parsed.reason === "error" || parsed.reason === "screened"
|
||||
? parsed.reason
|
||||
: "timeout";
|
||||
return {
|
||||
kind: "negative",
|
||||
reason,
|
||||
retryAt: parsed.retryAt,
|
||||
retryable: retryAtMs <= Date.now(),
|
||||
};
|
||||
} catch {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,65 @@ export type SupportedLocale = Locale;
|
||||
|
||||
const SUPPORTED_LOCALES: ReadonlySet<string> = new Set(locales);
|
||||
|
||||
/**
|
||||
* The presentation contract shared by assessment producers and consumers.
|
||||
*
|
||||
* `kind` is the only branch discriminator. In particular, provider
|
||||
* availability is metadata: it must never hide safe deterministic text.
|
||||
* Provider/error details deliberately do not belong in this envelope.
|
||||
*/
|
||||
export type AssessmentStatus =
|
||||
| {
|
||||
kind: "generated";
|
||||
text: string;
|
||||
hasProvider: true;
|
||||
updatedAt: string | null;
|
||||
retryable: false;
|
||||
}
|
||||
| {
|
||||
kind: "screened-fallback";
|
||||
text: string;
|
||||
hasProvider: boolean;
|
||||
updatedAt: null;
|
||||
retryable: true;
|
||||
reason: OutboundReason | "unparseable_output" | "ungrounded_output";
|
||||
}
|
||||
| {
|
||||
kind: "no-provider";
|
||||
text: null;
|
||||
hasProvider: false;
|
||||
updatedAt: null;
|
||||
retryable: false;
|
||||
}
|
||||
| {
|
||||
kind: "preparing";
|
||||
text: null;
|
||||
hasProvider: true;
|
||||
updatedAt: null;
|
||||
retryable: true;
|
||||
}
|
||||
| {
|
||||
kind: "revalidating";
|
||||
text: string;
|
||||
hasProvider: true;
|
||||
updatedAt: string | null;
|
||||
retryable: true;
|
||||
}
|
||||
| {
|
||||
kind: "timeout" | "error";
|
||||
text: null;
|
||||
hasProvider: true;
|
||||
updatedAt: null;
|
||||
retryable: true;
|
||||
}
|
||||
| {
|
||||
kind: "exhausted";
|
||||
text: null;
|
||||
hasProvider: true;
|
||||
updatedAt: null;
|
||||
retryable: false;
|
||||
};
|
||||
|
||||
/** Round to `digits` decimal places. */
|
||||
export function round(value: number, digits = 1): number {
|
||||
const factor = 10 ** digits;
|
||||
@@ -305,6 +364,12 @@ export async function persistStatusInsight(args: {
|
||||
providerType: args.providerType,
|
||||
model: args.model,
|
||||
tokensUsed: args.tokensUsed,
|
||||
statusKind: "generated",
|
||||
retryable: false,
|
||||
// Freshness is defined by the user's calendar day. Keeping the
|
||||
// boundary in the payload makes expiry explicit without inventing a
|
||||
// UTC timestamp for a day key that belongs to another timezone.
|
||||
expiresAfterDateKey: args.todayKey,
|
||||
...(args.snapshotHash ? { snapshotHash: args.snapshotHash } : {}),
|
||||
...(args.inputHash ? { inputHash: args.inputHash } : {}),
|
||||
}),
|
||||
@@ -320,8 +385,12 @@ export async function persistStatusInsight(args: {
|
||||
* withholds.
|
||||
*/
|
||||
export type StatusSummaryOutcome =
|
||||
| { ok: true; text: string }
|
||||
| { ok: false; reason: OutboundReason | "unparseable_output" };
|
||||
| { ok: true; kind: "generated"; text: string }
|
||||
| {
|
||||
ok: false;
|
||||
kind: "screened-fallback";
|
||||
reason: OutboundReason | "unparseable_output";
|
||||
};
|
||||
|
||||
/**
|
||||
* The single finalize step every per-metric / biomarker / derived-score status
|
||||
@@ -359,13 +428,21 @@ export function finalizeStatusSummary(
|
||||
// text to render. WITHHOLD — the caller serves its deterministic stub. Only
|
||||
// the `unparseable` discriminator is logged, never the assessment content.
|
||||
if (extracted.kind === "unparseable") {
|
||||
return { ok: false, reason: "unparseable_output" };
|
||||
return {
|
||||
ok: false,
|
||||
kind: "screened-fallback",
|
||||
reason: "unparseable_output",
|
||||
};
|
||||
}
|
||||
const text = normalizeSummaryText(extracted.text);
|
||||
if (!text) return { ok: true, text: "" };
|
||||
if (!text) return { ok: true, kind: "generated", text: "" };
|
||||
const decision = screenModelOutput(text, locale, INSIGHTS_CONTRACTS);
|
||||
if (decision.block && decision.reason) {
|
||||
return { ok: false, reason: decision.reason };
|
||||
return {
|
||||
ok: false,
|
||||
kind: "screened-fallback",
|
||||
reason: decision.reason,
|
||||
};
|
||||
}
|
||||
return { ok: true, text };
|
||||
return { ok: true, kind: "generated", text };
|
||||
}
|
||||
Reference in new issue
Block a user