mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 05:42:19 +00:00
fix(ai): give background work the High room on a model that always reasons
Background jobs ask for medium reasoning at most, and status cards ask for none. A model that cannot be switched down (GLM behind a vLLM gateway, for instance) ignores both, spends the whole room thinking and returns only reasoning, so the daily briefing and the status cards fail as reasoning_only and the chain pauses. The Coach on High works because it leaves 12k tokens to think in. The chain runner now marks every call of a job walk as unattended, in one place rather than at each call site. The gateway and local clients give such a call at least the High thinking room once its model has answered with reasoning alone. The room has to ride on the #1267 repeat with none: one call may send two requests, and a once-a-day job meets an expired lesson every morning, so a room that waited for the unswitchable lesson would never reach the first briefing of the day. It is a ceiling, not a charge: a model that does stop at none never touches it. Calls a person is reading, the Coach included, keep the room of their own level, and the requested effort is unchanged. The room calculation both clients shared is now one function. Reported by @Tio61 in #1276.
This commit is contained in:
1 parent
e2998183c8
commit
bf0142a626
7 files changed
+191
-26
No files matched your search
@@ -45,16 +45,22 @@ interface SeenRequest {
|
||||
* `always`: the thought fills the ceiling whatever the effort (gpt-oss, GLM).
|
||||
* `answers-unreasoned`: the same, except asked with `none` it answers
|
||||
* (Gemma 4 on Ollama).
|
||||
* `answers-with-room`: ignores the effort and answers once the ceiling leaves
|
||||
* it the `high` thinking room (GLM 5.3 behind a gateway, #1276).
|
||||
*/
|
||||
let mode: "always" | "answers-unreasoned" = "always";
|
||||
let mode: "always" | "answers-unreasoned" | "answers-with-room" = "always";
|
||||
let server: http.Server;
|
||||
let origin: string;
|
||||
let seen: SeenRequest[] = [];
|
||||
|
||||
const ANSWER = JSON.stringify({ summary: "Steady." });
|
||||
|
||||
function reply(effort: unknown) {
|
||||
const answers = mode === "answers-unreasoned" && effort === "none";
|
||||
function reply(effort: unknown, maxTokens: unknown) {
|
||||
const answers =
|
||||
(mode === "answers-unreasoned" && effort === "none") ||
|
||||
(mode === "answers-with-room" &&
|
||||
typeof maxTokens === "number" &&
|
||||
maxTokens >= 12_000);
|
||||
return {
|
||||
id: "chatcmpl-1279",
|
||||
object: "chat.completion",
|
||||
@@ -87,7 +93,14 @@ beforeAll(async () => {
|
||||
maxTokens: body.max_tokens ?? body.max_completion_tokens,
|
||||
});
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify(reply(body.reasoning_effort)));
|
||||
res.end(
|
||||
JSON.stringify(
|
||||
reply(
|
||||
body.reasoning_effort,
|
||||
body.max_tokens ?? body.max_completion_tokens,
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
|
||||
@@ -210,3 +223,77 @@ describe.each([
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
/** A daily briefing call as the chain runner hands it to a client. */
|
||||
function briefingParams(unattended: boolean): CompletionParams {
|
||||
return {
|
||||
...singleUserTurn({
|
||||
system: "You write the daily briefing.",
|
||||
user: "Snapshot: …",
|
||||
maxTokens: 300,
|
||||
timeoutMs: 30_000,
|
||||
responseFormat: "json",
|
||||
}),
|
||||
reasoning: { effort: "medium", summaries: false },
|
||||
...(unattended ? { unattended: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
describe.each([
|
||||
["the OpenAI-compatible gateway", gateway],
|
||||
["the Local client", local],
|
||||
] as const)(
|
||||
"#1276 — %s, background work for a model that cannot be switched down",
|
||||
(_, make) => {
|
||||
it("gives the #1267 repeat the High room, and answers on the first call", async () => {
|
||||
mode = "answers-with-room";
|
||||
const result = await make().generateCompletion(briefingParams(true));
|
||||
|
||||
expect(result.content).toBe(ANSWER);
|
||||
// Two requests, the call's budget: medium at its own room, then the
|
||||
// repeat with `none` and the High room the reply showed it needs.
|
||||
expect(seen).toEqual([
|
||||
{ reasoningEffort: "medium", maxTokens: 4396 },
|
||||
{ reasoningEffort: "none", maxTokens: 12_300 },
|
||||
]);
|
||||
// Every later background call starts with the High room.
|
||||
seen = [];
|
||||
await make().generateCompletion(briefingParams(true));
|
||||
expect(seen).toEqual([{ reasoningEffort: "none", maxTokens: 12_300 }]);
|
||||
});
|
||||
|
||||
it("gives a status card, which asks no level, the same room", async () => {
|
||||
mode = "answers-with-room";
|
||||
const result = await make().generateCompletion({
|
||||
...statusParams(),
|
||||
unattended: true,
|
||||
});
|
||||
expect(result.content).toBe(ANSWER);
|
||||
expect(seen.map((s) => s.maxTokens)).toEqual([300, 12_300]);
|
||||
});
|
||||
|
||||
it("keeps the High room once even `none` is learned useless, and stays at one request", async () => {
|
||||
const err = (await make()
|
||||
.generateCompletion(briefingParams(true))
|
||||
.catch((e) => e)) as Record<string, unknown>;
|
||||
expect(err.kind).toBe("reasoning_only");
|
||||
expect(seen.map((s) => s.maxTokens)).toEqual([4396, 12_300]);
|
||||
seen = [];
|
||||
await make()
|
||||
.generateCompletion(briefingParams(true))
|
||||
.catch(() => {});
|
||||
expect(seen).toEqual([{ reasoningEffort: "medium", maxTokens: 12_300 }]);
|
||||
});
|
||||
|
||||
it("leaves a call a person is reading at the room of its level", async () => {
|
||||
await make()
|
||||
.generateCompletion(briefingParams(false))
|
||||
.catch(() => {});
|
||||
seen = [];
|
||||
await make()
|
||||
.generateCompletion(briefingParams(false))
|
||||
.catch(() => {});
|
||||
expect(seen).toEqual([{ reasoningEffort: "medium", maxTokens: 4396 }]);
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -272,3 +272,29 @@ describe("#1279: background work and a benched provider", () => {
|
||||
expect((err as AllProvidersFailedError).tokensSpent).toBe(5400);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#1276 — the runner marks background calls", () => {
|
||||
class Recorder implements AIProvider {
|
||||
readonly type = "openai-compatible" as const;
|
||||
seen: CompletionParams[] = [];
|
||||
async generateCompletion(p: CompletionParams): Promise<CompletionResult> {
|
||||
this.seen.push(p);
|
||||
return { content: "{}", tokensUsed: 1 } as CompletionResult;
|
||||
}
|
||||
}
|
||||
|
||||
it.each([
|
||||
["job", true],
|
||||
["coach", undefined],
|
||||
] as const)("a %s walk carries unattended: %s", async (surface, expected) => {
|
||||
const instance = new Recorder();
|
||||
await walk(
|
||||
surface,
|
||||
`u-unattended-${surface}`,
|
||||
"openai-compatible",
|
||||
instance,
|
||||
createInMemoryProviderHealthLedger(),
|
||||
);
|
||||
expect(instance.seen[0]?.unattended).toBe(expected);
|
||||
});
|
||||
});
|
||||
@@ -27,11 +27,10 @@ import {
|
||||
noteReasoningSeen,
|
||||
refusesReasoningField,
|
||||
} from "./reasoning/dialect-cache";
|
||||
import { uncontrolledThinkingRoom } from "./reasoning/levels";
|
||||
import {
|
||||
askAgainUnreasoned,
|
||||
knownReasoner,
|
||||
reasonsUnasked,
|
||||
uncontrolledCallThinkingRoom,
|
||||
repeatUnreasoned,
|
||||
} from "./reasoning/answerless";
|
||||
import {
|
||||
@@ -149,11 +148,10 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
// `none` sent for Off or after the #1267 lesson must not leave it the
|
||||
// bare answer budget to think in. A call without a level gets the `low`
|
||||
// room once the model has shown it reasons unasked.
|
||||
const thinkingTokens = params.reasoning
|
||||
? uncontrolledThinkingRoom(params.reasoning.effort)
|
||||
: reasonsUnasked(this.reasoningDialectKey())
|
||||
? uncontrolledThinkingRoom("off")
|
||||
: 0;
|
||||
const thinkingTokens = uncontrolledCallThinkingRoom(
|
||||
this.reasoningDialectKey(),
|
||||
params,
|
||||
);
|
||||
const body = JSON.stringify({
|
||||
model: this.config.model,
|
||||
messages,
|
||||
|
||||
@@ -21,15 +21,12 @@ import {
|
||||
import {
|
||||
askAgainUnreasoned,
|
||||
knownReasoner,
|
||||
reasonsUnasked,
|
||||
uncontrolledCallThinkingRoom,
|
||||
repeatUnreasoned,
|
||||
} from "./reasoning/answerless";
|
||||
import { RequestHops } from "./request-hops";
|
||||
import { learnFromRefusal } from "./wire-refusal";
|
||||
import {
|
||||
REASONING_THINKING_BUDGET,
|
||||
uncontrolledThinkingRoom,
|
||||
} from "./reasoning/levels";
|
||||
import { REASONING_THINKING_BUDGET } from "./reasoning/levels";
|
||||
import {
|
||||
answerOf,
|
||||
isOpenRouterEndpoint,
|
||||
@@ -573,19 +570,15 @@ export class OpenAIClient implements AIProvider {
|
||||
* included, and a `none` sent for the person's Off or after the #1267
|
||||
* lesson must not leave it the bare answer budget to think in. Without a
|
||||
* level of its own, a call gets the `low` room once the model has shown it
|
||||
* reasons unasked.
|
||||
* reasons unasked, and background work the `high` room once the model has
|
||||
* answered with reasoning alone (`unattendedThinkingRoom`).
|
||||
*/
|
||||
private thinkingRoom(
|
||||
params: CompletionParams,
|
||||
wire: OpenAIReasoningWire,
|
||||
): number {
|
||||
if (this.isGateway) {
|
||||
if (params.reasoning) {
|
||||
return uncontrolledThinkingRoom(params.reasoning.effort);
|
||||
}
|
||||
return reasonsUnasked(this.reasoningDialectKey())
|
||||
? uncontrolledThinkingRoom("off")
|
||||
: 0;
|
||||
return uncontrolledCallThinkingRoom(this.reasoningDialectKey(), params);
|
||||
}
|
||||
if (wire.kind === "none" || wire.effort === "none") return 0;
|
||||
const level = levelOfWireEffort(wire.effort);
|
||||
|
||||
@@ -743,14 +743,17 @@ export async function runStreamingRawCompletionWithFallback(
|
||||
onDelta: (delta: string) => void;
|
||||
},
|
||||
): Promise<RunRawWithFallbackResult> {
|
||||
// #1276 — a job walk is background work; stamped here, not per call site.
|
||||
const params: CompletionParams =
|
||||
args.surface === "job" ? { ...args.params, unattended: true } : args.params;
|
||||
return runRawChain(
|
||||
args,
|
||||
// Prefer the provider's streaming wire; fall back to the buffered call for
|
||||
// providers (cloud clients today) that do not implement streaming.
|
||||
(candidate) =>
|
||||
candidate.instance.generateCompletionStream
|
||||
? candidate.instance.generateCompletionStream(args.params, args.onDelta)
|
||||
: candidate.instance.generateCompletion(args.params),
|
||||
? candidate.instance.generateCompletionStream(params, args.onDelta)
|
||||
: candidate.instance.generateCompletion(params),
|
||||
(candidate) => ({
|
||||
ai_chain_streamed: Boolean(candidate.instance.generateCompletionStream),
|
||||
}),
|
||||
@@ -789,7 +792,9 @@ export interface RunRawWithFallbackResult {
|
||||
export async function runRawCompletionWithFallback(
|
||||
args: ChainWalkArgs,
|
||||
): Promise<RunRawWithFallbackResult> {
|
||||
const params: CompletionParams =
|
||||
args.surface === "job" ? { ...args.params, unattended: true } : args.params;
|
||||
return runRawChain(args, (candidate) =>
|
||||
candidate.instance.generateCompletion(args.params),
|
||||
candidate.instance.generateCompletion(params),
|
||||
);
|
||||
}
|
||||
@@ -36,6 +36,11 @@ import {
|
||||
rememberReasoningDialect,
|
||||
type ReasoningDialectKey,
|
||||
} from "./dialect-cache";
|
||||
import {
|
||||
REASONING_THINKING_BUDGET,
|
||||
uncontrolledThinkingRoom,
|
||||
type ReasoningLevel,
|
||||
} from "./levels";
|
||||
|
||||
/**
|
||||
* What the client does next with an answerless reply.
|
||||
@@ -137,6 +142,49 @@ export function reasonsUnasked(key: ReasoningDialectKey): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — the thinking room of an unattended call (`CompletionParams.unattended`)
|
||||
* to a model that has answered with reasoning alone (`reasonsUnasked`): at
|
||||
* least the `high` room the Coach gets on High. Background work asks for
|
||||
* `medium` at most, and a model that cannot be switched down (GLM behind
|
||||
* vLLM) spends all of it thinking and answers nothing, the daily briefing's
|
||||
* `reasoning_only`. The room applies from the #1267 repeat with `none` on:
|
||||
* one call may send two requests (`request-hops.ts`), and a once-a-day job
|
||||
* meets an expired lesson every morning, so the room has to ride on the
|
||||
* repeat or the first job of the day never gets it. A ceiling, not a charge:
|
||||
* a model that does stop at `none` never touches it, one that does not
|
||||
* thinks as long as it does either way and reaches the answer. Every call a
|
||||
* person is reading keeps `room`.
|
||||
*/
|
||||
export function unattendedThinkingRoom(
|
||||
key: ReasoningDialectKey,
|
||||
room: number,
|
||||
unattended: boolean | undefined,
|
||||
): number {
|
||||
return unattended && reasonsUnasked(key)
|
||||
? Math.max(room, REASONING_THINKING_BUDGET.high)
|
||||
: room;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — the thinking room of one call on a wire whose reasoning switch the
|
||||
* client cannot verify (a gateway, a local server): by the call's level, not
|
||||
* by what the wire says, the `low` room for a call without a level once the
|
||||
* model has shown it reasons unasked, and the `high` room for background work
|
||||
* once it has answered with reasoning alone (`unattendedThinkingRoom`).
|
||||
*/
|
||||
export function uncontrolledCallThinkingRoom(
|
||||
key: ReasoningDialectKey,
|
||||
params: { reasoning?: { effort: ReasoningLevel }; unattended?: boolean },
|
||||
): number {
|
||||
const room = params.reasoning
|
||||
? uncontrolledThinkingRoom(params.reasoning.effort)
|
||||
: reasonsUnasked(key)
|
||||
? uncontrolledThinkingRoom("off")
|
||||
: 0;
|
||||
return unattendedThinkingRoom(key, room, params.unattended);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — true when the model has shown it reasons, for a request that did
|
||||
* (`asked`) or did not ask it to: a lesson that says it reasons unasked, or a
|
||||
|
||||
@@ -383,6 +383,14 @@ export interface CompletionParams {
|
||||
* client sends exactly what it sent before the field existed.
|
||||
*/
|
||||
reasoning?: { effort: ReasoningLevel; summaries: boolean };
|
||||
/**
|
||||
* #1276 — no person reads this reply as it streams: the call runs as
|
||||
* background work. Set by the chain runner for every `"job"` walk, never by
|
||||
* a call site. The gateway and local clients give such a call the `high`
|
||||
* thinking room once its model has answered with reasoning alone
|
||||
* (`unattendedThinkingRoom`); nothing else reads it.
|
||||
*/
|
||||
unattended?: boolean;
|
||||
/**
|
||||
* v1.41.2 — a stable key for calls that resend the same prompt prefix (the
|
||||
* rounds of one Coach turn, the turns of one conversation). Providers that
|
||||
|
||||
Reference in new issue
Block a user