test(onboarding): the AI panel names the variant it settled on

The panel paints before its status query answers, and every variant inside it
is conditional, so an assertion that a variant is absent could pass on a
question that had simply not been answered yet — which is how the journey for
the unmeasured-provider case passed on a retry against the very defect it was
written to catch.

The panel now carries the variant it settled on, absent until it has settled,
and the three journeys wait on that instead of racing it. The third journey
covers the state a freshly deployed instance is in before its first AI call:
no offer, and no claim of an outage either.
This commit is contained in:
Marc-André Bombeck committed 2026-09-11 11:15:39 +02:00
1 parent 41596d1cb8
commit 34caeeea3c
4 files changed
+92 -3

No files matched your search

+53 -3
View File
@@ -8,11 +8,12 @@
* on their first briefing. On 2026-09-11 the operator's own instance was
* answering HTTP 500 from its OAuth proxy while the screen kept promising.
*
* Two runs, one account, and the only difference between them is a row in
* Three runs, one account, and the only difference between them is a row in
* `provider_health`:
*
* a success inside the freshness window → the offer, and one tap grants
* a dead credential inside its cooldown → "not answering", and no button
* no row at all → neither: the panel says nothing
*
* What is deliberately absent: any call to a provider. The flow's rule holds
* — nothing in it talks to one — and a probe here would bill the operator for
@@ -28,6 +29,7 @@ import {
import {
clearAiConsent,
clearOperatorProvider,
clearSharedProviderResult,
seedOperatorProvider,
seedSharedProviderResult,
} from "./setup/ai-offer-fixture";
@@ -44,6 +46,9 @@ const OFFER = '[data-slot="onboarding-ai-offer"]';
const GRANT = '[data-slot="onboarding-ai-offer-grant"]';
const UNAVAILABLE = '[data-slot="onboarding-ai-unavailable"]';
const SHARED_KEY = '[data-slot="onboarding-ai-shared-key"]';
/** The panel once it knows which variant it is; absent while the status is in
* flight, so an assertion of absence cannot pass on an unanswered question. */
const SETTLED = '[data-slot="onboarding-ai-panel"][data-ai-state]';
/** Walk the account to the done screen through the API, then open it. */
async function openDoneScreen(page: import("@playwright/test").Page) {
@@ -102,7 +107,12 @@ test.describe("the setup flow's AI offer follows the shared provider's health",
await openDoneScreen(page);
await expect(page.locator(OFFER)).toBeVisible({ timeout: 15_000 });
await expect(page.locator(SETTLED)).toHaveAttribute(
"data-ai-state",
"offer",
{ timeout: 15_000 },
);
await expect(page.locator(OFFER)).toBeVisible();
await expect(page.locator(UNAVAILABLE)).toHaveCount(0);
// No provider was called to decide this. The flow reads a projection.
@@ -141,7 +151,12 @@ test.describe("the setup flow's AI offer follows the shared provider's health",
await openDoneScreen(page);
await expect(page.locator(UNAVAILABLE)).toBeVisible({ timeout: 15_000 });
await expect(page.locator(SETTLED)).toHaveAttribute(
"data-ai-state",
"unavailable",
{ timeout: 15_000 },
);
await expect(page.locator(UNAVAILABLE)).toBeVisible();
await expect(page.locator(GRANT)).toHaveCount(0);
await expect(page.locator(OFFER)).toHaveCount(0);
// The unproven promise is not shown to somebody who has not consented.
@@ -167,4 +182,39 @@ test.describe("the setup flow's AI offer follows the shared provider's health",
expect(data.serverProviderHealth).toBe("unhealthy");
expect(data.serverProviderOffer).toBe(false);
});
test("claims nothing on an instance whose provider nobody has used yet", async ({
page,
}) => {
test.setTimeout(90_000);
// The operator has configured a key and nothing has been asked of it —
// the state of every deployment until its first AI call, and so the most
// common way through this screen. The panel may not offer (nothing is
// known to work) and may not report an outage (nothing failed).
await clearSharedProviderResult();
await openDoneScreen(page);
await expect(page.locator(SETTLED)).toHaveAttribute(
"data-ai-state",
"neutral",
{ timeout: 15_000 },
);
await expect(
page.locator('[data-slot="onboarding-ai-keyless"]'),
).toBeVisible();
await expect(page.locator(UNAVAILABLE)).toHaveCount(0);
await expect(page.locator(OFFER)).toHaveCount(0);
await expect(page.locator(GRANT)).toHaveCount(0);
await expect(page.locator(SHARED_KEY)).toHaveCount(0);
const status = await page.request.get("/api/user/ai-provider");
const data = (
(await status.json()) as {
data: { serverProviderHealth: string; serverProviderOffer: boolean };
}
).data;
expect(data.serverProviderHealth).toBe("unknown");
expect(data.serverProviderOffer).toBe(false);
});
});
+17
View File
@@ -117,6 +117,23 @@ export async function seedSharedProviderResult(
}
}
/**
* The ledger holds nothing for either shared tag, with the operator's key
* still in place: a brand-new instance before its first AI call, which is
* what every deployment looks like until somebody makes one.
*/
export async function clearSharedProviderResult(): Promise<void> {
const pool = connect();
try {
await pool.query(
`DELETE FROM provider_health
WHERE provider_type IN ('admin-openai', 'admin-codex')`,
);
} finally {
await pool.end();
}
}
/** No receipt on file — the state a fresh account reaches the screen in. */
export async function clearAiConsent(username: string): Promise<void> {
const pool = connect();
@@ -179,6 +179,7 @@ describe("<DoneScreen> shared-provider offer", () => {
serverProviderConsent: false,
};
const html = render();
expect(html).toContain('data-ai-state="offer"');
expect(html).toContain('data-slot="onboarding-ai-offer"');
expect(html).toContain('data-slot="onboarding-ai-offer-grant"');
// The unproven promise is gone from this branch.
@@ -194,6 +195,7 @@ describe("<DoneScreen> shared-provider offer", () => {
serverProviderConsent: false,
};
const html = render();
expect(html).toContain('data-ai-state="unavailable"');
expect(html).toContain('data-slot="onboarding-ai-unavailable"');
expect(html).not.toContain('data-slot="onboarding-ai-offer-grant"');
expect(html).not.toContain('data-slot="onboarding-ai-shared-key"');
@@ -214,6 +216,7 @@ describe("<DoneScreen> shared-provider offer", () => {
serverProviderConsent: false,
};
const html = render();
expect(html).toContain('data-ai-state="neutral"');
expect(html).not.toContain('data-slot="onboarding-ai-unavailable"');
expect(html).not.toContain('data-slot="onboarding-ai-offer-grant"');
expect(html).not.toContain('data-slot="onboarding-ai-shared-key"');
@@ -230,6 +233,7 @@ describe("<DoneScreen> shared-provider offer", () => {
serverProviderConsent: true,
};
const html = render();
expect(html).toContain('data-ai-state="consent"');
expect(html).toContain('data-slot="onboarding-ai-shared-key"');
expect(html).not.toContain('data-slot="onboarding-ai-offer-grant"');
expect(html).not.toContain('data-slot="onboarding-ai-unavailable"');
@@ -252,6 +256,8 @@ describe("<DoneScreen> shared-provider offer", () => {
it("says nothing at all while the status is still in flight", () => {
aiProviderState.data = undefined;
const html = render();
// Not even the settled marker: the question is genuinely unanswered.
expect(html).not.toContain("data-ai-state");
expect(html).not.toContain('data-slot="onboarding-ai-offer"');
expect(html).not.toContain('data-slot="onboarding-ai-unavailable"');
});
+16
View File
@@ -225,6 +225,22 @@ export function DoneScreen({ state }: { state: OnboardingStateDto }) {
<section
aria-labelledby="onboarding-ai-panel-title"
data-slot="onboarding-ai-panel"
/* Which of the four variants the panel SETTLED on, and absent until
it has settled. The panel itself is unconditional and paints
immediately, so without this an assertion that a variant is absent
passes on a status that has simply not arrived yet. The value is
the same tri-state decision made once, above. */
data-ai-state={
aiProvider === undefined
? undefined
: consentOnFile && operatorProvides
? "consent"
: offerShared
? "offer"
: sharedUnavailable
? "unavailable"
: "neutral"
}
className="border-border bg-card mx-auto flex w-full max-w-md flex-col gap-4 rounded-xl border p-4 text-left md:p-6"
>
<header className="flex items-start gap-3">