test(e2e): gate every measurement on its element, and keep it that way

Four specs navigated and then read the DOM once, with nothing in between
that waits for anything but the network. An idle network says nothing about
whether React has rendered, so the read can sample a tree that is still
mounting and the assertion then blames the application for a race inside the
test. Two such reads went red on consecutive CI runs, on different specs.

The settings over-scroll guard also slept a fixed 500 ms, which is the wrong
shape twice over: too long when the page is ready, too short on a loaded
runner, and it never says what it is waiting for.

The structural guard runs in the unit suite rather than in Playwright,
because e2e runs at PR-CI only and flaky counts as failed there — one
unguarded read otherwise costs a whole pipeline before anyone sees it. It
refuses to count a wait whose rejection is thrown away, since that reads
like a guarantee and is not one, and would leave the check defeatable by
appending a catch. The one deliberate swallow is on the allowlist with its
reason: a genuine regression there still fails at the width assertion below
it, which reports the measured number instead of a timeout message.
This commit is contained in:
Marc-André Bombeck committed 2026-07-26 22:04:58 +02:00
1 parent 26f8325c26
commit d10ae9e71c
5 files changed
+238 -5

No files matched your search

+13 -1
View File
@@ -1,6 +1,7 @@
import { expect, test } from "./setup/test";
import { STORAGE_STATE_PATH } from "./setup/global-setup";
import { settleBeforeMeasure } from "./utils/settle";
import {
LONG_HEADLINE_BRIEFING,
MOCK_SCORE_RINGS,
@@ -162,7 +163,16 @@ test.describe("phone-width density guard", () => {
await mockMoodInsights(page);
}
await page.goto(route, { waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle");
if (route === "/insights") {
// The one route here that had no readiness signal, while its two
// siblings below both have one. An overflow guard against a page
// that has not finished rendering passes for the same reason an
// empty page passes: there is nothing wide on it yet.
await expect(
page.locator('[data-slot="section-heading"]').first(),
).toBeVisible();
}
if (route === "/") {
// The guard must not pass vacuously — the Today hero and its
@@ -186,6 +196,8 @@ test.describe("phone-width density guard", () => {
).toBeVisible();
}
await settleBeforeMeasure(page, page.locator("main#main-content"));
// The document/body pair alone cannot see the app's sideways scroll:
// `AuthShell` scrolls inside `<main id="main-content">`, whose
// `overflow-y:auto` makes `overflow-x` compute to `auto`, so an
+5 -1
View File
@@ -1,6 +1,7 @@
import { expect, test } from "./setup/test";
import { STORAGE_STATE_PATH } from "./setup/global-setup";
import { settleBeforeMeasure } from "./utils/settle";
import {
mockDashboardSnapshot,
WEIGHT_ONLY_SUMMARIES,
@@ -78,7 +79,10 @@ test.describe("mobile-viewport smoke", () => {
);
await page.goto("/", { waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle");
// Gate on the dashboard's own content, not on the network. Every read
// below is geometric and happens exactly once, so measuring a tree that
// is still mounting reports a layout nobody will ever see.
await settleBeforeMeasure(page, page.locator("main#main-content"));
// 1) No horizontal scroll
const dims = await page.evaluate(() => ({
+6 -3
View File
@@ -1,6 +1,7 @@
import { expect, test } from "./setup/test";
import { STORAGE_STATE_PATH } from "./setup/global-setup";
import { settleBeforeMeasure } from "./utils/settle";
/**
* Vertical over-scroll regression guard (the recurring class behind the
@@ -63,9 +64,11 @@ test.describe("settings + admin vertical over-scroll guard", () => {
test(`no over-scroll on ${route} (${vp.name})`, async ({ page }) => {
await page.setViewportSize({ width: vp.width, height: vp.height });
await page.goto(route, { waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle");
// Let entry transitions / late queries settle before measuring.
await page.waitForTimeout(500);
// Was a fixed 500 ms sleep, which is the wrong shape twice over: too
// long when the page is ready and too short on a loaded runner, and
// it never says what it is waiting for. Gate on the element the
// measurement reads instead.
await settleBeforeMeasure(page, page.locator("#main-content"));
const dims = await page.evaluate(() => {
const main = document.getElementById("main-content");
+39
View File
@@ -0,0 +1,39 @@
import { expect, type Locator, type Page } from "@playwright/test";
/**
* Gate a one-shot DOM read on the element it is about to measure.
*
* The failure this exists to prevent: a spec navigates with
* `waitUntil: "networkidle"`, then reads the DOM in a single
* `page.evaluate` / `evaluateAll` that does not retry. An idle network says
* nothing about whether React has rendered, so the read can land on a tree
* that is still mounting and the assertion then blames the application for
* something the test raced — "language + dob fields must exist" when the
* fields were simply not there yet, or a measurement helper accused of
* measuring the wrong element when it was measuring the right one too early.
*
* Two such tests went red on consecutive CI runs during v1.33.0. Neither
* reproduces locally: a developer machine is fast enough that the render
* always wins the race, so the only place this shows up is a loaded runner,
* and `failOnFlakyTests` means every occurrence costs a whole red pipeline
* even when the retry passes.
*
* `sentinel` must be something the measurement actually depends on, not just
* any element on the page. Gating on a layout shell that renders before the
* data proves nothing.
*/
export async function settleBeforeMeasure(
page: Page,
sentinel: Locator,
options: { attachedOnly?: boolean } = {},
): Promise<void> {
if (options.attachedOnly) {
await expect(sentinel).toBeAttached();
} else {
await expect(sentinel).toBeVisible();
}
// Fonts and late CSS change geometry after the element is visible, and
// every measurement in this suite is geometric.
await page.waitForLoadState("networkidle").catch(() => {});
await page.evaluate(() => document.fonts?.ready);
}