mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
ci: fail the mutation floors step, and scale nightly property runs by a factor
The floors step piped scripts/mutation-floors.mjs into tee without pipefail, so it took tee's exit status and a module under its floor could never fail the run. It now runs under bash with pipefail, as do the other steps that pipe into tee, and a guard reads every workflow and fails on a tee pipe without it. The other piped steps were checked: each either validates what the pipe produced or pipes a command that cannot fail. The nightly runs set FC_NUM_RUNS, an absolute count that replaced every property's own: the database backup round trip, which states twelve cases for its cost, ran a hundred times against a timeout sized for twelve. FC_RUNS_FACTOR now multiplies each property's count (five for the unit suite, four for integration), propertyTimeout grows a test's timeout by the same factor, and the backup round trip uses it.
This commit is contained in:
1 parent
422adf2986
commit
298515d156
7 files changed
+242
-61
No files matched your search
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* The nightly property runs multiply each property's own case count instead
|
||||
* of replacing it. A nightly count of 100 for every property ran the backup
|
||||
* round trip, which states twelve cases for its cost, eight times over
|
||||
* against a timeout sized for twelve.
|
||||
*/
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
async function loadWith(factor: string | undefined) {
|
||||
vi.resetModules();
|
||||
if (factor === undefined) vi.stubEnv("FC_RUNS_FACTOR", "");
|
||||
else vi.stubEnv("FC_RUNS_FACTOR", factor);
|
||||
return import("../../tests/property/fast-check");
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe("FC_RUNS_FACTOR", () => {
|
||||
it("keeps each property's own count on a pull request", async () => {
|
||||
const { propertyParameters, propertyTimeout } = await loadWith(undefined);
|
||||
expect(propertyParameters(12).numRuns).toBe(12);
|
||||
expect(propertyParameters().numRuns).toBe(100);
|
||||
expect(propertyTimeout(60_000)).toBe(60_000);
|
||||
});
|
||||
|
||||
it("multiplies each property's count and the timeout it was given", async () => {
|
||||
const { propertyParameters, propertyTimeout } = await loadWith("4");
|
||||
expect(propertyParameters(12).numRuns).toBe(48);
|
||||
expect(propertyParameters(300).numRuns).toBe(1200);
|
||||
expect(propertyTimeout(60_000)).toBe(240_000);
|
||||
});
|
||||
|
||||
it("ignores a value that is not a positive number", async () => {
|
||||
const { propertyParameters } = await loadWith("-3");
|
||||
expect(propertyParameters(12).numRuns).toBe(12);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* A workflow step that pipes a command into `tee` runs with pipefail.
|
||||
*
|
||||
* GitHub runs a `run:` script with `bash -e {0}` unless the step names a
|
||||
* shell, and without pipefail a pipeline's exit status is its last command's.
|
||||
* `tee` always succeeds, so `node scripts/mutation-floors.mjs | tee -a
|
||||
* "$GITHUB_STEP_SUMMARY"` passed whatever the floors said. A step that names
|
||||
* `shell: bash` runs as `bash --noprofile --norc -eo pipefail {0}`; this guard
|
||||
* asks for that and for `set -o pipefail` in the script as well, so the
|
||||
* intent is visible where the pipe is.
|
||||
*/
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import YAML from "yaml";
|
||||
|
||||
const WORKFLOWS = join(process.cwd(), ".github", "workflows");
|
||||
|
||||
interface Step {
|
||||
name?: string;
|
||||
run?: string;
|
||||
shell?: string;
|
||||
}
|
||||
|
||||
/** Whether a script pipes something into `tee` on a line that is not a comment. */
|
||||
function pipesIntoTee(script: string): boolean {
|
||||
return script
|
||||
.split("\n")
|
||||
.some((line) => !/^\s*#/.test(line) && /[^|]\|\s*tee\b/.test(line));
|
||||
}
|
||||
|
||||
/** The steps of a workflow that pipe into `tee` without pipefail. */
|
||||
function offenders(file: string, text: string): string[] {
|
||||
const doc = YAML.parse(text) as {
|
||||
defaults?: { run?: { shell?: string } };
|
||||
jobs?: Record<
|
||||
string,
|
||||
{ defaults?: { run?: { shell?: string } }; steps?: Step[] }
|
||||
>;
|
||||
};
|
||||
const out: string[] = [];
|
||||
for (const [jobName, job] of Object.entries(doc.jobs ?? {})) {
|
||||
(job.steps ?? []).forEach((step, index) => {
|
||||
if (!step.run || !pipesIntoTee(step.run)) return;
|
||||
const shell =
|
||||
step.shell ?? job.defaults?.run?.shell ?? doc.defaults?.run?.shell;
|
||||
const ok = shell === "bash" && /\bset -o pipefail\b/.test(step.run);
|
||||
if (!ok) out.push(`${file} ${jobName} #${index} ${step.name ?? ""}`);
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function workflowFiles(): string[] {
|
||||
return readdirSync(WORKFLOWS).filter((f) => /\.ya?ml$/.test(f));
|
||||
}
|
||||
|
||||
describe("workflow steps that pipe into tee", () => {
|
||||
it("reads the workflows it claims to check", () => {
|
||||
expect(workflowFiles().length).toBeGreaterThan(10);
|
||||
const teeSteps = workflowFiles().filter((f) =>
|
||||
pipesIntoTee(readFileSync(join(WORKFLOWS, f), "utf8")),
|
||||
);
|
||||
expect(teeSteps.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("run with pipefail", () => {
|
||||
const found = workflowFiles().flatMap((f) =>
|
||||
offenders(f, readFileSync(join(WORKFLOWS, f), "utf8")),
|
||||
);
|
||||
expect(found).toEqual([]);
|
||||
});
|
||||
|
||||
it("catches the shape it guards against", () => {
|
||||
const bad = [
|
||||
"jobs:",
|
||||
" check:",
|
||||
" steps:",
|
||||
" - name: floors",
|
||||
' run: node scripts/mutation-floors.mjs | tee -a "$GITHUB_STEP_SUMMARY"',
|
||||
].join("\n");
|
||||
expect(offenders("bad.yml", bad)).toEqual(["bad.yml check #0 floors"]);
|
||||
const good = [
|
||||
"jobs:",
|
||||
" check:",
|
||||
" steps:",
|
||||
" - name: floors",
|
||||
" shell: bash",
|
||||
" run: |",
|
||||
" set -o pipefail",
|
||||
' node scripts/mutation-floors.mjs | tee -a "$GITHUB_STEP_SUMMARY"',
|
||||
].join("\n");
|
||||
expect(offenders("good.yml", good)).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user