diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 000000000..cc92b01ad --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,48 @@ +name: Mutation tests + +# Stryker on the core libraries (stryker.config.mjs): encryption at rest, +# medication compliance, rollups, source ranking, unit conversion and backup +# validation. Too slow for a pull request; it runs once a night and on +# demand, and fails when a module's score drops below its floor in +# stryker.floors.json. The HTML report is uploaded either way. +on: + schedule: + - cron: "23 1 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + mutation: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + name: mutation (core libraries) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - run: pnpm db:generate + + - name: Run Stryker + run: pnpm exec stryker run --concurrency 4 + + - name: Check the per-module floors + run: node scripts/mutation-floors.mjs | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mutation-report + path: reports/mutation/ + if-no-files-found: ignore + retention-days: 30 diff --git a/.gitignore b/.gitignore index 9cd46eebf..655bb269e 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,6 @@ next-env.d.ts # same reason in a different tool. /.wt-*/ /.worktrees/ +# Mutation testing (stryker.config.mjs): reports and the sandbox copy. +/reports/mutation +/.stryker-tmp diff --git a/Dockerfile b/Dockerfile index 338364c89..daba7469c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ RUN corepack enable && corepack prepare pnpm@11.15.1 --activate WORKDIR /app COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +# The dependency patches pnpm-workspace.yaml names; the frozen install +# refuses to run without them. +COPY patches patches COPY scripts/prepare.mjs scripts/prepare.mjs RUN pnpm install --frozen-lockfile --prod=false diff --git a/patches/@stryker-mutator__vitest-runner@10.0.0.patch b/patches/@stryker-mutator__vitest-runner@10.0.0.patch new file mode 100644 index 000000000..6b0bd2739 --- /dev/null +++ b/patches/@stryker-mutator__vitest-runner@10.0.0.patch @@ -0,0 +1,48 @@ +diff --git a/dist/src/stryker-setup.js b/dist/src/stryker-setup.js +index aecfacbe47b285c272c47e54b3d66a716adfdc41..a6864060e466a4dde49f6d32d9fa8bc8e416baaa 100644 +--- a/dist/src/stryker-setup.js ++++ b/dist/src/stryker-setup.js +@@ -57,13 +57,17 @@ else { + } + // Stryker disable all: this file is copied to the sandbox dir + function collectTestName({ name, suite, }) { ++ // Vitest 5 joins a test's full name with " > " and matches ++ // testNamePattern against that; a space-joined name matched nothing under ++ // a describe block, so every such test was skipped in mutant runs. The ++ // file task ends the chain and is not part of the name. + const nameParts = [name]; + let currentSuite = suite; +- while (currentSuite) { ++ while (currentSuite && currentSuite.file !== currentSuite) { + nameParts.unshift(currentSuite.name); + currentSuite = currentSuite.suite; + } +- return nameParts.join(' ').trim(); ++ return nameParts.filter(Boolean).join(' > ').trim(); + } + function toRawTestId(test) { + return `${test.file?.filepath ?? 'unknown.js'}#${collectTestName(test)}`; +diff --git a/dist/src/test-helpers.js b/dist/src/test-helpers.js +index 56cc786f30c524ae18ce87eef1a67fdf26b703c5..1ad0552d5764d6b8c1ff2215e2e6fb49d7f6bc83 100644 +--- a/dist/src/test-helpers.js ++++ b/dist/src/test-helpers.js +@@ -2,13 +2,17 @@ + // This file is used from the testing environment (via stryker-setup.js) and thus could be loaded into the browser (when using vitest with browser mode). + // Thus we should avoid unnecessary dependencies in this file. + export function collectTestName({ name, suite, }) { ++ // Vitest 5 joins a test's full name with " > " and matches ++ // testNamePattern against that; a space-joined name matched nothing under ++ // a describe block, so every such test was skipped in mutant runs. The ++ // file task ends the chain and is not part of the name. + const nameParts = [name]; + let currentSuite = suite; +- while (currentSuite) { ++ while (currentSuite && currentSuite.file !== currentSuite) { + nameParts.unshift(currentSuite.name); + currentSuite = currentSuite.suite; + } +- return nameParts.join(' ').trim(); ++ return nameParts.filter(Boolean).join(' > ').trim(); + } + export function toRawTestId(test) { + return `${test.file?.filepath ?? 'unknown.js'}#${collectTestName(test)}`; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb70028fa..aee6822ad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,6 +37,9 @@ overrides: brace-expansion@>=4.0.0 <5.0.12: ^5.0.12 undici@>=7.0.0 <7.29.1: ^7.29.1 +patchedDependencies: + '@stryker-mutator/vitest-runner@10.0.0': 9ad3e9855fd9781fd36e872ed42c5f934f91b962d6830e6cc56d56bf16e1dabd + importers: .: @@ -206,7 +209,7 @@ importers: version: 10.0.0(@types/node@26.6.5) '@stryker-mutator/vitest-runner': specifier: 10.0.0 - version: 10.0.0(@stryker-mutator/core@10.0.0(@types/node@26.6.5))(vitest@5.0.4) + version: 10.0.0(patch_hash=9ad3e9855fd9781fd36e872ed42c5f934f91b962d6830e6cc56d56bf16e1dabd)(@stryker-mutator/core@10.0.0(@types/node@26.6.5))(vitest@5.0.4) '@tailwindcss/postcss': specifier: ^4 version: 4.3.3 @@ -1047,10 +1050,6 @@ packages: cpu: [x64] os: [win32] - '@inquirer/ansi@2.0.7': - resolution: {integrity: sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==} - engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} - '@inquirer/ansi@2.0.9': resolution: {integrity: sha512-0Ywt3S8tqM6TWUcsRlBFQ1x+X3/o8kktX0ZiKAnByYb9GFDknnOhse1PVtfiD2BXPI8ar+BWx9PV9R0lQIcygw==} engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} @@ -1064,15 +1063,6 @@ packages: '@types/node': optional: true - '@inquirer/confirm@6.1.1': - resolution: {integrity: sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==} - engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} - peerDependencies: - '@types/node': '>=18' - peerDependenciesMeta: - '@types/node': - optional: true - '@inquirer/confirm@6.3.4': resolution: {integrity: sha512-ZwnaZZDa1zxsyVrN32s447ydMnKcw7e8WWNT+R+Qsu0zAn8Rs6yi9d1B80Kob1Ul2cfUi7t8JD2t4/K7uEZpqA==} engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} @@ -1082,15 +1072,6 @@ packages: '@types/node': optional: true - '@inquirer/core@11.2.1': - resolution: {integrity: sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==} - engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} - peerDependencies: - '@types/node': '>=18' - peerDependenciesMeta: - '@types/node': - optional: true - '@inquirer/core@12.0.5': resolution: {integrity: sha512-c0ZSnO9oVk/IlEGoZK3u8b3IZQeu9I6jUkXJSQnQEVRXsjKvtAq3AGqFL2LgfJhlzMZne08pZHWRWasIcI3Rew==} engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} @@ -1127,10 +1108,6 @@ packages: '@types/node': optional: true - '@inquirer/figures@2.0.7': - resolution: {integrity: sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==} - engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} - '@inquirer/figures@2.0.9': resolution: {integrity: sha512-EAWgUTGQ/Umgga51dE3B2PUHbufuXarDfg86uVgoSgNHNNQnyFKcOrQLWVqYMghuSyHh8+2HUH0Js9cTC1WAdg==} engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} @@ -1198,15 +1175,6 @@ packages: '@types/node': optional: true - '@inquirer/type@4.0.7': - resolution: {integrity: sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==} - engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} - peerDependencies: - '@types/node': '>=18' - peerDependenciesMeta: - '@types/node': - optional: true - '@inquirer/type@4.2.0': resolution: {integrity: sha512-2rnHX8RPo4glJYqnFc/v3nQeIhDcjyRRyaT7MGcfL/8xv/8K03SXYoA2tX06WVSKywWOE/ncbG3dtuzLjR5TCQ==} engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} @@ -8183,9 +8151,6 @@ snapshots: '@img/sharp-win32-x64@0.35.5': optional: true - '@inquirer/ansi@2.0.7': - optional: true - '@inquirer/ansi@2.0.9': {} '@inquirer/checkbox@5.2.7(@types/node@26.6.5)': @@ -8197,14 +8162,6 @@ snapshots: optionalDependencies: '@types/node': 26.6.5 - '@inquirer/confirm@6.1.1(@types/node@26.6.5)': - dependencies: - '@inquirer/core': 11.2.1(@types/node@26.6.5) - '@inquirer/type': 4.0.7(@types/node@26.6.5) - optionalDependencies: - '@types/node': 26.6.5 - optional: true - '@inquirer/confirm@6.3.4(@types/node@26.6.5)': dependencies: '@inquirer/core': 12.0.5(@types/node@26.6.5) @@ -8212,19 +8169,6 @@ snapshots: optionalDependencies: '@types/node': 26.6.5 - '@inquirer/core@11.2.1(@types/node@26.6.5)': - dependencies: - '@inquirer/ansi': 2.0.7 - '@inquirer/figures': 2.0.7 - '@inquirer/type': 4.0.7(@types/node@26.6.5) - cli-width: 4.1.0 - fast-wrap-ansi: 0.2.2 - mute-stream: 3.0.0 - signal-exit: 4.1.0 - optionalDependencies: - '@types/node': 26.6.5 - optional: true - '@inquirer/core@12.0.5(@types/node@26.6.5)': dependencies: '@inquirer/ansi': 2.0.9 @@ -8259,9 +8203,6 @@ snapshots: optionalDependencies: '@types/node': 26.6.5 - '@inquirer/figures@2.0.7': - optional: true - '@inquirer/figures@2.0.9': {} '@inquirer/input@5.1.8(@types/node@26.6.5)': @@ -8325,11 +8266,6 @@ snapshots: optionalDependencies: '@types/node': 26.6.5 - '@inquirer/type@4.0.7(@types/node@26.6.5)': - optionalDependencies: - '@types/node': 26.6.5 - optional: true - '@inquirer/type@4.2.0(@types/node@26.6.5)': optionalDependencies: '@types/node': 26.6.5 @@ -10070,7 +10006,7 @@ snapshots: '@stryker-mutator/util@10.0.0': {} - '@stryker-mutator/vitest-runner@10.0.0(@stryker-mutator/core@10.0.0(@types/node@26.6.5))(vitest@5.0.4)': + '@stryker-mutator/vitest-runner@10.0.0(patch_hash=9ad3e9855fd9781fd36e872ed42c5f934f91b962d6830e6cc56d56bf16e1dabd)(@stryker-mutator/core@10.0.0(@types/node@26.6.5))(vitest@5.0.4)': dependencies: '@stryker-mutator/api': 10.0.0 '@stryker-mutator/core': 10.0.0(@types/node@26.6.5) @@ -12751,7 +12687,7 @@ snapshots: msw@2.14.6(@types/node@26.6.5)(typescript@6.0.3): dependencies: - '@inquirer/confirm': 6.1.1(@types/node@26.6.5) + '@inquirer/confirm': 6.3.4(@types/node@26.6.5) '@mswjs/interceptors': 0.41.9 '@open-draft/deferred-promise': 3.0.0 '@types/statuses': 2.0.6 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a699c5cd7..5f13e535e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -131,3 +131,6 @@ auditConfig: # never runs. No patched release is published yet (latest is 1.4.0); # remove this entry and raise the floor once 1.4.1 is out. - GHSA-86w9-cpqp-85rv + +patchedDependencies: + "@stryker-mutator/vitest-runner@10.0.0": patches/@stryker-mutator__vitest-runner@10.0.0.patch diff --git a/scripts/__tests__/ci-determinism.test.ts b/scripts/__tests__/ci-determinism.test.ts index f9254aaa0..fbe1a1d59 100644 --- a/scripts/__tests__/ci-determinism.test.ts +++ b/scripts/__tests__/ci-determinism.test.ts @@ -157,6 +157,9 @@ describe("container dependency installation", () => { expect(dockerfile).toContain( "COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./\n" + + "# The dependency patches pnpm-workspace.yaml names; the frozen install\n" + + "# refuses to run without them.\n" + + "COPY patches patches\n" + "COPY scripts/prepare.mjs scripts/prepare.mjs\n" + "RUN pnpm install --frozen-lockfile --prod=false", ); diff --git a/scripts/mutation-floors.mjs b/scripts/mutation-floors.mjs new file mode 100644 index 000000000..2824385cd --- /dev/null +++ b/scripts/mutation-floors.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +/** + * Check each module's mutation score against its floor. + * + * node scripts/mutation-floors.mjs [reports/mutation/mutation.json] + * + * Stryker's own `thresholds.break` is one number for the whole run, which + * lets a well-tested module hide a weak one. `stryker.floors.json` names a + * floor per module instead (a module is a set of path prefixes), and this + * script reads the JSON report Stryker writes and fails when any module + * scores below its floor. The score is Stryker's: killed and timed-out + * mutants over every mutant that was valid and covered or not, so a mutant no + * test reaches counts against the module. + * + * Floors sit just below the last measured score. Raise one when a change + * raises the score; a floor that is lowered to let a run pass has stopped + * being a floor. + */ +import { existsSync, readFileSync } from "node:fs"; +import { relative, resolve } from "node:path"; + +const root = resolve(import.meta.dirname, ".."); +const reportPath = resolve( + root, + process.argv[2] ?? "reports/mutation/mutation.json", +); +const floorsPath = resolve(root, "stryker.floors.json"); + +if (!existsSync(reportPath)) { + console.error( + `no mutation report at ${reportPath}; run pnpm exec stryker run`, + ); + process.exit(2); +} + +const report = JSON.parse(readFileSync(reportPath, "utf8")); +const { modules } = JSON.parse(readFileSync(floorsPath, "utf8")); + +const DETECTED = new Set(["Killed", "Timeout"]); +const UNDETECTED = new Set(["Survived", "NoCoverage"]); + +const files = Object.entries(report.files ?? {}).map(([path, file]) => [ + relative(root, resolve(root, path)).split("\\").join("/"), + file, +]); + +let failed = false; +const rows = []; +for (const [name, { paths, floor }] of Object.entries(modules)) { + let detected = 0; + let undetected = 0; + let matched = 0; + for (const [path, file] of files) { + if (!paths.some((prefix) => path.startsWith(prefix))) continue; + matched += 1; + for (const mutant of file.mutants ?? []) { + if (DETECTED.has(mutant.status)) detected += 1; + else if (UNDETECTED.has(mutant.status)) undetected += 1; + } + } + const total = detected + undetected; + const score = total === 0 ? null : (100 * detected) / total; + const ok = matched > 0 && score !== null && score >= floor; + if (!ok) failed = true; + rows.push( + `${ok ? "ok " : "FAIL"} ${name.padEnd(16)} ${ + score === null ? "no mutants" : `${score.toFixed(1)} %` + } (floor ${floor} %, ${detected}/${total} mutants, ${matched} files)`, + ); +} + +console.log(rows.join("\n")); +if (failed) { + console.error( + "\nA module scored below its floor, or the report covered none of its files.", + ); + process.exit(1); +} diff --git a/stryker.config.mjs b/stryker.config.mjs new file mode 100644 index 000000000..4f4e93b0d --- /dev/null +++ b/stryker.config.mjs @@ -0,0 +1,63 @@ +/** + * Mutation testing for the core libraries: the modules where a wrong answer + * is silent and costly (encryption at rest, medication compliance, rollups, + * source ranking, unit conversion, backup validation). + * + * Not on pull requests: the full run takes the better part of an hour. It + * runs nightly and on demand (`.github/workflows/mutation.yml`), and locally + * with `pnpm exec stryker run`. The score of each module is checked against + * the floor in `stryker.floors.json` by `node scripts/mutation-floors.mjs`, + * so a floor is per module rather than one number for the whole run. + * + * The tests that run are the unit tests next to these modules (see + * `vitest.mutation.config.mts`), not the whole suite: a mutant that only a + * route test or an integration test would catch counts as surviving here, + * so a score is a lower bound on what the full suite catches. + * + * Vitest 5 names a test "describe > test" where @stryker-mutator/vitest-runner + * 10.0.0 still built "describe test", so every test inside a describe block + * was filtered out of the mutant runs and nearly every mutant survived. + * `patches/@stryker-mutator__vitest-runner@10.0.0.patch` builds the name the + * Vitest 5 way; drop it once a runner release does the same. + */ +const config = { + testRunner: "vitest", + vitest: { configFile: "vitest.mutation.config.mts" }, + plugins: ["@stryker-mutator/vitest-runner"], + mutate: [ + "src/lib/crypto.ts", + "src/lib/analytics/compliance/*.ts", + "src/lib/rollups/measurement-rollups.ts", + "src/lib/rollups/tiered-context.ts", + "src/lib/analytics/source-priority.ts", + "src/lib/analytics/source-rank-sql.ts", + "src/lib/measurements/display-transform.ts", + "src/lib/glucose.ts", + "src/lib/validations/backup.ts", + ], + coverageAnalysis: "perTest", + incremental: true, + incrementalFile: "reports/mutation/stryker-incremental.json", + reporters: ["clear-text", "progress", "html", "json"], + htmlReporter: { fileName: "reports/mutation/index.html" }, + jsonReporter: { fileName: "reports/mutation/mutation.json" }, + // The per-module floors live in stryker.floors.json; the run itself never + // fails on a score, only on an error. + thresholds: { high: 80, low: 60, break: null }, + timeoutMS: 20_000, + tempDirName: ".stryker-tmp", + cleanTempDir: "always", + ignorePatterns: [ + "/.next", + "/e2e", + "/docs", + "/public", + "/assets", + "/reports", + "/playwright-report", + "/test-results", + "/tests/integration", + ], +}; + +export default config; diff --git a/stryker.floors.json b/stryker.floors.json new file mode 100644 index 000000000..ca25a0571 --- /dev/null +++ b/stryker.floors.json @@ -0,0 +1,35 @@ +{ + "$comment": "Per-module mutation-score floors, checked by scripts/mutation-floors.mjs against reports/mutation/mutation.json. Each floor sits just below the score measured when it was set; raise it when a change raises the score, never lower it to let a run pass. See docs/contributing/testing.md#mutation-testing.", + "modules": { + "crypto": { + "paths": ["src/lib/crypto.ts"], + "floor": 45 + }, + "compliance": { + "paths": ["src/lib/analytics/compliance/"], + "floor": 65 + }, + "rollups": { + "paths": ["src/lib/rollups/"], + "floor": 40 + }, + "source-ranking": { + "paths": [ + "src/lib/analytics/source-priority.ts", + "src/lib/analytics/source-rank-sql.ts" + ], + "floor": 55 + }, + "units": { + "paths": [ + "src/lib/measurements/display-transform.ts", + "src/lib/glucose.ts" + ], + "floor": 85 + }, + "backup-validation": { + "paths": ["src/lib/validations/backup.ts"], + "floor": 24 + } + } +} diff --git a/vitest.mutation.config.mts b/vitest.mutation.config.mts new file mode 100644 index 000000000..5bee89f3b --- /dev/null +++ b/vitest.mutation.config.mts @@ -0,0 +1,36 @@ +import { defineConfig } from "vitest/config"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** + * The unit tests Stryker runs against each mutant (see `stryker.config.mjs`): + * the tests that live next to the mutated modules, property tests included. + * Kept narrow on purpose, because Stryker runs this set once to map coverage + * and then the covering tests once per mutant. Same environment as the unit + * suite (`vitest.config.mts`): UTC host zone, the same setup file. + */ +export default defineConfig({ + test: { + globals: true, + environment: "node", + env: { TZ: "UTC" }, + setupFiles: ["./vitest.setup.ts"], + include: [ + "src/lib/__tests__/crypto*.test.ts", + "src/lib/analytics/compliance/__tests__/**/*.test.ts", + "src/lib/rollups/__tests__/**/*.test.ts", + "src/lib/analytics/__tests__/source-*.test.ts", + "src/lib/measurements/__tests__/display-transform*.test.ts", + "src/lib/measurements/__tests__/units.property.test.ts", + "src/lib/__tests__/glucose*.test.ts", + "src/lib/validations/__tests__/backup*.test.ts", + ], + }, + resolve: { + alias: { + "@": resolve(__dirname, "src"), + }, + }, +});