diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85dc38d..1eccf73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,10 +29,11 @@ jobs: # (the lo-roundtrip job below runs that leg). - run: npm test - # The re-save-through-LibreOffice leg, which self-skips everywhere soffice is absent. + # The LibreOffice legs (self-skipping everywhere soffice is absent) and the layout run, + # which holds the editor's page counts against LibreOffice's PDFs in Chromium. lo-roundtrip: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -43,6 +44,8 @@ jobs: - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libreoffice-writer fonts-liberation poppler-utils - run: npm ci - run: npm run test:lo + - run: npx playwright-core install --with-deps chromium + - run: npm run test:layout # Production build booted in the three engines behind the supported browsers # (vite build.target). WebKit runs on macOS: its text shaping there is Safari's. @@ -69,3 +72,7 @@ jobs: - run: npm run build - run: npm run test:smoke - run: npm run test:dom + # xmllint validates the files the monkey saves (macOS ships it). + - if: runner.os == 'Linux' + run: sudo apt-get install -y --no-install-recommends libxml2-utils + - run: npm run test:monkey diff --git a/CLAUDE.md b/CLAUDE.md index 40250a6..f87e1b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,26 +14,24 @@ npm run build # production build → dist/ npm run preview # serve the dist/ build locally npm run check # svelte-check type-check (svelte + ts) npm test # Vitest suite once (tests/**/*.test.ts) -npm run test:watch # Vitest in watch mode -npm run test:lo # LibreOffice legs: round trip + ODT/DOCX render consistency (needs `soffice`) +npm run test:lo # LibreOffice legs: round trip, fuzz re-read, ODT/DOCX render consistency (needs `soffice`) npm run test:smoke # boots the dist/ build headless (tests/smoke/run.mjs); BROWSER=firefox|webkit npm run test:dom # pagination + editing in the real browser (tests/dom/run.mjs); same BROWSER +npm run test:layout # page counts vs LibreOffice + layout invariants over corpus, showcase, fuzz seeds (tests/layout/) +npm run test:monkey # random editing under invariants: schema, undo/redo, the saved file reads back (tests/monkey/) npm run test:coverage # vitest + v8 coverage over src/ → coverage/index.html npm run test:parity # render parity vs LibreOffice (tests/render-parity/README.md) -node scripts/make-thesaurus.mjs # re-vendor public/thesaurus/ from LibreOffice's MyThes data -node scripts/collect-licenses.mjs # regenerate public/licenses.txt after a dependency change +node scripts/make-thesaurus.mjs; node scripts/collect-licenses.mjs # re-vendor public/thesaurus/ (MyThes); regenerate public/licenses.txt node scripts/showcase/run.mjs # rebuild docs/showcase/ (sample .odt/.docx + README screenshots); [regex] limits it ``` Tests live in `tests/` (outside `src/`, so `svelte-check` ignores them), jsdom via Vitest. `roundtrip.test.ts` covers the ODF export↔import round trip + a foreign-doc/style-resolver leg; -`lo-roundtrip.test.ts` re-saves through LibreOffice and **self-skips** without `soffice`, so -`npm test`/CI stay green; `corpus.test.ts` round-trips the committed `tests/corpus/` -documents (by `make-fixtures.mjs` + Word re-saves in `corpus/word/`, never our exporter); +`lo-roundtrip.test.ts` re-saves through LibreOffice and `lo-fuzz.test.ts` reads the fuzz seeds back through it (`LO_SEEDS`, `LO_DUMP=` for triage) — both **self-skip** without `soffice`, so `npm test`/CI stay green; +`corpus.test.ts` round-trips the committed `tests/corpus/` documents (by `make-fixtures.mjs` + Word re-saves in `corpus/word/`, never our exporter); `fuzz-roundtrip.test.ts` round-trips seeded random documents under random export options (`fuzzDoc.ts`, `fuzzOptions.ts`) through both formats and validates every export against the schemas (`schemaValidate.ts`; `FUZZ_SEEDS=500` widens it); -`schema-validation.test.ts` validates the `kitchenSink.ts` exports against the vendored -schemas in `tests/schemas/` (self-skips without `xmllint`); `package-lint.test.ts` checks the semantic invariants schemas can't express (unique ids, dangling references); `tests/unit/` holds fast helper tests; the two browser runs (`tests/smoke/`, `tests/dom/`) share `tests/browser.mjs`. All test tooling stays a -`devDependency`. No linter/formatter. CI (`.github/workflows/ci.yml`) runs `check` + `test`. +`schema-validation.test.ts` validates the `kitchenSink.ts` exports against the vendored schemas in `tests/schemas/` (self-skips without `xmllint`); `package-lint.test.ts` checks the semantic invariants schemas can't express (unique ids, dangling references); `tests/unit/` holds fast helper tests; +the four browser runs (`tests/smoke/`, `tests/dom/`, `tests/layout/`, `tests/monkey/`) share `tests/browser.mjs`. All test tooling stays a `devDependency`. No linter/formatter. CI (`.github/workflows/ci.yml`) runs `check` + `test`. ## Rules diff --git a/docs/headless-testing.md b/docs/headless-testing.md index 3069505..a4ff849 100644 --- a/docs/headless-testing.md +++ b/docs/headless-testing.md @@ -9,7 +9,7 @@ Then load `npm run dev`, inject a document into `localStorage['edentext-doc']`, For PDF-export repros specifically: replicate `pdf.ts`'s clone + `html2canvas(...)` inside `page.evaluate` and read the canvas back as a PNG — capturing the real jsPDF `doc.save()` download tends to hang in headless. Inspect output PDFs with poppler-utils (`apt-get install -y poppler-utils`: `pdftoppm`, `pdfimages`, `pdftotext`). -## The two committed browser runs +## The four committed browser runs `npm run test:smoke` and `npm run test:dom` are these probes made permanent: both boot `dist/` headless through `tests/browser.mjs` (preview server, checklist, `pageerror` @@ -17,6 +17,22 @@ collector) and fail on any uncaught page error. The DOM run keeps the pagination invariants of pass 4 below — one page count through a settle, a reload, a zoom, a page break and its undo — which is the only test `src/lib/components/**` has. +`npm run test:layout` and `npm run test:monkey` run against the dev server (Vite serves +TypeScript, so a page imports `src/` and `tests/` modules directly). The **layout run** +opens every corpus and showcase document plus `LAYOUT_SEEDS` fuzz seeds, holds the editor's +page count against LibreOffice's PDF (`soffice` + `pdfinfo`, a page or a tenth of slack) +and checks the rendered lines: none overlap, none sit in the page gap or past the sheet, +none cross a single-section document's margins, no heading ends a page or column, nothing +is wider than the page. A tab's box, a frame, a formula and the decor are not lines. +The **monkey run** replays `MONKEY_OPS` random keys and commands per seed +(`MONKEY_SEED`, `MONKEY_RUNS`, `MONKEY_DOC`) on a corpus document and checks after each: +no uncaught error, a document its own schema accepts; at the end undo back to the opened +file and redo forward, then Save As both formats, xmllint each against the schemas and read +it back through the app's importer. A failing seed prints its last ops and keeps the file +and both documents under the OS temp dir. Found so far: a note inserted in a text box or a +text box in a note breaks the ODT's XML, and a table in a cell is dropped on save — both +commands now refuse there. + `BROWSER=chromium|firefox|webkit` picks the engine (Chromium by default; the others via `npx playwright-core install firefox webkit`). CI runs both on all three, one per matrix leg, WebKit on macOS because only there its text shaping is Safari's. The engines are diff --git a/package.json b/package.json index 8cdbd75..57238f7 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,11 @@ "check": "svelte-check --tsconfig ./tsconfig.json", "test": "vitest run", "test:watch": "vitest", - "test:lo": "vitest run tests/lo-roundtrip.test.ts tests/lo-render-consistency.test.ts tests/lo-password.test.ts", + "test:lo": "vitest run tests/lo-roundtrip.test.ts tests/lo-fuzz.test.ts tests/lo-render-consistency.test.ts tests/lo-password.test.ts", "test:smoke": "node tests/smoke/run.mjs", "test:dom": "node tests/dom/run.mjs", + "test:layout": "node tests/layout/run.mjs", + "test:monkey": "node tests/monkey/run.mjs", "test:coverage": "vitest run --coverage", "test:parity": "node tests/render-parity/run.mjs", "test:parity:fixtures": "node tests/render-parity/make-fixtures.mjs" diff --git a/scripts/showcase/run.mjs b/scripts/showcase/run.mjs index 8d651d8..32bb54b 100644 --- a/scripts/showcase/run.mjs +++ b/scripts/showcase/run.mjs @@ -2,27 +2,17 @@ // modules are TypeScript the browser imports straight from the dev server, and the app's // own exporters write the .odt/.docx; each is then opened through the file input and shot. // `node scripts/showcase/run.mjs [regex]` limits the run to matching document names. -import { spawn } from 'node:child_process'; import { writeFile, mkdir } from 'node:fs/promises'; import { join } from 'node:path'; -import { ROOT, openApp } from '../../tests/browser.mjs'; +import { ROOT, openApp, devServer } from '../../tests/browser.mjs'; const PORT = 4187; const OUT = join(ROOT, 'docs/showcase'); const only = process.argv[2] ? new RegExp(process.argv[2]) : null; const DOCS = ['thesis', 'thesis-review', 'book', 'newsletter'].filter((n) => !only || only.test(n)); -async function devServer() { - const up = () => fetch(`http://localhost:${PORT}/`).then(() => true).catch(() => false); - if (await up()) return null; - const proc = spawn('npx', ['vite', '--port', String(PORT), '--strictPort'], - { cwd: ROOT, stdio: 'ignore', detached: true }); - for (let i = 0; i < 60 && !(await up()); i++) await new Promise((r) => setTimeout(r, 500)); - return proc; -} - await mkdir(OUT, { recursive: true }); -const server = await devServer(); +const server = await devServer(PORT); const { browser, page, pageErrors } = await openApp(PORT); const url = `http://localhost:${PORT}/`; diff --git a/tests/browser.mjs b/tests/browser.mjs index cad6b7a..e47d651 100644 --- a/tests/browser.mjs +++ b/tests/browser.mjs @@ -1,5 +1,6 @@ -// What the browser runs (smoke, dom) share: the dist/ preview server, the checklist -// both print, and a page wired to fail the run on an uncaught error. +// What the browser runs share: the preview and dev servers, the checklist they print, a +// page wired to fail the run on an uncaught error, and the layout readers (settle, +// extractLayout) the parity, layout and monkey runs measure with. import { spawn, execSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import { join, resolve, dirname } from 'node:path'; @@ -34,6 +35,18 @@ export async function previewServer(port) { throw new Error('preview server did not start'); } +// The dev server: Vite serves TypeScript as is, so a page can import src/ and tests/ +// modules straight from it. Reused when one is already up on the port. +export async function devServer(port) { + const up = () => fetch(`http://localhost:${port}/`).then(() => true).catch(() => false); + if (await up()) return null; + const proc = spawn('npx', ['vite', '--port', String(port), '--strictPort'], + { cwd: ROOT, stdio: 'ignore', detached: true }); + for (let i = 0; i < 120 && !(await up()); i++) await new Promise((r) => setTimeout(r, 500)); + if (!(await up())) throw new Error('dev server did not start'); + return proc; +} + // BROWSER=chromium|firefox|webkit picks the engine (Chromium by default). Fixed locale, // so the UI labels a test clicks are deterministic across machines. export async function openApp(port) { @@ -49,3 +62,119 @@ export async function openApp(port) { page.on('dialog', (d) => d.accept()); return { browser, page, pageErrors }; } + +// Pagination (and the columns flow) runs over several frames; wait for the page +// count and document height to stop moving. An image still decoding has no height +// yet and would settle the layout at a page count it leaves again a frame later. +// `loaded`: the caller knows the file is in (a document may be empty for good). +export async function settle(page, loaded = false) { + await page.waitForFunction((loaded) => { + const el = document.querySelector('.tiptap'); + // Importing a large file takes seconds; an editor still empty is not "settled", + // it is the blank document the file has not replaced yet. + if (!el || (!loaded && !(el.textContent.trim() || el.querySelector('img, table, .textbox-node, .formula, .toc')))) return false; + const imgs = Array.from(document.querySelectorAll('.paper img')); + if (imgs.some((i) => !i.complete)) return false; + // The spacers' heights too, not just their count: they go on settling for a while + // after the last one is placed, and the page a line lands on moves with them. + const spacers = Array.from(document.querySelectorAll('[data-page-break-spacer]')); + // The index's page numbers settle after the spacers do — a stale one is a line of + // text that still changes. + const toc = Array.from(document.querySelectorAll('.toc-page')).map((t) => t.textContent).join(','); + const key = el.style.minHeight + '|' + el.children.length + '|' + spacers.length + '|' + + spacers.reduce((sum, s) => sum + s.offsetHeight, 0) + '|' + toc; + const w = window; + if (w.__parityKey !== key) { w.__parityKey = key; w.__paritySince = performance.now(); return false; } + return performance.now() - (w.__paritySince ?? 0) > 3000; + // Polled, not per frame: the predicate walks every spacer and index row, and on a + // 60-page document doing that each frame starves the layout it is waiting for. + }, loaded, { timeout: 180_000, polling: 500 }); +} + +// Runs in the browser: every rendered word with its page and mm position. +export function extractLayout() { + const PAGE_GAP = 20, PX_MM = 25.4 / 96; + const paper = document.querySelector('.paper'); + const cs = getComputedStyle(document.documentElement); + const pageH = parseFloat(cs.getPropertyValue('--user-page-height')); + const pageW = parseFloat(cs.getPropertyValue('--user-page-width')); + const cycle = pageH + PAGE_GAP; + const origin = paper.getBoundingClientRect(); + + const skip = (node) => { + for (let e = node.parentElement; e && e !== paper; e = e.parentElement) { + if (e.hasAttribute('data-page-break-spacer') || e.classList.contains('band-layer')) return true; + const s = getComputedStyle(e); + if (s.display === 'none' || s.visibility === 'hidden') return true; + } + return false; + }; + + // A word broken mid-word — a token wider than its line, or one carrying a hyphen or a + // slash — has a client rect per line, and pdftotext reads those as separate words. So + // split it the same way, by the line each character lands on. + const fragments = (node, from, to) => { + const r = document.createRange(); + r.setStart(node, from); r.setEnd(node, to); + const rects = r.getClientRects(); + const whole = (rect) => ({ top: rect.top, left: rect.left, width: rect.width, height: rect.height, text: node.nodeValue.slice(from, to) }); + if (rects.length < 2) return rects.length ? [whole(rects[0])] : []; + const out = []; + for (let i = from; i < to; i++) { + const c = document.createRange(); + c.setStart(node, i); c.setEnd(node, i + 1); + const rect = c.getClientRects()[0]; + if (!rect) continue; + const last = out[out.length - 1]; + if (last && Math.abs(last.top - rect.top) < 1) { last.text += node.nodeValue[i]; last.right = rect.right; } + else out.push({ top: rect.top, left: rect.left, right: rect.right, height: rect.height, text: node.nodeValue[i] }); + } + return out.map((f) => ({ top: f.top, left: f.left, width: f.right - f.left, height: f.height, text: f.text })); + }; + + // What the page paints, not what the node holds: a style's fo:text-transform / + // w:caps is a CSS transform here and real uppercase in the reference's PDF. + const painted = (text, el) => { + const t = getComputedStyle(el).textTransform; + if (t === 'uppercase') return text.toUpperCase(); + if (t === 'lowercase') return text.toLowerCase(); + if (t === 'capitalize') return text.replace(/^\p{L}/u, (c) => c.toUpperCase()); + return text; + }; + + const words = []; + const walker = document.createTreeWalker(paper, NodeFilter.SHOW_TEXT); + for (let n = walker.nextNode(); n; n = walker.nextNode()) { + const text = n.nodeValue; + if (!text || !text.trim() || skip(n)) continue; + for (const m of text.matchAll(/\S+/g)) { + for (const f of fragments(n, m.index, m.index + m[0].length)) { + if (!f.width) continue; + const y = f.top - origin.top; + const page = Math.floor(y / cycle); + words.push({ + text: painted(f.text, n.parentElement), + page, + x: (f.left - origin.left) * PX_MM, + y: (y - page * cycle) * PX_MM, + w: f.width * PX_MM, + h: f.height * PX_MM, + }); + } + } + } + // Folded, not spread: a several-hundred-page document has more words than a call + // takes arguments, and Math.max(...words) then blows the stack instead of measuring. + const numPages = words.reduce((m, w) => (w.page > m ? w.page : m), 0) + 1; + const pages = Array.from({ length: numPages }, (_, i) => ({ + words: words.filter((w) => w.page === i).map(({ page, ...r }) => r), + width: pageW * PX_MM, + height: pageH * PX_MM, + })); + const mm = (v) => parseFloat(cs.getPropertyValue(v)) * PX_MM; + return { + pages, + margins: { top: mm('--user-margin-top'), bottom: mm('--user-margin-bottom'), + left: mm('--user-margin-left'), right: mm('--user-margin-right') }, + }; +} diff --git a/tests/fuzz-roundtrip.test.ts b/tests/fuzz-roundtrip.test.ts index 14c20f7..f6d7ba5 100644 --- a/tests/fuzz-roundtrip.test.ts +++ b/tests/fuzz-roundtrip.test.ts @@ -9,19 +9,15 @@ import { buildDocx } from '../src/lib/export/docx'; import { importDocx } from '../src/lib/import/docx'; import { normalize, firstDiff, stripFontHoist } from './normalize'; import { mulberry32 } from './fuzzDoc'; -import { genCase, exportArgs, expectedOptions, importedOptions, diffLoose, omit } from './fuzzOptions'; +import { genCase, exportArgs, expectedOptions, importedOptions, diffLoose, omit, DOCX_LOSSY, docWideOddEven } from './fuzzOptions'; import { hasXmllint, validateOdt, validateDocx } from './schemaValidate'; const SEEDS = Number(process.env.FUZZ_SEEDS ?? 50); // FUZZ_SEEDS=500 for a wide sweep // The schema leg pays ~0.7s of xmllint per seed, so it takes fewer by default. const SCHEMA_SEEDS = Number(process.env.SCHEMA_SEEDS ?? Math.min(SEEDS, 20)); -// What each format has no place for (see the importers' CLAUDE.md); dropped from both -// sides of its comparison. Word spaces paragraphs by the larger of the two values and -// draws its note separator its own way; it has no note prefix/suffix and counts every line. +// What ODT has no place for: nothing so far (DOCX_LOSSY lists Word's). const ODT_LOSSY: string[] = []; -const DOCX_LOSSY: string[] = ['spacingModel', 'notes.separator', 'notes.footnote.prefix', 'notes.footnote.suffix', - 'notes.endnote.prefix', 'notes.endnote.suffix', 'lineNumbering.countEmpty']; describe('fuzz round-trip: editor → buildOdt → importOdt', () => { it(`${SEEDS} seeded random documents come back identical`, async () => { @@ -47,17 +43,6 @@ describe('fuzz round-trip: editor → buildOdt → importOdt', () => { return node; } - // Odd/even pages are a document setting in Word (w:evenAndOddHeaders): once any - // section asks for it, every section has it, repeating its running zones. - function docWideOddEven(canon: any): any { - if (!canon.sections.some((s: any) => s.differentOddEven)) return canon; - for (const s of canon.sections) { - if (s.differentOddEven) continue; - Object.assign(s, { differentOddEven: true, headerEven: s.header, footerEven: s.footer }); - } - return canon; - } - // The DOCX leg: its options against what went out, and its content against the ODT // reading — catches one exporter silently losing what the other keeps. it(`${SEEDS} seeded documents survive DOCX and agree with the ODT round-trip`, async () => { diff --git a/tests/fuzzOptions.ts b/tests/fuzzOptions.ts index 55c7f5a..1885755 100644 --- a/tests/fuzzOptions.ts +++ b/tests/fuzzOptions.ts @@ -311,6 +311,23 @@ export function diffLoose(a: N, b: N, path = '$'): string | null { return null; } +// What DOCX has no place for (see the importers' CLAUDE.md), dropped from both sides of a +// comparison: Word spaces paragraphs by the larger of the two values, draws its note +// separator its own way, has no note prefix/suffix and counts every line. +export const DOCX_LOSSY = ['spacingModel', 'notes.separator', 'notes.footnote.prefix', 'notes.footnote.suffix', + 'notes.endnote.prefix', 'notes.endnote.suffix', 'lineNumbering.countEmpty']; + +// Odd/even pages are a document setting in Word (w:evenAndOddHeaders): once any section +// asks for it, every section has it, repeating its running zones. +export function docWideOddEven(canon: N): N { + if (!canon.sections.some((s: N) => s.differentOddEven)) return canon; + for (const s of canon.sections) { + if (s.differentOddEven) continue; + Object.assign(s, { differentOddEven: true, headerEven: s.header, footerEven: s.footer }); + } + return canon; +} + // Drops the paths a format cannot carry (`a.b`, `sections.*.c`) from a canonical reading. export function omit(canon: N, paths: readonly string[]): N { const out = structuredClone(canon); diff --git a/tests/layout/run.mjs b/tests/layout/run.mjs new file mode 100644 index 0000000..a650c67 --- /dev/null +++ b/tests/layout/run.mjs @@ -0,0 +1,202 @@ +// Layout run: the documents the repo carries — corpus, showcase and fuzz seeds — opened in +// the editor, the page count held against LibreOffice's PDF, and the rendered lines checked +// for what no layout may show: overlapping lines, text off the page or past the margins, a +// heading alone at the foot of a page or column. `[regex]` limits the run by file name. +import { execFileSync, execSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, existsSync, copyFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, basename, extname, relative } from 'node:path'; +import { ROOT, checker, devServer, openApp, settle } from '../browser.mjs'; + +const PORT = +(process.env.LAYOUT_PORT ?? 4188); +const SEEDS = +(process.env.LAYOUT_SEEDS ?? 10); +const only = process.argv[2] ? new RegExp(process.argv[2]) : null; +const has = (tool) => { try { execSync(`command -v ${tool}`, { stdio: 'ignore' }); return true; } catch { return false; } }; +const LO = has('soffice') && has('pdfinfo'); +const { check, failures } = checker(); + +const files = (dir) => (existsSync(dir) + ? readdirSync(dir).filter((f) => /\.(odt|docx)$/.test(f) && !f.startsWith('~$')).sort().map((f) => join(dir, f)) + : []); +const docs = [...files(join(ROOT, 'tests/corpus')), ...files(join(ROOT, 'tests/corpus/word')), ...files(join(ROOT, 'docs/showcase'))]; +const work = mkdtempSync(join(tmpdir(), 'layout-')); +const server = await devServer(PORT); +const { browser, page, pageErrors } = await openApp(PORT); +if (!LO) console.log('soffice/pdfinfo missing: page counts are not held against LibreOffice'); + +// LibreOffice's page count from its PDF, the blank pages it inserts itself kept as the +// editor keeps them; one soffice call for the whole batch, under unique names. +function loPages(paths) { + const inDir = join(work, 'in'), out = join(work, 'pdf'); + mkdirSync(inDir, { recursive: true }); mkdirSync(out, { recursive: true }); + const named = paths.map((p, i) => { const f = join(inDir, `${i}-${basename(p)}`); copyFileSync(p, f); return f; }); + execFileSync('soffice', ['--headless', '--norestore', `-env:UserInstallation=file://${work}/profile`, '--convert-to', + 'pdf:writer_pdf_Export:{"IsSkipEmptyPages":{"type":"boolean","value":"false"}}', '--outdir', out, ...named], + { stdio: 'pipe', timeout: 900_000 }); + const pages = new Map(); + named.forEach((f, i) => { + const pdf = join(out, basename(f, extname(f)) + '.pdf'); + const m = existsSync(pdf) && /Pages:\s+(\d+)/.exec(execFileSync('pdfinfo', [pdf], { encoding: 'utf8' })); + if (m) pages.set(paths[i], +m[1]); + }); + return pages; +} + +// Runs in the browser: the rendered lines against the page sheets and the document's +// margins (single-section documents only: a later section may have margins of its own). +function lintLayout() { + const paper = document.querySelector('.paper'); + const origin = paper.getBoundingClientRect(); + const rel = (r) => ({ top: r.top - origin.top, bottom: r.bottom - origin.top, left: r.left - origin.left, right: r.right - origin.left }); + const sheets = [...document.querySelectorAll('.page-sheet')].map((s) => rel(s.getBoundingClientRect())); + const pageOf = (y) => sheets.findIndex((s) => y >= s.top - 1 && y <= s.bottom + 1); + const cs = getComputedStyle(document.documentElement); + const px = (v) => parseFloat(cs.getPropertyValue(v)); + const margins = { top: px('--user-margin-top'), bottom: px('--user-margin-bottom'), left: px('--user-margin-left'), right: px('--user-margin-right') }; + const oneSection = !cs.getPropertyValue('--pb-section-page').includes(','); + const mirrored = /[1-9]/.test(cs.getPropertyValue('--pb-section-mirror')); + const side = mirrored ? Math.min(margins.left, margins.right) : null; + + // Where a text node is painted: a frame may sit on the text by design, a formula + // stacks its own boxes, and a hidden or spacer node is not painted at all. + const originOf = (node) => { + for (let e = node.parentElement; e && e !== paper; e = e.parentElement) { + if (e.hasAttribute('data-page-break-spacer') || e.classList.contains('band-layer')) return null; + if (e.classList.contains('page-decor-layer') || e.classList.contains('line-number-layer')) return null; + if (e.classList.contains('hf-layer')) return 'hf'; + if (e.classList.contains('textbox-node') || e.classList.contains('image-node')) return 'frame'; + if (e.classList.contains('formula')) return 'formula'; + if (e.classList.contains('note')) return 'note'; + const s = getComputedStyle(e); + if (s.display === 'none' || s.visibility === 'hidden') return null; + } + return 'body'; + }; + const lines = []; + const walker = document.createTreeWalker(paper, NodeFilter.SHOW_TEXT); + for (let n = walker.nextNode(); n; n = walker.nextNode()) { + if (!n.nodeValue.trim()) continue; + const kind = originOf(n); + if (!kind) continue; + // Between the tabs: a tab's own box runs to its stop, past a cell's edge if need be. + for (const m of n.nodeValue.matchAll(/[^\t]+/g)) { + const range = document.createRange(); + range.setStart(n, m.index); range.setEnd(n, m.index + m[0].length); + for (const r of range.getClientRects()) { + if (r.width < 0.5 || r.height < 0.5) continue; + const b = rel(r); + lines.push({ ...b, kind, page: pageOf((b.top + b.bottom) / 2), text: m[0].trim().slice(0, 40) }); + } + } + } + const issues = []; + const box = (b) => [b.left, b.top, b.right, b.bottom].map(Math.round).join(','); + const issue = (kind, l, extra = {}) => { if (issues.length < 60) issues.push({ kind, page: l.page + 1, text: l.text, at: l.left != null ? box(l) : undefined, ...extra }); }; + + // Two painted lines on one page may not share their box (a frame may, a formula's + // pieces may). ponytail: a sorted sweep, quadratic only across lines that share a band. + lines.sort((a, b) => a.top - b.top); + for (let i = 0; i < lines.length; i++) { + const a = lines[i]; + if (a.kind === 'frame' || a.kind === 'formula') continue; + for (let j = i + 1; j < lines.length && lines[j].top < a.bottom; j++) { + const b = lines[j]; + if (b.kind === 'frame' || b.kind === 'formula' || b.page !== a.page) continue; + const dy = Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top); + const dx = Math.min(a.right, b.right) - Math.max(a.left, b.left); + if (dy > 0.5 * Math.min(a.bottom - a.top, b.bottom - b.top) && dx > 2) issue('overlap', a, { other: b.text, otherKind: b.kind, otherAt: box(b) }); + } + } + // Body text stays on a sheet and, in a single-section document, inside the margins. + for (const l of lines) { + if (l.kind !== 'body' && l.kind !== 'note') continue; + if (l.page < 0) { issue('in the page gap', l); continue; } + const s = sheets[l.page]; + if (l.left < s.left - 1 || l.right > s.right + 1) issue('off the page', l, { sheet: box(s) }); + if (!oneSection) continue; + const [ml, mr] = side == null ? [margins.left, margins.right] : [side, side]; + const body = { left: s.left + ml, top: s.top + margins.top, right: s.right - mr, bottom: s.bottom - margins.bottom }; + if (l.top < body.top - 3) issue('above the top margin', l, { body: box(body) }); + if (l.bottom > body.bottom + 3) issue('below the bottom margin', l, { body: box(body) }); + // ponytail: 6px of slack for a justified line's overhang and the sheet's own border. + if (l.left < body.left - 6 || l.right > body.right + 6) issue('beyond the side margin', l, { body: box(body) }); + } + // A heading stays with the block it introduces, across a page and a column alike, + // unless that block asks for the break itself. + const blocks = (parent) => [...parent.children].filter((e) => !e.hasAttribute('data-page-break-spacer')); + for (const h of paper.querySelectorAll('.tiptap :is(h1, h2, h3, h4, h5, h6)')) { + if (h.closest('td, th, li, .textbox-node, .hf-layer')) continue; + const sib = blocks(h.parentElement); + const next = sib[sib.indexOf(h) + 1]; + if (!next || next.classList.contains('note') || next.hasAttribute('data-page-break-before')) continue; + const hr = rel(h.getBoundingClientRect()), nr = rel(next.getBoundingClientRect()); + if (hr.bottom - hr.top < 1 || nr.bottom - nr.top < 1) continue; + const l = { page: pageOf(hr.bottom - 1), text: h.textContent.trim().slice(0, 40) }; + if (pageOf(nr.top + 1) !== l.page) issue('heading alone at the page end', l); + else if (h.closest('.columns-node') && nr.left > hr.left + 5) issue('heading alone at the column end', l); + } + const tiptap = document.querySelector('.tiptap'); + if (tiptap.scrollWidth > tiptap.clientWidth + 1) issues.push({ kind: 'content wider than the page', page: 0, text: `${tiptap.scrollWidth}px in ${tiptap.clientWidth}px` }); + return { pages: sheets.length, lines: lines.length, issues }; +} + +const b64 = (u8) => { let s = ''; for (let i = 0; i < u8.length; i += 0x8000) s += String.fromCharCode(...u8.subarray(i, i + 0x8000)); return btoa(s); }; + +try { + await page.goto(`http://localhost:${PORT}/`, { waitUntil: 'load' }); + await page.waitForSelector('.tiptap', { timeout: 30_000 }); + // The fuzz seeds, written by the app's own exporter in the page. + mkdirSync(join(work, 'fuzz'), { recursive: true }); + for (let seed = 1; seed <= SEEDS; seed++) { + const odt = await page.evaluate(async ({ seed, b64 }) => { + const [{ buildOdt }, { mulberry32 }, { genCase, exportArgs }] = await Promise.all([ + import('/src/lib/export/odt.ts'), import('/tests/fuzzDoc.ts'), import('/tests/fuzzOptions.ts')]); + const { doc, opts } = genCase(mulberry32(seed)); + return new Function('return ' + b64)()(await buildOdt(doc, ...exportArgs(opts))); + }, { seed, b64: b64.toString() }); + const f = join(work, 'fuzz', `seed-${seed}.odt`); + writeFileSync(f, Buffer.from(odt, 'base64')); + docs.push(f); + } + const todo = docs.filter((d) => !only || only.test(d)); + const retried = new Set(); + const lo = LO ? loPages(todo) : new Map(); + + for (const file of todo) { + const name = file.startsWith(ROOT) ? relative(ROOT, file) : `fuzz/${basename(file)}`; + try { + // A fresh editor for each document, so its settle cannot be the previous one's. + await page.evaluate(() => localStorage.clear()); + await page.reload({ waitUntil: 'load' }); + await page.waitForSelector('.tiptap', { timeout: 30_000 }); + await page.setInputFiles('input.file-input', file); + // The document's name says the file is in; a document may hold nothing to wait for. + await page.waitForFunction((n) => document.querySelector('.doc-name-input')?.value.startsWith(n), basename(file, extname(file)), { timeout: 60_000 }); + await settle(page, true); + const r = await page.evaluate(lintLayout); + const ref = lo.get(file); + const off = ref == null ? 0 : r.pages - ref; + // A page or a tenth either way is the slack two layout engines take. + const pagesOk = Math.abs(off) <= Math.max(1, Math.round((ref ?? 0) / 10)); + const label = `${name} editor ${r.pages}p${ref != null ? ` / LO ${ref}p` : ''}${off && pagesOk ? ' (~)' : ''} ${r.issues.length} issue(s)`; + check(pagesOk && !r.issues.length, label); + for (const i of r.issues.slice(0, 8)) { + const where = i.other ? ` × ${i.otherKind} "${i.other}" @${i.otherAt}` : i.body ? ` (body ${i.body})` : i.sheet ? ` (sheet ${i.sheet})` : ''; + console.log(` p${i.page} ${i.kind}: "${i.text}"${i.at ? ` @${i.at}` : ''}${where}`); + } + if (r.issues.length > 8) console.log(` … ${r.issues.length - 8} more`); + } catch (err) { + // A layout that never settles is a finding of its own, not the end of the run; a + // page torn down under the probe (the app's own reload) gets one more go. + if (/Execution context was destroyed/.test(err.message) && !retried.has(file)) { retried.add(file); todo.push(file); continue; } + check(false, `${name}: ${err.message.split('\n')[0]}`); + } + } +} catch (err) { + check(false, `layout run threw: ${err.stack ?? err}`); +} finally { + check(pageErrors.length === 0, pageErrors.length ? `no uncaught page errors — got: ${pageErrors.join(' | ')}` : 'no uncaught page errors'); + await browser.close(); + if (server) process.kill(-server.pid); +} +process.exit(failures.length ? 1 : 0); diff --git a/tests/lo-fuzz.test.ts b/tests/lo-fuzz.test.ts new file mode 100644 index 0000000..4e041ed --- /dev/null +++ b/tests/lo-fuzz.test.ts @@ -0,0 +1,250 @@ +// LibreOffice as the fuzz's second reader: each seed's ODT re-saved by soffice, and its +// DOCX converted to ODT, both read back and held against what went out. Where LibreOffice +// understands a file differently from our importer, one of the two is wrong. +import { describe, it, expect } from 'vitest'; +import { execFileSync, execSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { buildOdt } from '../src/lib/export/odt'; +import { buildDocx } from '../src/lib/export/docx'; +import { importOdt } from '../src/lib/import/odt'; +import { normalize, stripFontHoist } from './normalize'; +import { mulberry32 } from './fuzzDoc'; +import { genCase, exportArgs, expectedOptions, importedOptions, diffLoose, omit, DOCX_LOSSY, docWideOddEven, type FuzzOptions } from './fuzzOptions'; + +type N = any; +type Fmt = 'odt' | 'docx'; + +const SEEDS = Number(process.env.LO_SEEDS ?? 10); // LO_SEEDS=100 for a wide sweep +const DUMP = process.env.LO_DUMP; // a path: every diff of every seed, for triage +const SOFFICE = (() => { try { execSync('command -v soffice', { stdio: 'ignore' }); return true; } catch { return false; } })(); + +// What LibreOffice's DOCX reader has no place for beyond Word's own list: fold marks are +// our construct, and heading numbering comes back as list styles on the heading styles. +const LOSSY: Record = { odt: [], docx: [...DOCX_LOSSY, 'foldMarks', 'outline', 'spacingAtPageStart'] }; + +// One soffice call per batch: a cold start costs seconds, a document milliseconds. Its +// own profile, so a LibreOffice the user has open does not swallow the call. +function convertAll(files: string[], outDir: string): void { + mkdirSync(outDir, { recursive: true }); + execFileSync('soffice', ['--headless', '--norestore', `-env:UserInstallation=file://${join(outDir, '..', 'profile')}`, + '--convert-to', 'odt', '--outdir', outDir, ...files], { stdio: 'pipe', timeout: 900_000 }); +} + +// A font a paragraph carries for all its runs is written on the paragraph alone by +// LibreOffice; read it back onto the runs, where the authored document keeps it. +function unhoist(node: N): N { + const { fontSize, fontFamily } = node.attrs ?? {}; + if ((fontSize || fontFamily) && node.content) { + for (const c of node.content) { + if (c.type !== 'text') continue; + const ts = (c.marks ??= []).find((m: N) => m.type === 'textStyle') ?? (c.marks.push({ type: 'textStyle', attrs: {} }), c.marks[c.marks.length - 1]); + ts.attrs = { ...(fontSize ? { fontSize } : {}), ...(fontFamily ? { fontFamily } : {}), ...ts.attrs }; + } + } + for (const c of node.content ?? []) unhoist(c); + return node; +} + +// LibreOffice's own reading: pictures re-encoded, sizes rounded through cm, a width on +// every column, a hyperlink in its Internet Link style, a comment dated by its own clock; +// from DOCX an index mark is dropped, a list style "WWNum", a lone operator quoted text. +function loNoise(node: N, fmt: Fmt): N { + const a = node.attrs ?? {}; + for (const [k, v] of Object.entries(a)) { + if (k === 'src') a.src = 'IMG'; + else if ((k === 'width' || k === 'height' || k === 'rowHeight') && typeof v === 'number') a[k] = Math.round(v / 3) * 3; + else if (k === 'colwidth' || (k === 'wrapOffsetY' && v === 0)) delete a[k]; + else if ((k === 'marginLeft' || k === 'marginRight') && typeof v === 'number') { if (Math.abs(v) < 0.05) delete a[k]; else a[k] = Math.round(v * 10) / 10; } + else if ((k === 'bulletChar' || k === 'textVertical' || k === 'alt') && fmt === 'docx') delete a[k]; + else if (k === 'textPosition' && typeof v === 'number') a[k] = Math.round(v * 2) / 2; + else if (k.startsWith('border') && typeof v === 'string') a[k] = v.replace(/^([\d.]+)pt/, (_m, w) => `${Math.round(Number(w) * 4) / 4}pt`); + else if (k === 'latex' && fmt === 'docx') a.latex = String(v).replace(/\\text\{(.)\}/g, '$1'); + else if ((k === 'listStyleName' || k === 'listStyleType') && fmt === 'docx') delete a[k]; + else if ((k === 'styleName' || k === 'name') && typeof v === 'string') a[k] = v.replace(/ \(WW\)$/, ''); + } + // A note's label and a citation's fields are recomputed on its side; a table style is + // its look baked into the cells there, so a styled table compares by its text alone. + if (node.type === 'noteRef' || node.type === 'note') delete a.text; + // A note's text opens with the tab LibreOffice's DOCX reader puts after the number; a + // figure or table index is a paragraph to it. + if (node.type === 'note' && fmt === 'docx') { + // A custom mark comes back as the note's first run, in the symbol style. + if (node.content?.[0]?.marks?.some((m: N) => /Symbol$/.test(m.attrs?.name ?? ''))) node.content.shift(); + if (node.content?.[0]?.text) node.content[0].text = node.content[0].text.replace(/^\t/, ''); + } + // Its DOCX reader spells a heading paragraph's zero spacing out. + if (fmt === 'docx' && node.type === 'paragraph') { if (a.spaceAfter === 0) delete a.spaceAfter; if (a.spaceBefore === 0) delete a.spaceBefore; } + if (node.type === 'tableOfContents' && fmt === 'docx' && ['tables', 'figures'].includes(a.index)) { node.attrs = undefined; delete node.content; node.type = 'paragraph'; } + if (node.type === 'bibliographyEntry' && fmt === 'docx') node.attrs = {}; + // Word centres a display formula's paragraph; LibreOffice reads that as its alignment. + if (fmt === 'docx' && node.type === 'paragraph' && node.content?.length === 1 && node.content[0].type === 'formula' && node.content[0].attrs?.display && a.textAlign === 'center') delete a.textAlign; + if (node.attrs && !Object.keys(a).length) delete node.attrs; + // A comment's date comes back in LibreOffice's local time; a tracked deletion wears + // its review look (strike, underline) as formatting; a run in a character style has + // that style's font written out beside it. + for (const m of node.marks ?? []) { + if (m.type === 'comment' && m.attrs) delete m.attrs.date; + // A style name LibreOffice's DOCX reader reserves for a Word name comes back "… (WW)". + if (m.type === 'charStyle' && m.attrs?.name) m.attrs.name = m.attrs.name.replace(/ \(WW\)$/, ''); + } + // A tracked deletion comes back in the look of the run beside it; a raised run's + // percentage is rounded on the way. + if (node.marks?.some((m: N) => m.type === 'deletion' || m.type === 'insertion')) { + node.marks = node.marks.filter((m: N) => ['deletion', 'insertion', 'link'].includes(m.type)); + } + for (const m of node.marks ?? []) if (typeof m.attrs?.textPosition === 'number') m.attrs.textPosition = Math.round(m.attrs.textPosition * 2) / 2; + if (node.marks?.some((m: N) => m.type === 'charStyle')) for (const m of node.marks) if (m.type === 'textStyle' && m.attrs) delete m.attrs.fontFamily; + if (node.marks) { + node.marks = node.marks.filter((m: N) => m.type !== 'textStyle' || Object.keys(m.attrs ?? {}).length); + if (!node.marks.length) delete node.marks; + } + if (node.marks?.some((m: N) => m.type === 'link')) { + node.marks = node.marks.filter((m: N) => m.type !== 'underline'); + for (const m of node.marks) if (m.type === 'textStyle' && m.attrs) delete m.attrs.color; + node.marks = node.marks.filter((m: N) => m.type !== 'textStyle' || Object.keys(m.attrs ?? {}).length); + if (!node.marks.length) delete node.marks; + } + // A paragraph in a text box loses its own field and rule lines on the way. + if (node.type === 'textBox') for (const p of node.content ?? []) for (const k of Object.keys(p.attrs ?? {})) if (k === 'backgroundColor' || k.startsWith('border')) delete p.attrs[k]; + if (node.content) { + // Its DOCX reader drops an index mark; beside a ruby it moves one and flattens the + // ruby to its base text, so both compare as that. + const ruby = node.content.some((c: N) => c.type === 'ruby'), mark = node.content.some((c: N) => c.type === 'indexEntry'); + node.content = node.content.filter((c: N) => !((fmt === 'docx' || ruby) && c.type === 'indexEntry')) + .map((c: N) => (fmt === 'odt' && ruby && mark && c.type === 'ruby' ? { type: 'text', text: c.attrs.base } : loNoise(c, fmt))); + // A document may not end on a table in Writer, so one gets a paragraph after it + // (ahead of the notes, which the model keeps last). + const end = node.content.length - (node.content[node.content.length - 1]?.type === 'noteSection' ? 2 : 1); + const last = node.content[end], prev = node.content[end - 1]; + if (node.type === 'doc' && last?.type === 'paragraph' && !last.content && !last.attrs && prev?.type === 'table') node.content.splice(end, 1); + if (!node.content.length) delete node.content; + } + return node; +} +// A table style is its look baked into the cells on LibreOffice's side, and a formula +// cell shows the value it recomputes in its locale's format: the authored document's +// styled tables compare by their text alone, its formula cells by nothing. +type Marked = { styled: Set; formulas: Set }; +function markedTables(doc: N): Marked { + const styled = new Set(), formulas = new Set(); + (doc.content ?? []).forEach((b: N, i: number) => { + if (b.type !== 'table') return; + if (b.attrs?.tableStyle) styled.add(i); + b.content?.forEach((row: N, r: number) => row.content?.forEach((cell: N, c: number) => { if (cell.attrs?.formula) formulas.add(`${i}/${r}/${c}`); })); + }); + return { styled, formulas }; +} +function bareTables(doc: N, marked: Marked, fmt: Fmt): N { + doc.content?.forEach((b: N, i: number) => { + if (b.type !== 'table') return; + if (marked.styled.has(i)) { + delete b.attrs?.tableStyle; delete b.attrs?.tableLook; + (function bare(n: N) { delete n.marks; if (n.attrs) { delete n.attrs.region; if (!Object.keys(n.attrs).length) delete n.attrs; } for (const c of n.content ?? []) bare(c); })(b); + } + b.content?.forEach((row: N, r: number) => row.content?.forEach((cell: N, c: number) => { + if (!marked.formulas.has(`${i}/${r}/${c}`)) return; + delete cell.content; + if (fmt === 'docx' && cell.attrs) { delete cell.attrs.formula; delete cell.attrs.cellFormat; if (!Object.keys(cell.attrs).length) delete cell.attrs; } + })); + }); + return doc; +} +const loContent = (doc: N, fmt: Fmt, marked: Marked) => bareTables(loNoise(stripFontHoist(normalize(unhoist(structuredClone(doc)))), fmt), marked, fmt); + +// The canonical options as LibreOffice hands them back: a chapter field's cached name is +// recomputed (and via DOCX, a STYLEREF field is not read at all); a watermark rides a +// header band it floors at 0.499cm, so a section without a header of its own grows by it. +function loOptions(canon: N, fmt: Fmt, opts: FuzzOptions, authored: boolean): N { + // Via DOCX a zone's runs come back in the look LibreOffice gives fields and headings, + // so there a zone compares by its text and fields alone. + const zoneFix = (zone: N): void => { + for (const p of zone?.content ?? []) { + if (fmt === 'docx') p.content = p.content?.filter((c: N) => c.type !== 'chapterField').map((c: N) => (c.type === 'text' ? { type: 'text', text: c.text } : { type: c.type })); + for (const c of p.content ?? []) if (c.type === 'chapterField') delete c.attrs.text; + if (p.content && !p.content.length) delete p.content; + } + }; + // A style naming itself as next names the default; a rule's width is rounded through cm. + for (const [name, st] of Object.entries(canon.styles ?? {}) as [string, N][]) { + if (!st) continue; + if (st.next === name) st.next = null; + for (const [k, v] of Object.entries(st.para ?? {})) if (k.startsWith('border') && typeof v === 'string') st.para[k] = (v as string).replace(/^([\d.]+)pt/, (_m, w) => `${Math.round(Number(w) * 4) / 4}pt`); + } + for (const s of canon.sections) { + for (const k of ['header', 'footer', 'headerFirst', 'footerFirst', 'headerEven', 'footerEven']) zoneFix(s[k]); + if (authored && fmt === 'odt' && (opts.decor.watermark || opts.foldMarks) && !s.header && !s.headerFirst && !s.headerEven && s.margins) { + s.margins.top = Math.round((s.margins.top + 0.499) * 1000) / 1000; + } + } + return fmt === 'docx' && authored ? docWideOddEven(canon) : canon; +} + +// Every leaf that differs, for the LO_DUMP triage file (firstDiff stops at the first). +function allDiffs(a: N, b: N, path = '$', out: string[] = []): string[] { + if (out.length >= 12 || (a == null && b == null)) return out; + if (typeof a === 'number' && typeof b === 'number') { if (Math.abs(a - b) > 0.02) out.push(`${path}: ${a} vs ${b}`); return out; } + if (typeof a !== typeof b || a == null || b == null || typeof a !== 'object') { + if (!Object.is(a, b)) out.push(`${path}: ${JSON.stringify(a)?.slice(0, 300)} vs ${JSON.stringify(b)?.slice(0, 300)}`); + return out; + } + for (const k of new Set([...Object.keys(a), ...Object.keys(b)])) allDiffs(a[k], b[k], `${path}.${k}`, out); + return out; +} + +// Writer joins two tables that touch, as its DOCX reader finds ours; that reader also +// swallows the blocks after an index field with no entries, and restructures a list in a +// table cell, a floating picture in a list item and a list three deep: nothing to compare. +const adjacentTables = (doc: N) => (doc.content ?? []).some((b: N, i: number) => b.type === 'table' && doc.content[i + 1]?.type === 'table'); +const emptyIndex = (doc: N) => (doc.content ?? []).some((b: N) => b.type === 'tableOfContents' && !(b.attrs?.entries ?? []).length); +const has = (n: N, test: (x: N) => boolean): boolean => test(n) || (n.content ?? []).some((c: N) => has(c, test)); +const listInCell = (doc: N) => has(doc, (n) => n.type === 'tableCell' && has(n, (x) => /List$/.test(x.type))); +const floating = (n: N) => has(n, (x) => x.type === 'image' && x.attrs?.wrap && x.attrs.wrap !== 'inline'); +const floatInList = (doc: N) => has(doc, (n) => n.type === 'listItem' && floating(n)) + || (doc.content ?? []).some((b: N, i: number) => isList(b) && doc.content[i + 1] && floating(doc.content[i + 1])); +const isList = (n: N) => /List$/.test(n.type); +const deepList = (doc: N) => has(doc, (n) => isList(n) && (n.content ?? []).some((li: N) => (li.content ?? []).some((x: N) => isList(x) && (x.content ?? []).some((li2: N) => (li2.content ?? []).some(isList))))); +const hardForDocx = (doc: N) => adjacentTables(doc) || emptyIndex(doc) || listInCell(doc) || floatInList(doc) || deepList(doc); + +describe.skipIf(!SOFFICE)('fuzz through LibreOffice (needs soffice on PATH)', () => { + const work = mkdtempSync(join(tmpdir(), 'lo-fuzz-')); + const cases = Array.from({ length: SEEDS }, (_, i) => ({ seed: i + 1, ...genCase(mulberry32(i + 1)) })); + const dump: string[] = []; + + for (const [fmt, build] of [['odt', buildOdt], ['docx', buildDocx]] as const) { + it(`${SEEDS} seeds as ${fmt.toUpperCase()}: LibreOffice reads them as they were written`, async () => { + const dir = join(work, fmt); + mkdirSync(dir, { recursive: true }); + const files: string[] = []; + for (const c of cases) { + const f = join(dir, `seed-${c.seed}.${fmt}`); + writeFileSync(f, await build(c.doc, ...exportArgs(c.opts))); + files.push(f); + } + convertAll(files, join(dir, 'out')); + for (const c of cases) { + const f = join(dir, 'out', `seed-${c.seed}.odt`); + if (!existsSync(f)) { expect.soft(f, `seed ${c.seed}: LibreOffice wrote no file`).toBe(''); continue; } + const res = importOdt(new Uint8Array(readFileSync(f))); + const marked = markedTables(c.doc); + const [want, got] = [loContent(c.doc, fmt, marked), loContent(res.content, fmt, marked)]; + const skipContent = fmt === 'docx' && hardForDocx(c.doc); + // Loose on numbers: a size or padding comes back a thousandth off through cm. + const diff = skipContent ? null : diffLoose(want, got); + expect.soft(diff, `seed ${c.seed}: content via ${fmt}`).toBeNull(); + const exp = omit(loOptions(expectedOptions(c.opts), fmt, c.opts, true), LOSSY[fmt]); + const imp = omit(loOptions(importedOptions(res, c.opts), fmt, c.opts, false), LOSSY[fmt]); + // A file naming no language gets LibreOffice's own; via DOCX a page-number start + // needs a leading paragraph to ride on. + if (exp.language == null) { delete exp.language; delete imp.language; } + if (fmt === 'docx' && !['paragraph', 'heading'].includes(c.doc.content?.[0]?.type)) { delete exp.pageNumberStart; delete imp.pageNumberStart; } + const optDiff = diffLoose(exp, imp); + expect.soft(optDiff, `seed ${c.seed}: options via ${fmt}`).toBeNull(); + if (DUMP) dump.push(`## seed ${c.seed} ${fmt}: warnings=${JSON.stringify(res.warnings)}`, + ...(skipContent ? [' C (not compared)'] : allDiffs(want, got).map((d) => ` C ${d}`)), ...allDiffs(exp, imp).map((d) => ` O ${d}`)); + } + if (DUMP) writeFileSync(DUMP, dump.join('\n') + '\n'); + }, 900_000); + } +}); diff --git a/tests/monkey/run.mjs b/tests/monkey/run.mjs new file mode 100644 index 0000000..1d0e227 --- /dev/null +++ b/tests/monkey/run.mjs @@ -0,0 +1,222 @@ +// Monkey run: random editing in the real browser under the invariants any sequence of +// commands must keep — no uncaught error, a document its schema accepts, undo back to the +// opened file and redo forward again, and a saved file the schema validates and the +// importer reads back as the editor holds it. A failure names the seed and the last ops. +import { readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { unzipSync } from 'fflate'; +import { JSDOM } from 'jsdom'; +import { ROOT, MOD, checker, devServer, openApp } from '../browser.mjs'; +import { hasXmllint, validateOdt, validateDocx } from '../schemaValidate.ts'; + +// The schema helper strips foreign markup with the browser's DOM parser. +Object.assign(globalThis, (({ DOMParser, XMLSerializer }) => ({ DOMParser, XMLSerializer }))(new JSDOM('').window)); + +const PORT = +(process.env.MONKEY_PORT ?? 4189); +const SEED = +(process.env.MONKEY_SEED ?? 1); +const RUNS = +(process.env.MONKEY_RUNS ?? 3); +const OPS = +(process.env.MONKEY_OPS ?? 60); +const DOCS = process.env.MONKEY_DOC ? [process.env.MONKEY_DOC] : ['02-blocks.odt', '04-table.odt', '08-lists.odt', '12-notes.odt', '15-chapters.odt']; +const PNG = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; +const { check, failures } = checker(); +if (!hasXmllint) console.log('xmllint missing: the saved files are not validated against the schemas'); + +// fuzzDoc.ts's generator, so a seed replays. +function mulberry32(seed) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} +const pick = (r, arr) => arr[Math.floor(r() * arr.length)]; +const int = (r, min, max) => min + Math.floor(r() * (max - min + 1)); + +const WORDS = ['lorem ', 'Ipsum', ' dolor sit ', 'äöü ', '42', '. ', 'A', ' — ', 'x']; +const KEYS = ['Enter', 'Enter', 'Backspace', 'Backspace', 'Delete', 'Tab', 'Shift+Tab', 'ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown', + 'Shift+ArrowLeft', 'Shift+ArrowRight', 'Shift+ArrowDown', 'Home', 'End', `${MOD}+b`, `${MOD}+i`, `${MOD}+u`, `${MOD}+Shift+7`, `${MOD}+Shift+8`, + `${MOD}+Enter`, `${MOD}+z`, `${MOD}+Shift+z`]; +const STYLES = ['Standard', 'Heading 1', 'Heading 2', 'Heading 3', 'Title', 'Quotations']; +// Each entry: a weight and a draw of the op — a key, typed text, or an editor command. +const OPS_TABLE = [ + [24, (r) => ({ type: pick(r, WORDS) + (r() < 0.3 ? pick(r, WORDS) : '') })], + [20, (r) => ({ key: pick(r, KEYS) })], + [10, (r, size) => ({ cmd: 'focus', args: [int(r, 1, Math.max(1, size - 2))] })], + [6, (r, size) => { const a = int(r, 1, Math.max(1, size - 2)), b = int(r, 1, Math.max(1, size - 2)); return { cmd: 'setTextSelection', args: [{ from: Math.min(a, b), to: Math.max(a, b) }] }; }], + [8, (r) => pick(r, [ + { cmd: 'toggleBold' }, { cmd: 'toggleItalic' }, { cmd: 'toggleUnderline' }, { cmd: 'toggleStrike' }, + { cmd: 'toggleSubscript' }, { cmd: 'toggleSuperscript' }, { cmd: 'setColor', args: ['#c00000'] }, + { cmd: 'setFontSize', args: ['14pt'] }, { cmd: 'setFontFamily', args: ['Liberation Sans'] }, { cmd: 'unsetAllMarks' }, + ])], + [12, (r) => pick(r, [ + { cmd: 'setParagraphStyle', args: [pick(r, STYLES)] }, { cmd: 'setTextAlign', args: [pick(r, ['left', 'center', 'right', 'justify'])] }, + { cmd: 'indentMore' }, { cmd: 'indentLess' }, { cmd: 'setLineHeight', args: ['1.5'] }, { cmd: 'setSpaceBefore', args: [6] }, + { cmd: 'toggleBulletList' }, { cmd: 'toggleOrderedList' }, { cmd: 'sinkListItem', args: ['listItem'] }, { cmd: 'liftListItem', args: ['listItem'] }, + { cmd: 'insertPageBreak' }, { cmd: 'setColumns', args: [2] }, { cmd: 'setColumns', args: [1] }, { cmd: 'clearDirectFormatting' }, + ])], + [8, (r) => pick(r, [ + { cmd: 'insertTable', args: [{ rows: int(r, 2, 3), cols: int(r, 2, 3), withHeaderRow: r() < 0.5 }] }, + { cmd: 'addRowAfter' }, { cmd: 'addColumnAfter' }, { cmd: 'deleteRow' }, { cmd: 'deleteColumn' }, + { cmd: 'mergeCells' }, { cmd: 'splitCell' }, { cmd: 'deleteTable' }, { cmd: 'goToNextCell' }, + ])], + [8, (r) => pick(r, [ + { cmd: 'insertNote', args: ['footnote'] }, { cmd: 'insertNote', args: ['endnote'] }, + { cmd: 'insertFormula', args: [{ latex: 'x^{2}+1', display: false }] }, { cmd: 'insertTextBox' }, + { cmd: 'setImage', args: [{ src: PNG, width: 80, height: 60 }] }, { cmd: 'setHardBreak' }, + { cmd: 'addComment', args: [{ author: 'Monkey', text: 'hm' }] }, { cmd: 'insertContent', args: ['—'] }, + ])], + [3, (r) => ({ cmd: pick(r, ['undo', 'redo']) })], +]; +const TOTAL = OPS_TABLE.reduce((n, [w]) => n + w, 0); +const drawOp = (r, size) => { + let x = r() * TOTAL; + for (const [w, draw] of OPS_TABLE) { if ((x -= w) < 0) return draw(r, size); } + return { key: 'ArrowRight' }; +}; + +const server = await devServer(PORT); +const { browser, page, pageErrors } = await openApp(PORT); +const consoleErrors = []; +page.on('console', (m) => { if (m.type() === 'error') consoleErrors.push(m.text()); }); + +const ed = (fn, arg) => page.evaluate(fn, arg); +const focused = async () => { + const err = await ed(() => { try { document.querySelector('.tiptap').editor.commands.focus(); return null; } catch (e) { return String(e); } }); + await page.waitForFunction(() => document.activeElement === document.querySelector('.tiptap'), null, { timeout: 2000 }).catch(() => {}); + return err; +}; +const runCmd = (op) => ed(({ cmd, args }) => { + const editor = document.querySelector('.tiptap').editor; + try { return { ok: editor.commands[cmd](...(args ?? [])) }; } catch (e) { return { threw: String(e?.stack ?? e) }; } +}, op); +// The schema's own check of the document after every op — a command may leave a node +// with content its spec rejects, which nothing else reports. +const docCheck = () => ed(() => { + try { document.querySelector('.tiptap').editor.state.doc.check(); return null; } catch (e) { return String(e); } +}); +const docJson = () => ed(() => JSON.stringify(document.querySelector('.tiptap').editor.getJSON())); +const docSize = () => ed(() => document.querySelector('.tiptap').editor.state.doc.content.size); + +const saveAs = async (ext) => { + await page.click('.ribbon-tab-file'); + const [dl] = await Promise.all([ + page.waitForEvent('download', { timeout: 60_000 }), + page.locator('.ribbon-menu button', { hasText: `(.${ext})` }).first().click(), + ]); + await page.keyboard.press('Escape'); + return new Uint8Array(await readFile(await dl.path())); +}; +// The file the app saved, read back by the app's importer against the editor's document +// (its paragraphs joined across a column or page merged, as the export merges them). +const roundTrip = (bytes, ext) => ed(async ({ b64, ext }) => { + const [{ importOdt }, { importDocx }, { mergeJoinedParagraphsJson }, { normalize, firstDiff, stripFontHoist }] = await Promise.all([ + import('/src/lib/import/odt.ts'), import('/src/lib/import/docx.ts'), import('/src/lib/export/odt.ts'), import('/tests/normalize.ts')]); + const bin = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)); + const res = ext === 'odt' ? importOdt(bin) : importDocx(bin); + const editor = document.querySelector('.tiptap').editor; + const json = editor.getJSON(); + // What the importer suppresses as the style's own: a block attribute equal to what its + // named style resolves to, and bold in a header cell. + const { resolveStyle } = await import('/src/lib/styles/styleSheet.ts'); + const sheet = editor.extensionManager.extensions.find((e) => e.name === 'paragraphStyle')?.options.sheet(); + const styled = (n, header = false) => { + if (n.attrs?.styleName && sheet) { + const own = resolveStyle(sheet, n.attrs.styleName, 'paragraph').para ?? {}; + for (const [k, v] of Object.entries(n.attrs)) if (k !== 'styleName' && own[k] === v) delete n.attrs[k]; + } + if (header && n.type === 'text' && n.marks) n.marks = n.marks.filter((m) => m.type !== 'bold'); + // A formula alone in its paragraph is a display formula to both files. + if (n.type === 'paragraph' && n.content?.length === 1 && n.content[0].type === 'formula') n.content[0].attrs = { ...n.content[0].attrs, display: true }; + for (const c of n.content ?? []) styled(c, header || n.type === 'tableHeader'); + return n; + }; + const held = styled({ ...json, content: mergeJoinedParagraphsJson(json.content ?? []) }); + return { diff: firstDiff(stripFontHoist(normalize(held)), stripFontHoist(normalize(styled(res.content)))), warnings: res.warnings, held: JSON.stringify(held), read: JSON.stringify(res.content) }; +}, { b64: Buffer.from(bytes).toString('base64'), ext }); + +try { + await page.goto(`http://localhost:${PORT}/`, { waitUntil: 'load' }); + await page.evaluate(() => { localStorage.clear(); window.showSaveFilePicker = undefined; }); + await page.reload({ waitUntil: 'load' }); + await page.waitForSelector('.tiptap', { timeout: 30_000 }); + await page.evaluate(() => { window.showSaveFilePicker = undefined; }); + + for (let run = 0; run < RUNS; run++) { + const seed = SEED + run; + const doc = DOCS[run % DOCS.length]; + const r = mulberry32(seed); + const before = await docJson(); + await page.setInputFiles('input.file-input', join(ROOT, 'tests/corpus', doc)); + await page.waitForFunction((b) => JSON.stringify(document.querySelector('.tiptap').editor.getJSON()) !== b, before, { timeout: 30_000 }); + await page.waitForTimeout(1500); + const initial = await docJson(); + const log = []; + let broken = null; + for (let i = 0; i < OPS && !broken; i++) { + const op = drawOp(r, await docSize()); + log.push(op); + if (op.type || op.key) { + const err = await focused(); + if (err) broken = `focus threw: ${err}`; + else if (op.type) await page.keyboard.type(op.type); + else await page.keyboard.press(op.key); + } else { + const res = await runCmd(op); + if (res.threw) broken = `${op.cmd} threw: ${res.threw}`; + } + const invalid = await docCheck(); + if (invalid) broken = `schema-invalid document: ${invalid}`; + if (pageErrors.length) broken = `uncaught: ${pageErrors.join(' | ')}`; + } + const final = await docJson(); + await writeFile(join(tmpdir(), `monkey-${seed}-ops.json`), JSON.stringify(log)); + // Undo until the opened document is back, redo as often, and the edited one is back. + const undone = broken ? -1 : await ed((initial) => { + const editor = document.querySelector('.tiptap').editor; + for (let n = 0; n < 500; n++) { + if (JSON.stringify(editor.getJSON()) === initial) return n; + if (!editor.can().undo()) return -1; + try { editor.commands.undo(); } catch (e) { return `undo threw: ${String(e).slice(0, 300)}`; } + } + return -1; + }, initial); + const redone = typeof undone !== 'number' || undone < 0 ? null : await ed((n) => { + const editor = document.querySelector('.tiptap').editor; + for (let i = 0; i < n; i++) editor.commands.redo(); + return JSON.stringify(editor.getJSON()); + }, undone); + const history = broken ?? (typeof undone === 'string' ? undone : undone < 0 ? 'undo does not reach the opened document' : redone !== final ? 'redo does not restore the edited document' : null); + const label = `seed ${seed} on ${doc}: ${log.length} ops, ${undone} undo steps`; + check(!history, history ? `${label} — ${history}` : label); + if (history) for (const op of log.slice(-12)) console.log(' ', JSON.stringify(op)); + if (broken) break; + + for (const ext of ['odt', 'docx']) { + const bytes = await saveAs(ext); + const schema = hasXmllint ? (ext === 'odt' ? validateOdt : validateDocx)(unzipSync(bytes)) : []; + const rt = await roundTrip(bytes, ext).catch((err) => ({ diff: null, warnings: [], threw: String(err.message ?? err).split('\n')[0] })); + const bad = rt.threw ? `import threw: ${rt.threw}` : schema.length ? `schema: ${schema[0].slice(0, 300)}` + : rt.diff ? `round trip: ${rt.diff}` : rt.warnings.length ? `warnings: ${rt.warnings.join(' | ')}` : null; + check(!bad, `seed ${seed}: the saved .${ext} validates and reads back${bad ? ` — ${bad}` : ''}`); + if (bad) { + // The file and both documents, for the repro. + const stem = join(tmpdir(), `monkey-${seed}`); + await writeFile(`${stem}.${ext}`, bytes); + if (rt.held) await Promise.all([writeFile(`${stem}-${ext}-held.json`, rt.held), writeFile(`${stem}-${ext}-read.json`, rt.read)]); + console.log(` kept: ${stem}.${ext}`); + } + } + } +} catch (err) { + check(false, `monkey run threw: ${err.stack ?? err}`); +} finally { + if (consoleErrors.length) console.log(`console errors (${consoleErrors.length}): ${[...new Set(consoleErrors)].slice(0, 5).join(' | ')}`); + check(pageErrors.length === 0, pageErrors.length ? `no uncaught page errors — got: ${pageErrors.join(' | ')}` : 'no uncaught page errors'); + await browser.close(); + if (server) process.kill(-server.pid); +} +process.exit(failures.length ? 1 : 0); diff --git a/tests/normalize.ts b/tests/normalize.ts index de8638b..97f44f2 100644 --- a/tests/normalize.ts +++ b/tests/normalize.ts @@ -10,6 +10,9 @@ const ORDERED_DEFAULTS: Record = { type: null, level: undefined, rotation: 0, wrap: 'inline', shapeKind: 'textbox', fillColor: '#FFFFFF', strokeColor: '#000000', strokeWidthPt: 1, fixed: false, key1: '', + joinPrev: false, // pageBreaks.ts's split marker; the export merges the halves + marginLeft: 0, marginRight: 0, marginTop: 0, marginBottom: 0, // a table's, once a command has touched them + alt: '', inFront: false, paddingCm: 0.15, flipV: false, textVertical: false, // the editor's own picture and box defaults // index attrs the DOCX TOC field has no switch for come back at their defaults maxLevel: MAX_HEADING_LEVEL, leader: '.', citationStyle: 'key', }; @@ -56,12 +59,14 @@ export function normalize(node: N): N { if (node.type === 'doc') canonNoteIds(node); const out: N = { type: node.type }; if (node.text != null) out.text = node.text; - if (node.marks?.length) { + // A formula adopts the caret's marks for its font; neither file carries them. + if (node.marks?.length && node.type !== 'formula') { out.marks = node.marks .map((m: N) => { const mm: N = { type: m.type }; const attrs = Object.fromEntries(Object.entries(m.attrs ?? {}) - .filter(([k, v]) => v != null && MARK_DEFAULTS[k] !== v)); + .filter(([k, v]) => v != null && MARK_DEFAULTS[k] !== v) + .map(([k, v]) => [k, typeof v === 'string' && /^#[0-9a-f]{6}$/i.test(v) ? v.toUpperCase() : v])); // ODF keeps the authored (naive local) comment/revision date, DOCX re-serializes // the same instant as UTC — compare the instant. if ((m.type === 'comment' || m.type === 'insertion' || m.type === 'deletion') diff --git a/tests/render-parity/run.mjs b/tests/render-parity/run.mjs index 1143dc5..b4c45c0 100644 --- a/tests/render-parity/run.mjs +++ b/tests/render-parity/run.mjs @@ -2,7 +2,6 @@ // compares the resulting text layout (pages, lines, positions in mm). // Usage and prerequisites — including the mandatory font setup — are in README.md. import { execFileSync } from 'node:child_process'; -import { spawn } from 'node:child_process'; import { readFileSync, writeFileSync, copyFileSync, mkdirSync, mkdtempSync, readdirSync, rmSync, existsSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; @@ -10,6 +9,7 @@ import { join, basename, extname, resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { chromium } from 'playwright-core'; import { compare, toLines } from './compare.mjs'; +import { devServer, settle, extractLayout } from '../browser.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(HERE, '../..'); @@ -91,135 +91,8 @@ async function editorRender(browser, file) { return result; } -// Pagination (and the columns flow) runs over several frames; wait for the page -// count and document height to stop moving. An image still decoding has no height -// yet and would settle the layout at a page count it leaves again a frame later. -async function settle(page) { - await page.waitForFunction(() => { - const el = document.querySelector('.tiptap'); - // Importing a large file takes seconds; an editor still empty is not "settled", - // it is the blank document the file has not replaced yet. - if (!el || !el.textContent.trim()) return false; - const imgs = Array.from(document.querySelectorAll('.paper img')); - if (imgs.some((i) => !i.complete)) return false; - // The spacers' heights too, not just their count: they go on settling for a while - // after the last one is placed, and the page a line lands on moves with them. - const spacers = Array.from(document.querySelectorAll('[data-page-break-spacer]')); - // The index's page numbers settle after the spacers do — a stale one is a line of - // text that still changes. - const toc = Array.from(document.querySelectorAll('.toc-page')).map((t) => t.textContent).join(','); - const key = el.style.minHeight + '|' + el.children.length + '|' + spacers.length + '|' - + spacers.reduce((sum, s) => sum + s.offsetHeight, 0) + '|' + toc; - const w = window; - if (w.__parityKey !== key) { w.__parityKey = key; w.__paritySince = performance.now(); return false; } - return performance.now() - (w.__paritySince ?? 0) > 3000; - // Polled, not per frame: the predicate walks every spacer and index row, and on a - // 60-page document doing that each frame starves the layout it is waiting for. - }, null, { timeout: 180_000, polling: 500 }); -} - -// Runs in the browser: every rendered word with its page and mm position. -function extractLayout() { - const PAGE_GAP = 20, PX_MM = 25.4 / 96; - const paper = document.querySelector('.paper'); - const cs = getComputedStyle(document.documentElement); - const pageH = parseFloat(cs.getPropertyValue('--user-page-height')); - const pageW = parseFloat(cs.getPropertyValue('--user-page-width')); - const cycle = pageH + PAGE_GAP; - const origin = paper.getBoundingClientRect(); - - const skip = (node) => { - for (let e = node.parentElement; e && e !== paper; e = e.parentElement) { - if (e.hasAttribute('data-page-break-spacer') || e.classList.contains('band-layer')) return true; - const s = getComputedStyle(e); - if (s.display === 'none' || s.visibility === 'hidden') return true; - } - return false; - }; - - // A word broken mid-word — a token wider than its line, or one carrying a hyphen or a - // slash — has a client rect per line, and pdftotext reads those as separate words. So - // split it the same way, by the line each character lands on. - const fragments = (node, from, to) => { - const r = document.createRange(); - r.setStart(node, from); r.setEnd(node, to); - const rects = r.getClientRects(); - const whole = (rect) => ({ top: rect.top, left: rect.left, width: rect.width, height: rect.height, text: node.nodeValue.slice(from, to) }); - if (rects.length < 2) return rects.length ? [whole(rects[0])] : []; - const out = []; - for (let i = from; i < to; i++) { - const c = document.createRange(); - c.setStart(node, i); c.setEnd(node, i + 1); - const rect = c.getClientRects()[0]; - if (!rect) continue; - const last = out[out.length - 1]; - if (last && Math.abs(last.top - rect.top) < 1) { last.text += node.nodeValue[i]; last.right = rect.right; } - else out.push({ top: rect.top, left: rect.left, right: rect.right, height: rect.height, text: node.nodeValue[i] }); - } - return out.map((f) => ({ top: f.top, left: f.left, width: f.right - f.left, height: f.height, text: f.text })); - }; - - // What the page paints, not what the node holds: a style's fo:text-transform / - // w:caps is a CSS transform here and real uppercase in the reference's PDF. - const painted = (text, el) => { - const t = getComputedStyle(el).textTransform; - if (t === 'uppercase') return text.toUpperCase(); - if (t === 'lowercase') return text.toLowerCase(); - if (t === 'capitalize') return text.replace(/^\p{L}/u, (c) => c.toUpperCase()); - return text; - }; - - const words = []; - const walker = document.createTreeWalker(paper, NodeFilter.SHOW_TEXT); - for (let n = walker.nextNode(); n; n = walker.nextNode()) { - const text = n.nodeValue; - if (!text || !text.trim() || skip(n)) continue; - for (const m of text.matchAll(/\S+/g)) { - for (const f of fragments(n, m.index, m.index + m[0].length)) { - if (!f.width) continue; - const y = f.top - origin.top; - const page = Math.floor(y / cycle); - words.push({ - text: painted(f.text, n.parentElement), - page, - x: (f.left - origin.left) * PX_MM, - y: (y - page * cycle) * PX_MM, - w: f.width * PX_MM, - h: f.height * PX_MM, - }); - } - } - } - // Folded, not spread: a several-hundred-page document has more words than a call - // takes arguments, and Math.max(...words) then blows the stack instead of measuring. - const numPages = words.reduce((m, w) => (w.page > m ? w.page : m), 0) + 1; - const pages = Array.from({ length: numPages }, (_, i) => ({ - words: words.filter((w) => w.page === i).map(({ page, ...r }) => r), - width: pageW * PX_MM, - height: pageH * PX_MM, - })); - const mm = (v) => parseFloat(cs.getPropertyValue(v)) * PX_MM; - return { - pages, - margins: { top: mm('--user-margin-top'), bottom: mm('--user-margin-bottom'), - left: mm('--user-margin-left'), right: mm('--user-margin-right') }, - }; -} - // ------------------------------------------------------------------- driver -async function devServer() { - const up = await fetch(`http://localhost:${PORT}/`).then(() => true).catch(() => false); - if (up) return null; - const proc = spawn('npm', ['run', 'dev', '--', '--port', String(PORT), '--strictPort'], - { cwd: ROOT, stdio: 'ignore', detached: true }); - for (let i = 0; i < 60; i++) { - await new Promise((r) => setTimeout(r, 500)); - if (await fetch(`http://localhost:${PORT}/`).then(() => true).catch(() => false)) return proc; - } - throw new Error('dev server did not start'); -} - function corpus(args) { const paths = args.length ? args : [join(HERE, '..', 'corpus'), join(HERE, 'fixtures')]; const out = []; @@ -256,7 +129,7 @@ if (args.includes('--quick')) { } if (!files.length) { console.error('no .docx/.odt files found'); process.exit(2); } -const server = await devServer(); +const server = await devServer(PORT); const browser = await chromium.launch({ executablePath: CHROME, args: ['--no-sandbox'] }); const work = mkdtempSync(join(tmpdir(), 'parity-')); const report = []; diff --git a/tests/schemaValidate.ts b/tests/schemaValidate.ts index f7ee9bc..f21725f 100644 --- a/tests/schemaValidate.ts +++ b/tests/schemaValidate.ts @@ -5,10 +5,11 @@ import { spawnSync } from 'node:child_process'; import { mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { strFromU8 } from 'fflate'; export const hasXmllint = spawnSync('xmllint', ['--version']).status === 0; -const SCHEMAS = resolve(__dirname, 'schemas'); +const SCHEMAS = resolve(fileURLToPath(import.meta.url), '..', 'schemas'); const MC_NS = 'http://schemas.openxmlformats.org/markup-compatibility/2006'; // A DrawingML text box (wps:wsp inside a:graphicData) is post-ECMA-376 Microsoft // markup Word accepts natively; no ECMA schema covers it, so it is stripped like an