test: LibreOffice, the layout lint and a monkey read the editor's work back
lo-fuzz.test.ts hands every fuzz seed to LibreOffice as .odt and .docx and compares what comes back, listing the differences LibreOffice makes by design. test:layout opens the corpus, the showcase and fuzz seeds in the browser, holds the page count against LibreOffice and checks the lines for overlap, margin escapes and stranded headings. test:monkey types at random on corpus documents and asserts the schema, undo to the start, redo, and that both saved formats read back. browser.mjs now carries the dev server and the settle helper.
This commit is contained in:
Steffen Becker committed
1 parent
3b6cb06e6f
commit
d156482a3f
14 files changed
+874
-177
No files matched your search
@@ -29,10 +29,11 @@ jobs:
|
||||
# (the lo-roundtrip job below runs that leg).
|
||||
- run: npm test
|
||||
|
||||
# The re-save-through-LibreOffice leg, which self-skips everywhere soffice is absent.
|
||||
# The LibreOffice legs (self-skipping everywhere soffice is absent) and the layout run,
|
||||
# which holds the editor's page counts against LibreOffice's PDFs in Chromium.
|
||||
lo-roundtrip:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
@@ -43,6 +44,8 @@ jobs:
|
||||
- run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libreoffice-writer fonts-liberation poppler-utils
|
||||
- run: npm ci
|
||||
- run: npm run test:lo
|
||||
- run: npx playwright-core install --with-deps chromium
|
||||
- run: npm run test:layout
|
||||
|
||||
# Production build booted in the three engines behind the supported browsers
|
||||
# (vite build.target). WebKit runs on macOS: its text shaping there is Safari's.
|
||||
@@ -69,3 +72,7 @@ jobs:
|
||||
- run: npm run build
|
||||
- run: npm run test:smoke
|
||||
- run: npm run test:dom
|
||||
# xmllint validates the files the monkey saves (macOS ships it).
|
||||
- if: runner.os == 'Linux'
|
||||
run: sudo apt-get install -y --no-install-recommends libxml2-utils
|
||||
- run: npm run test:monkey
|
||||
@@ -14,26 +14,24 @@ npm run build # production build → dist/
|
||||
npm run preview # serve the dist/ build locally
|
||||
npm run check # svelte-check type-check (svelte + ts)
|
||||
npm test # Vitest suite once (tests/**/*.test.ts)
|
||||
npm run test:watch # Vitest in watch mode
|
||||
npm run test:lo # LibreOffice legs: round trip + ODT/DOCX render consistency (needs `soffice`)
|
||||
npm run test:lo # LibreOffice legs: round trip, fuzz re-read, ODT/DOCX render consistency (needs `soffice`)
|
||||
npm run test:smoke # boots the dist/ build headless (tests/smoke/run.mjs); BROWSER=firefox|webkit
|
||||
npm run test:dom # pagination + editing in the real browser (tests/dom/run.mjs); same BROWSER
|
||||
npm run test:layout # page counts vs LibreOffice + layout invariants over corpus, showcase, fuzz seeds (tests/layout/)
|
||||
npm run test:monkey # random editing under invariants: schema, undo/redo, the saved file reads back (tests/monkey/)
|
||||
npm run test:coverage # vitest + v8 coverage over src/ → coverage/index.html
|
||||
npm run test:parity # render parity vs LibreOffice (tests/render-parity/README.md)
|
||||
node scripts/make-thesaurus.mjs # re-vendor public/thesaurus/ from LibreOffice's MyThes data
|
||||
node scripts/collect-licenses.mjs # regenerate public/licenses.txt after a dependency change
|
||||
node scripts/make-thesaurus.mjs; node scripts/collect-licenses.mjs # re-vendor public/thesaurus/ (MyThes); regenerate public/licenses.txt
|
||||
node scripts/showcase/run.mjs # rebuild docs/showcase/ (sample .odt/.docx + README screenshots); [regex] limits it
|
||||
```
|
||||
|
||||
Tests live in `tests/` (outside `src/`, so `svelte-check` ignores them), jsdom via Vitest.
|
||||
`roundtrip.test.ts` covers the ODF export↔import round trip + a foreign-doc/style-resolver leg;
|
||||
`lo-roundtrip.test.ts` re-saves through LibreOffice and **self-skips** without `soffice`, so
|
||||
`npm test`/CI stay green; `corpus.test.ts` round-trips the committed `tests/corpus/`
|
||||
documents (by `make-fixtures.mjs` + Word re-saves in `corpus/word/`, never our exporter);
|
||||
`lo-roundtrip.test.ts` re-saves through LibreOffice and `lo-fuzz.test.ts` reads the fuzz seeds back through it (`LO_SEEDS`, `LO_DUMP=<file>` for triage) — both **self-skip** without `soffice`, so `npm test`/CI stay green;
|
||||
`corpus.test.ts` round-trips the committed `tests/corpus/` documents (by `make-fixtures.mjs` + Word re-saves in `corpus/word/`, never our exporter);
|
||||
`fuzz-roundtrip.test.ts` round-trips seeded random documents under random export options (`fuzzDoc.ts`, `fuzzOptions.ts`) through both formats and validates every export against the schemas (`schemaValidate.ts`; `FUZZ_SEEDS=500` widens it);
|
||||
`schema-validation.test.ts` validates the `kitchenSink.ts` exports against the vendored
|
||||
schemas in `tests/schemas/` (self-skips without `xmllint`); `package-lint.test.ts` checks the semantic invariants schemas can't express (unique ids, dangling references); `tests/unit/` holds fast helper tests; the two browser runs (`tests/smoke/`, `tests/dom/`) share `tests/browser.mjs`. All test tooling stays a
|
||||
`devDependency`. No linter/formatter. CI (`.github/workflows/ci.yml`) runs `check` + `test`.
|
||||
`schema-validation.test.ts` validates the `kitchenSink.ts` exports against the vendored schemas in `tests/schemas/` (self-skips without `xmllint`); `package-lint.test.ts` checks the semantic invariants schemas can't express (unique ids, dangling references); `tests/unit/` holds fast helper tests;
|
||||
the four browser runs (`tests/smoke/`, `tests/dom/`, `tests/layout/`, `tests/monkey/`) share `tests/browser.mjs`. All test tooling stays a `devDependency`. No linter/formatter. CI (`.github/workflows/ci.yml`) runs `check` + `test`.
|
||||
|
||||
## Rules
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ Then load `npm run dev`, inject a document into `localStorage['edentext-doc']`,
|
||||
|
||||
For PDF-export repros specifically: replicate `pdf.ts`'s clone + `html2canvas(...)` inside `page.evaluate` and read the canvas back as a PNG — capturing the real jsPDF `doc.save()` download tends to hang in headless. Inspect output PDFs with poppler-utils (`apt-get install -y poppler-utils`: `pdftoppm`, `pdfimages`, `pdftotext`).
|
||||
|
||||
## The two committed browser runs
|
||||
## The four committed browser runs
|
||||
|
||||
`npm run test:smoke` and `npm run test:dom` are these probes made permanent: both boot
|
||||
`dist/` headless through `tests/browser.mjs` (preview server, checklist, `pageerror`
|
||||
@@ -17,6 +17,22 @@ collector) and fail on any uncaught page error. The DOM run keeps the pagination
|
||||
invariants of pass 4 below — one page count through a settle, a reload, a zoom, a page
|
||||
break and its undo — which is the only test `src/lib/components/**` has.
|
||||
|
||||
`npm run test:layout` and `npm run test:monkey` run against the dev server (Vite serves
|
||||
TypeScript, so a page imports `src/` and `tests/` modules directly). The **layout run**
|
||||
opens every corpus and showcase document plus `LAYOUT_SEEDS` fuzz seeds, holds the editor's
|
||||
page count against LibreOffice's PDF (`soffice` + `pdfinfo`, a page or a tenth of slack)
|
||||
and checks the rendered lines: none overlap, none sit in the page gap or past the sheet,
|
||||
none cross a single-section document's margins, no heading ends a page or column, nothing
|
||||
is wider than the page. A tab's box, a frame, a formula and the decor are not lines.
|
||||
The **monkey run** replays `MONKEY_OPS` random keys and commands per seed
|
||||
(`MONKEY_SEED`, `MONKEY_RUNS`, `MONKEY_DOC`) on a corpus document and checks after each:
|
||||
no uncaught error, a document its own schema accepts; at the end undo back to the opened
|
||||
file and redo forward, then Save As both formats, xmllint each against the schemas and read
|
||||
it back through the app's importer. A failing seed prints its last ops and keeps the file
|
||||
and both documents under the OS temp dir. Found so far: a note inserted in a text box or a
|
||||
text box in a note breaks the ODT's XML, and a table in a cell is dropped on save — both
|
||||
commands now refuse there.
|
||||
|
||||
`BROWSER=chromium|firefox|webkit` picks the engine (Chromium by default; the others via
|
||||
`npx playwright-core install firefox webkit`). CI runs both on all three, one per matrix
|
||||
leg, WebKit on macOS because only there its text shaping is Safari's. The engines are
|
||||
|
||||
+3
-1
@@ -17,9 +17,11 @@
|
||||
"check": "svelte-check --tsconfig ./tsconfig.json",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:lo": "vitest run tests/lo-roundtrip.test.ts tests/lo-render-consistency.test.ts tests/lo-password.test.ts",
|
||||
"test:lo": "vitest run tests/lo-roundtrip.test.ts tests/lo-fuzz.test.ts tests/lo-render-consistency.test.ts tests/lo-password.test.ts",
|
||||
"test:smoke": "node tests/smoke/run.mjs",
|
||||
"test:dom": "node tests/dom/run.mjs",
|
||||
"test:layout": "node tests/layout/run.mjs",
|
||||
"test:monkey": "node tests/monkey/run.mjs",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:parity": "node tests/render-parity/run.mjs",
|
||||
"test:parity:fixtures": "node tests/render-parity/make-fixtures.mjs"
|
||||
|
||||
@@ -2,27 +2,17 @@
|
||||
// modules are TypeScript the browser imports straight from the dev server, and the app's
|
||||
// own exporters write the .odt/.docx; each is then opened through the file input and shot.
|
||||
// `node scripts/showcase/run.mjs [regex]` limits the run to matching document names.
|
||||
import { spawn } from 'node:child_process';
|
||||
import { writeFile, mkdir } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { ROOT, openApp } from '../../tests/browser.mjs';
|
||||
import { ROOT, openApp, devServer } from '../../tests/browser.mjs';
|
||||
|
||||
const PORT = 4187;
|
||||
const OUT = join(ROOT, 'docs/showcase');
|
||||
const only = process.argv[2] ? new RegExp(process.argv[2]) : null;
|
||||
const DOCS = ['thesis', 'thesis-review', 'book', 'newsletter'].filter((n) => !only || only.test(n));
|
||||
|
||||
async function devServer() {
|
||||
const up = () => fetch(`http://localhost:${PORT}/`).then(() => true).catch(() => false);
|
||||
if (await up()) return null;
|
||||
const proc = spawn('npx', ['vite', '--port', String(PORT), '--strictPort'],
|
||||
{ cwd: ROOT, stdio: 'ignore', detached: true });
|
||||
for (let i = 0; i < 60 && !(await up()); i++) await new Promise((r) => setTimeout(r, 500));
|
||||
return proc;
|
||||
}
|
||||
|
||||
await mkdir(OUT, { recursive: true });
|
||||
const server = await devServer();
|
||||
const server = await devServer(PORT);
|
||||
const { browser, page, pageErrors } = await openApp(PORT);
|
||||
const url = `http://localhost:${PORT}/`;
|
||||
|
||||
|
||||
+131
-2
@@ -1,5 +1,6 @@
|
||||
// What the browser runs (smoke, dom) share: the dist/ preview server, the checklist
|
||||
// both print, and a page wired to fail the run on an uncaught error.
|
||||
// What the browser runs share: the preview and dev servers, the checklist they print, a
|
||||
// page wired to fail the run on an uncaught error, and the layout readers (settle,
|
||||
// extractLayout) the parity, layout and monkey runs measure with.
|
||||
import { spawn, execSync } from 'node:child_process';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join, resolve, dirname } from 'node:path';
|
||||
@@ -34,6 +35,18 @@ export async function previewServer(port) {
|
||||
throw new Error('preview server did not start');
|
||||
}
|
||||
|
||||
// The dev server: Vite serves TypeScript as is, so a page can import src/ and tests/
|
||||
// modules straight from it. Reused when one is already up on the port.
|
||||
export async function devServer(port) {
|
||||
const up = () => fetch(`http://localhost:${port}/`).then(() => true).catch(() => false);
|
||||
if (await up()) return null;
|
||||
const proc = spawn('npx', ['vite', '--port', String(port), '--strictPort'],
|
||||
{ cwd: ROOT, stdio: 'ignore', detached: true });
|
||||
for (let i = 0; i < 120 && !(await up()); i++) await new Promise((r) => setTimeout(r, 500));
|
||||
if (!(await up())) throw new Error('dev server did not start');
|
||||
return proc;
|
||||
}
|
||||
|
||||
// BROWSER=chromium|firefox|webkit picks the engine (Chromium by default). Fixed locale,
|
||||
// so the UI labels a test clicks are deterministic across machines.
|
||||
export async function openApp(port) {
|
||||
@@ -49,3 +62,119 @@ export async function openApp(port) {
|
||||
page.on('dialog', (d) => d.accept());
|
||||
return { browser, page, pageErrors };
|
||||
}
|
||||
|
||||
// Pagination (and the columns flow) runs over several frames; wait for the page
|
||||
// count and document height to stop moving. An image still decoding has no height
|
||||
// yet and would settle the layout at a page count it leaves again a frame later.
|
||||
// `loaded`: the caller knows the file is in (a document may be empty for good).
|
||||
export async function settle(page, loaded = false) {
|
||||
await page.waitForFunction((loaded) => {
|
||||
const el = document.querySelector('.tiptap');
|
||||
// Importing a large file takes seconds; an editor still empty is not "settled",
|
||||
// it is the blank document the file has not replaced yet.
|
||||
if (!el || (!loaded && !(el.textContent.trim() || el.querySelector('img, table, .textbox-node, .formula, .toc')))) return false;
|
||||
const imgs = Array.from(document.querySelectorAll('.paper img'));
|
||||
if (imgs.some((i) => !i.complete)) return false;
|
||||
// The spacers' heights too, not just their count: they go on settling for a while
|
||||
// after the last one is placed, and the page a line lands on moves with them.
|
||||
const spacers = Array.from(document.querySelectorAll('[data-page-break-spacer]'));
|
||||
// The index's page numbers settle after the spacers do — a stale one is a line of
|
||||
// text that still changes.
|
||||
const toc = Array.from(document.querySelectorAll('.toc-page')).map((t) => t.textContent).join(',');
|
||||
const key = el.style.minHeight + '|' + el.children.length + '|' + spacers.length + '|'
|
||||
+ spacers.reduce((sum, s) => sum + s.offsetHeight, 0) + '|' + toc;
|
||||
const w = window;
|
||||
if (w.__parityKey !== key) { w.__parityKey = key; w.__paritySince = performance.now(); return false; }
|
||||
return performance.now() - (w.__paritySince ?? 0) > 3000;
|
||||
// Polled, not per frame: the predicate walks every spacer and index row, and on a
|
||||
// 60-page document doing that each frame starves the layout it is waiting for.
|
||||
}, loaded, { timeout: 180_000, polling: 500 });
|
||||
}
|
||||
|
||||
// Runs in the browser: every rendered word with its page and mm position.
|
||||
export function extractLayout() {
|
||||
const PAGE_GAP = 20, PX_MM = 25.4 / 96;
|
||||
const paper = document.querySelector('.paper');
|
||||
const cs = getComputedStyle(document.documentElement);
|
||||
const pageH = parseFloat(cs.getPropertyValue('--user-page-height'));
|
||||
const pageW = parseFloat(cs.getPropertyValue('--user-page-width'));
|
||||
const cycle = pageH + PAGE_GAP;
|
||||
const origin = paper.getBoundingClientRect();
|
||||
|
||||
const skip = (node) => {
|
||||
for (let e = node.parentElement; e && e !== paper; e = e.parentElement) {
|
||||
if (e.hasAttribute('data-page-break-spacer') || e.classList.contains('band-layer')) return true;
|
||||
const s = getComputedStyle(e);
|
||||
if (s.display === 'none' || s.visibility === 'hidden') return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// A word broken mid-word — a token wider than its line, or one carrying a hyphen or a
|
||||
// slash — has a client rect per line, and pdftotext reads those as separate words. So
|
||||
// split it the same way, by the line each character lands on.
|
||||
const fragments = (node, from, to) => {
|
||||
const r = document.createRange();
|
||||
r.setStart(node, from); r.setEnd(node, to);
|
||||
const rects = r.getClientRects();
|
||||
const whole = (rect) => ({ top: rect.top, left: rect.left, width: rect.width, height: rect.height, text: node.nodeValue.slice(from, to) });
|
||||
if (rects.length < 2) return rects.length ? [whole(rects[0])] : [];
|
||||
const out = [];
|
||||
for (let i = from; i < to; i++) {
|
||||
const c = document.createRange();
|
||||
c.setStart(node, i); c.setEnd(node, i + 1);
|
||||
const rect = c.getClientRects()[0];
|
||||
if (!rect) continue;
|
||||
const last = out[out.length - 1];
|
||||
if (last && Math.abs(last.top - rect.top) < 1) { last.text += node.nodeValue[i]; last.right = rect.right; }
|
||||
else out.push({ top: rect.top, left: rect.left, right: rect.right, height: rect.height, text: node.nodeValue[i] });
|
||||
}
|
||||
return out.map((f) => ({ top: f.top, left: f.left, width: f.right - f.left, height: f.height, text: f.text }));
|
||||
};
|
||||
|
||||
// What the page paints, not what the node holds: a style's fo:text-transform /
|
||||
// w:caps is a CSS transform here and real uppercase in the reference's PDF.
|
||||
const painted = (text, el) => {
|
||||
const t = getComputedStyle(el).textTransform;
|
||||
if (t === 'uppercase') return text.toUpperCase();
|
||||
if (t === 'lowercase') return text.toLowerCase();
|
||||
if (t === 'capitalize') return text.replace(/^\p{L}/u, (c) => c.toUpperCase());
|
||||
return text;
|
||||
};
|
||||
|
||||
const words = [];
|
||||
const walker = document.createTreeWalker(paper, NodeFilter.SHOW_TEXT);
|
||||
for (let n = walker.nextNode(); n; n = walker.nextNode()) {
|
||||
const text = n.nodeValue;
|
||||
if (!text || !text.trim() || skip(n)) continue;
|
||||
for (const m of text.matchAll(/\S+/g)) {
|
||||
for (const f of fragments(n, m.index, m.index + m[0].length)) {
|
||||
if (!f.width) continue;
|
||||
const y = f.top - origin.top;
|
||||
const page = Math.floor(y / cycle);
|
||||
words.push({
|
||||
text: painted(f.text, n.parentElement),
|
||||
page,
|
||||
x: (f.left - origin.left) * PX_MM,
|
||||
y: (y - page * cycle) * PX_MM,
|
||||
w: f.width * PX_MM,
|
||||
h: f.height * PX_MM,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Folded, not spread: a several-hundred-page document has more words than a call
|
||||
// takes arguments, and Math.max(...words) then blows the stack instead of measuring.
|
||||
const numPages = words.reduce((m, w) => (w.page > m ? w.page : m), 0) + 1;
|
||||
const pages = Array.from({ length: numPages }, (_, i) => ({
|
||||
words: words.filter((w) => w.page === i).map(({ page, ...r }) => r),
|
||||
width: pageW * PX_MM,
|
||||
height: pageH * PX_MM,
|
||||
}));
|
||||
const mm = (v) => parseFloat(cs.getPropertyValue(v)) * PX_MM;
|
||||
return {
|
||||
pages,
|
||||
margins: { top: mm('--user-margin-top'), bottom: mm('--user-margin-bottom'),
|
||||
left: mm('--user-margin-left'), right: mm('--user-margin-right') },
|
||||
};
|
||||
}
|
||||
@@ -9,19 +9,15 @@ import { buildDocx } from '../src/lib/export/docx';
|
||||
import { importDocx } from '../src/lib/import/docx';
|
||||
import { normalize, firstDiff, stripFontHoist } from './normalize';
|
||||
import { mulberry32 } from './fuzzDoc';
|
||||
import { genCase, exportArgs, expectedOptions, importedOptions, diffLoose, omit } from './fuzzOptions';
|
||||
import { genCase, exportArgs, expectedOptions, importedOptions, diffLoose, omit, DOCX_LOSSY, docWideOddEven } from './fuzzOptions';
|
||||
import { hasXmllint, validateOdt, validateDocx } from './schemaValidate';
|
||||
|
||||
const SEEDS = Number(process.env.FUZZ_SEEDS ?? 50); // FUZZ_SEEDS=500 for a wide sweep
|
||||
// The schema leg pays ~0.7s of xmllint per seed, so it takes fewer by default.
|
||||
const SCHEMA_SEEDS = Number(process.env.SCHEMA_SEEDS ?? Math.min(SEEDS, 20));
|
||||
|
||||
// What each format has no place for (see the importers' CLAUDE.md); dropped from both
|
||||
// sides of its comparison. Word spaces paragraphs by the larger of the two values and
|
||||
// draws its note separator its own way; it has no note prefix/suffix and counts every line.
|
||||
// What ODT has no place for: nothing so far (DOCX_LOSSY lists Word's).
|
||||
const ODT_LOSSY: string[] = [];
|
||||
const DOCX_LOSSY: string[] = ['spacingModel', 'notes.separator', 'notes.footnote.prefix', 'notes.footnote.suffix',
|
||||
'notes.endnote.prefix', 'notes.endnote.suffix', 'lineNumbering.countEmpty'];
|
||||
|
||||
describe('fuzz round-trip: editor → buildOdt → importOdt', () => {
|
||||
it(`${SEEDS} seeded random documents come back identical`, async () => {
|
||||
@@ -47,17 +43,6 @@ describe('fuzz round-trip: editor → buildOdt → importOdt', () => {
|
||||
return node;
|
||||
}
|
||||
|
||||
// Odd/even pages are a document setting in Word (w:evenAndOddHeaders): once any
|
||||
// section asks for it, every section has it, repeating its running zones.
|
||||
function docWideOddEven(canon: any): any {
|
||||
if (!canon.sections.some((s: any) => s.differentOddEven)) return canon;
|
||||
for (const s of canon.sections) {
|
||||
if (s.differentOddEven) continue;
|
||||
Object.assign(s, { differentOddEven: true, headerEven: s.header, footerEven: s.footer });
|
||||
}
|
||||
return canon;
|
||||
}
|
||||
|
||||
// The DOCX leg: its options against what went out, and its content against the ODT
|
||||
// reading — catches one exporter silently losing what the other keeps.
|
||||
it(`${SEEDS} seeded documents survive DOCX and agree with the ODT round-trip`, async () => {
|
||||
|
||||
@@ -311,6 +311,23 @@ export function diffLoose(a: N, b: N, path = '$'): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
// What DOCX has no place for (see the importers' CLAUDE.md), dropped from both sides of a
|
||||
// comparison: Word spaces paragraphs by the larger of the two values, draws its note
|
||||
// separator its own way, has no note prefix/suffix and counts every line.
|
||||
export const DOCX_LOSSY = ['spacingModel', 'notes.separator', 'notes.footnote.prefix', 'notes.footnote.suffix',
|
||||
'notes.endnote.prefix', 'notes.endnote.suffix', 'lineNumbering.countEmpty'];
|
||||
|
||||
// Odd/even pages are a document setting in Word (w:evenAndOddHeaders): once any section
|
||||
// asks for it, every section has it, repeating its running zones.
|
||||
export function docWideOddEven(canon: N): N {
|
||||
if (!canon.sections.some((s: N) => s.differentOddEven)) return canon;
|
||||
for (const s of canon.sections) {
|
||||
if (s.differentOddEven) continue;
|
||||
Object.assign(s, { differentOddEven: true, headerEven: s.header, footerEven: s.footer });
|
||||
}
|
||||
return canon;
|
||||
}
|
||||
|
||||
// Drops the paths a format cannot carry (`a.b`, `sections.*.c`) from a canonical reading.
|
||||
export function omit(canon: N, paths: readonly string[]): N {
|
||||
const out = structuredClone(canon);
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
// Layout run: the documents the repo carries — corpus, showcase and fuzz seeds — opened in
|
||||
// the editor, the page count held against LibreOffice's PDF, and the rendered lines checked
|
||||
// for what no layout may show: overlapping lines, text off the page or past the margins, a
|
||||
// heading alone at the foot of a page or column. `[regex]` limits the run by file name.
|
||||
import { execFileSync, execSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, existsSync, copyFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, basename, extname, relative } from 'node:path';
|
||||
import { ROOT, checker, devServer, openApp, settle } from '../browser.mjs';
|
||||
|
||||
const PORT = +(process.env.LAYOUT_PORT ?? 4188);
|
||||
const SEEDS = +(process.env.LAYOUT_SEEDS ?? 10);
|
||||
const only = process.argv[2] ? new RegExp(process.argv[2]) : null;
|
||||
const has = (tool) => { try { execSync(`command -v ${tool}`, { stdio: 'ignore' }); return true; } catch { return false; } };
|
||||
const LO = has('soffice') && has('pdfinfo');
|
||||
const { check, failures } = checker();
|
||||
|
||||
const files = (dir) => (existsSync(dir)
|
||||
? readdirSync(dir).filter((f) => /\.(odt|docx)$/.test(f) && !f.startsWith('~$')).sort().map((f) => join(dir, f))
|
||||
: []);
|
||||
const docs = [...files(join(ROOT, 'tests/corpus')), ...files(join(ROOT, 'tests/corpus/word')), ...files(join(ROOT, 'docs/showcase'))];
|
||||
const work = mkdtempSync(join(tmpdir(), 'layout-'));
|
||||
const server = await devServer(PORT);
|
||||
const { browser, page, pageErrors } = await openApp(PORT);
|
||||
if (!LO) console.log('soffice/pdfinfo missing: page counts are not held against LibreOffice');
|
||||
|
||||
// LibreOffice's page count from its PDF, the blank pages it inserts itself kept as the
|
||||
// editor keeps them; one soffice call for the whole batch, under unique names.
|
||||
function loPages(paths) {
|
||||
const inDir = join(work, 'in'), out = join(work, 'pdf');
|
||||
mkdirSync(inDir, { recursive: true }); mkdirSync(out, { recursive: true });
|
||||
const named = paths.map((p, i) => { const f = join(inDir, `${i}-${basename(p)}`); copyFileSync(p, f); return f; });
|
||||
execFileSync('soffice', ['--headless', '--norestore', `-env:UserInstallation=file://${work}/profile`, '--convert-to',
|
||||
'pdf:writer_pdf_Export:{"IsSkipEmptyPages":{"type":"boolean","value":"false"}}', '--outdir', out, ...named],
|
||||
{ stdio: 'pipe', timeout: 900_000 });
|
||||
const pages = new Map();
|
||||
named.forEach((f, i) => {
|
||||
const pdf = join(out, basename(f, extname(f)) + '.pdf');
|
||||
const m = existsSync(pdf) && /Pages:\s+(\d+)/.exec(execFileSync('pdfinfo', [pdf], { encoding: 'utf8' }));
|
||||
if (m) pages.set(paths[i], +m[1]);
|
||||
});
|
||||
return pages;
|
||||
}
|
||||
|
||||
// Runs in the browser: the rendered lines against the page sheets and the document's
|
||||
// margins (single-section documents only: a later section may have margins of its own).
|
||||
function lintLayout() {
|
||||
const paper = document.querySelector('.paper');
|
||||
const origin = paper.getBoundingClientRect();
|
||||
const rel = (r) => ({ top: r.top - origin.top, bottom: r.bottom - origin.top, left: r.left - origin.left, right: r.right - origin.left });
|
||||
const sheets = [...document.querySelectorAll('.page-sheet')].map((s) => rel(s.getBoundingClientRect()));
|
||||
const pageOf = (y) => sheets.findIndex((s) => y >= s.top - 1 && y <= s.bottom + 1);
|
||||
const cs = getComputedStyle(document.documentElement);
|
||||
const px = (v) => parseFloat(cs.getPropertyValue(v));
|
||||
const margins = { top: px('--user-margin-top'), bottom: px('--user-margin-bottom'), left: px('--user-margin-left'), right: px('--user-margin-right') };
|
||||
const oneSection = !cs.getPropertyValue('--pb-section-page').includes(',');
|
||||
const mirrored = /[1-9]/.test(cs.getPropertyValue('--pb-section-mirror'));
|
||||
const side = mirrored ? Math.min(margins.left, margins.right) : null;
|
||||
|
||||
// Where a text node is painted: a frame may sit on the text by design, a formula
|
||||
// stacks its own boxes, and a hidden or spacer node is not painted at all.
|
||||
const originOf = (node) => {
|
||||
for (let e = node.parentElement; e && e !== paper; e = e.parentElement) {
|
||||
if (e.hasAttribute('data-page-break-spacer') || e.classList.contains('band-layer')) return null;
|
||||
if (e.classList.contains('page-decor-layer') || e.classList.contains('line-number-layer')) return null;
|
||||
if (e.classList.contains('hf-layer')) return 'hf';
|
||||
if (e.classList.contains('textbox-node') || e.classList.contains('image-node')) return 'frame';
|
||||
if (e.classList.contains('formula')) return 'formula';
|
||||
if (e.classList.contains('note')) return 'note';
|
||||
const s = getComputedStyle(e);
|
||||
if (s.display === 'none' || s.visibility === 'hidden') return null;
|
||||
}
|
||||
return 'body';
|
||||
};
|
||||
const lines = [];
|
||||
const walker = document.createTreeWalker(paper, NodeFilter.SHOW_TEXT);
|
||||
for (let n = walker.nextNode(); n; n = walker.nextNode()) {
|
||||
if (!n.nodeValue.trim()) continue;
|
||||
const kind = originOf(n);
|
||||
if (!kind) continue;
|
||||
// Between the tabs: a tab's own box runs to its stop, past a cell's edge if need be.
|
||||
for (const m of n.nodeValue.matchAll(/[^\t]+/g)) {
|
||||
const range = document.createRange();
|
||||
range.setStart(n, m.index); range.setEnd(n, m.index + m[0].length);
|
||||
for (const r of range.getClientRects()) {
|
||||
if (r.width < 0.5 || r.height < 0.5) continue;
|
||||
const b = rel(r);
|
||||
lines.push({ ...b, kind, page: pageOf((b.top + b.bottom) / 2), text: m[0].trim().slice(0, 40) });
|
||||
}
|
||||
}
|
||||
}
|
||||
const issues = [];
|
||||
const box = (b) => [b.left, b.top, b.right, b.bottom].map(Math.round).join(',');
|
||||
const issue = (kind, l, extra = {}) => { if (issues.length < 60) issues.push({ kind, page: l.page + 1, text: l.text, at: l.left != null ? box(l) : undefined, ...extra }); };
|
||||
|
||||
// Two painted lines on one page may not share their box (a frame may, a formula's
|
||||
// pieces may). ponytail: a sorted sweep, quadratic only across lines that share a band.
|
||||
lines.sort((a, b) => a.top - b.top);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const a = lines[i];
|
||||
if (a.kind === 'frame' || a.kind === 'formula') continue;
|
||||
for (let j = i + 1; j < lines.length && lines[j].top < a.bottom; j++) {
|
||||
const b = lines[j];
|
||||
if (b.kind === 'frame' || b.kind === 'formula' || b.page !== a.page) continue;
|
||||
const dy = Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top);
|
||||
const dx = Math.min(a.right, b.right) - Math.max(a.left, b.left);
|
||||
if (dy > 0.5 * Math.min(a.bottom - a.top, b.bottom - b.top) && dx > 2) issue('overlap', a, { other: b.text, otherKind: b.kind, otherAt: box(b) });
|
||||
}
|
||||
}
|
||||
// Body text stays on a sheet and, in a single-section document, inside the margins.
|
||||
for (const l of lines) {
|
||||
if (l.kind !== 'body' && l.kind !== 'note') continue;
|
||||
if (l.page < 0) { issue('in the page gap', l); continue; }
|
||||
const s = sheets[l.page];
|
||||
if (l.left < s.left - 1 || l.right > s.right + 1) issue('off the page', l, { sheet: box(s) });
|
||||
if (!oneSection) continue;
|
||||
const [ml, mr] = side == null ? [margins.left, margins.right] : [side, side];
|
||||
const body = { left: s.left + ml, top: s.top + margins.top, right: s.right - mr, bottom: s.bottom - margins.bottom };
|
||||
if (l.top < body.top - 3) issue('above the top margin', l, { body: box(body) });
|
||||
if (l.bottom > body.bottom + 3) issue('below the bottom margin', l, { body: box(body) });
|
||||
// ponytail: 6px of slack for a justified line's overhang and the sheet's own border.
|
||||
if (l.left < body.left - 6 || l.right > body.right + 6) issue('beyond the side margin', l, { body: box(body) });
|
||||
}
|
||||
// A heading stays with the block it introduces, across a page and a column alike,
|
||||
// unless that block asks for the break itself.
|
||||
const blocks = (parent) => [...parent.children].filter((e) => !e.hasAttribute('data-page-break-spacer'));
|
||||
for (const h of paper.querySelectorAll('.tiptap :is(h1, h2, h3, h4, h5, h6)')) {
|
||||
if (h.closest('td, th, li, .textbox-node, .hf-layer')) continue;
|
||||
const sib = blocks(h.parentElement);
|
||||
const next = sib[sib.indexOf(h) + 1];
|
||||
if (!next || next.classList.contains('note') || next.hasAttribute('data-page-break-before')) continue;
|
||||
const hr = rel(h.getBoundingClientRect()), nr = rel(next.getBoundingClientRect());
|
||||
if (hr.bottom - hr.top < 1 || nr.bottom - nr.top < 1) continue;
|
||||
const l = { page: pageOf(hr.bottom - 1), text: h.textContent.trim().slice(0, 40) };
|
||||
if (pageOf(nr.top + 1) !== l.page) issue('heading alone at the page end', l);
|
||||
else if (h.closest('.columns-node') && nr.left > hr.left + 5) issue('heading alone at the column end', l);
|
||||
}
|
||||
const tiptap = document.querySelector('.tiptap');
|
||||
if (tiptap.scrollWidth > tiptap.clientWidth + 1) issues.push({ kind: 'content wider than the page', page: 0, text: `${tiptap.scrollWidth}px in ${tiptap.clientWidth}px` });
|
||||
return { pages: sheets.length, lines: lines.length, issues };
|
||||
}
|
||||
|
||||
const b64 = (u8) => { let s = ''; for (let i = 0; i < u8.length; i += 0x8000) s += String.fromCharCode(...u8.subarray(i, i + 0x8000)); return btoa(s); };
|
||||
|
||||
try {
|
||||
await page.goto(`http://localhost:${PORT}/`, { waitUntil: 'load' });
|
||||
await page.waitForSelector('.tiptap', { timeout: 30_000 });
|
||||
// The fuzz seeds, written by the app's own exporter in the page.
|
||||
mkdirSync(join(work, 'fuzz'), { recursive: true });
|
||||
for (let seed = 1; seed <= SEEDS; seed++) {
|
||||
const odt = await page.evaluate(async ({ seed, b64 }) => {
|
||||
const [{ buildOdt }, { mulberry32 }, { genCase, exportArgs }] = await Promise.all([
|
||||
import('/src/lib/export/odt.ts'), import('/tests/fuzzDoc.ts'), import('/tests/fuzzOptions.ts')]);
|
||||
const { doc, opts } = genCase(mulberry32(seed));
|
||||
return new Function('return ' + b64)()(await buildOdt(doc, ...exportArgs(opts)));
|
||||
}, { seed, b64: b64.toString() });
|
||||
const f = join(work, 'fuzz', `seed-${seed}.odt`);
|
||||
writeFileSync(f, Buffer.from(odt, 'base64'));
|
||||
docs.push(f);
|
||||
}
|
||||
const todo = docs.filter((d) => !only || only.test(d));
|
||||
const retried = new Set();
|
||||
const lo = LO ? loPages(todo) : new Map();
|
||||
|
||||
for (const file of todo) {
|
||||
const name = file.startsWith(ROOT) ? relative(ROOT, file) : `fuzz/${basename(file)}`;
|
||||
try {
|
||||
// A fresh editor for each document, so its settle cannot be the previous one's.
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await page.reload({ waitUntil: 'load' });
|
||||
await page.waitForSelector('.tiptap', { timeout: 30_000 });
|
||||
await page.setInputFiles('input.file-input', file);
|
||||
// The document's name says the file is in; a document may hold nothing to wait for.
|
||||
await page.waitForFunction((n) => document.querySelector('.doc-name-input')?.value.startsWith(n), basename(file, extname(file)), { timeout: 60_000 });
|
||||
await settle(page, true);
|
||||
const r = await page.evaluate(lintLayout);
|
||||
const ref = lo.get(file);
|
||||
const off = ref == null ? 0 : r.pages - ref;
|
||||
// A page or a tenth either way is the slack two layout engines take.
|
||||
const pagesOk = Math.abs(off) <= Math.max(1, Math.round((ref ?? 0) / 10));
|
||||
const label = `${name} editor ${r.pages}p${ref != null ? ` / LO ${ref}p` : ''}${off && pagesOk ? ' (~)' : ''} ${r.issues.length} issue(s)`;
|
||||
check(pagesOk && !r.issues.length, label);
|
||||
for (const i of r.issues.slice(0, 8)) {
|
||||
const where = i.other ? ` × ${i.otherKind} "${i.other}" @${i.otherAt}` : i.body ? ` (body ${i.body})` : i.sheet ? ` (sheet ${i.sheet})` : '';
|
||||
console.log(` p${i.page} ${i.kind}: "${i.text}"${i.at ? ` @${i.at}` : ''}${where}`);
|
||||
}
|
||||
if (r.issues.length > 8) console.log(` … ${r.issues.length - 8} more`);
|
||||
} catch (err) {
|
||||
// A layout that never settles is a finding of its own, not the end of the run; a
|
||||
// page torn down under the probe (the app's own reload) gets one more go.
|
||||
if (/Execution context was destroyed/.test(err.message) && !retried.has(file)) { retried.add(file); todo.push(file); continue; }
|
||||
check(false, `${name}: ${err.message.split('\n')[0]}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
check(false, `layout run threw: ${err.stack ?? err}`);
|
||||
} finally {
|
||||
check(pageErrors.length === 0, pageErrors.length ? `no uncaught page errors — got: ${pageErrors.join(' | ')}` : 'no uncaught page errors');
|
||||
await browser.close();
|
||||
if (server) process.kill(-server.pid);
|
||||
}
|
||||
process.exit(failures.length ? 1 : 0);
|
||||
@@ -0,0 +1,250 @@
|
||||
// LibreOffice as the fuzz's second reader: each seed's ODT re-saved by soffice, and its
|
||||
// DOCX converted to ODT, both read back and held against what went out. Where LibreOffice
|
||||
// understands a file differently from our importer, one of the two is wrong.
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { execFileSync, execSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { buildOdt } from '../src/lib/export/odt';
|
||||
import { buildDocx } from '../src/lib/export/docx';
|
||||
import { importOdt } from '../src/lib/import/odt';
|
||||
import { normalize, stripFontHoist } from './normalize';
|
||||
import { mulberry32 } from './fuzzDoc';
|
||||
import { genCase, exportArgs, expectedOptions, importedOptions, diffLoose, omit, DOCX_LOSSY, docWideOddEven, type FuzzOptions } from './fuzzOptions';
|
||||
|
||||
type N = any;
|
||||
type Fmt = 'odt' | 'docx';
|
||||
|
||||
const SEEDS = Number(process.env.LO_SEEDS ?? 10); // LO_SEEDS=100 for a wide sweep
|
||||
const DUMP = process.env.LO_DUMP; // a path: every diff of every seed, for triage
|
||||
const SOFFICE = (() => { try { execSync('command -v soffice', { stdio: 'ignore' }); return true; } catch { return false; } })();
|
||||
|
||||
// What LibreOffice's DOCX reader has no place for beyond Word's own list: fold marks are
|
||||
// our construct, and heading numbering comes back as list styles on the heading styles.
|
||||
const LOSSY: Record<Fmt, string[]> = { odt: [], docx: [...DOCX_LOSSY, 'foldMarks', 'outline', 'spacingAtPageStart'] };
|
||||
|
||||
// One soffice call per batch: a cold start costs seconds, a document milliseconds. Its
|
||||
// own profile, so a LibreOffice the user has open does not swallow the call.
|
||||
function convertAll(files: string[], outDir: string): void {
|
||||
mkdirSync(outDir, { recursive: true });
|
||||
execFileSync('soffice', ['--headless', '--norestore', `-env:UserInstallation=file://${join(outDir, '..', 'profile')}`,
|
||||
'--convert-to', 'odt', '--outdir', outDir, ...files], { stdio: 'pipe', timeout: 900_000 });
|
||||
}
|
||||
|
||||
// A font a paragraph carries for all its runs is written on the paragraph alone by
|
||||
// LibreOffice; read it back onto the runs, where the authored document keeps it.
|
||||
function unhoist(node: N): N {
|
||||
const { fontSize, fontFamily } = node.attrs ?? {};
|
||||
if ((fontSize || fontFamily) && node.content) {
|
||||
for (const c of node.content) {
|
||||
if (c.type !== 'text') continue;
|
||||
const ts = (c.marks ??= []).find((m: N) => m.type === 'textStyle') ?? (c.marks.push({ type: 'textStyle', attrs: {} }), c.marks[c.marks.length - 1]);
|
||||
ts.attrs = { ...(fontSize ? { fontSize } : {}), ...(fontFamily ? { fontFamily } : {}), ...ts.attrs };
|
||||
}
|
||||
}
|
||||
for (const c of node.content ?? []) unhoist(c);
|
||||
return node;
|
||||
}
|
||||
|
||||
// LibreOffice's own reading: pictures re-encoded, sizes rounded through cm, a width on
|
||||
// every column, a hyperlink in its Internet Link style, a comment dated by its own clock;
|
||||
// from DOCX an index mark is dropped, a list style "WWNum", a lone operator quoted text.
|
||||
function loNoise(node: N, fmt: Fmt): N {
|
||||
const a = node.attrs ?? {};
|
||||
for (const [k, v] of Object.entries(a)) {
|
||||
if (k === 'src') a.src = 'IMG';
|
||||
else if ((k === 'width' || k === 'height' || k === 'rowHeight') && typeof v === 'number') a[k] = Math.round(v / 3) * 3;
|
||||
else if (k === 'colwidth' || (k === 'wrapOffsetY' && v === 0)) delete a[k];
|
||||
else if ((k === 'marginLeft' || k === 'marginRight') && typeof v === 'number') { if (Math.abs(v) < 0.05) delete a[k]; else a[k] = Math.round(v * 10) / 10; }
|
||||
else if ((k === 'bulletChar' || k === 'textVertical' || k === 'alt') && fmt === 'docx') delete a[k];
|
||||
else if (k === 'textPosition' && typeof v === 'number') a[k] = Math.round(v * 2) / 2;
|
||||
else if (k.startsWith('border') && typeof v === 'string') a[k] = v.replace(/^([\d.]+)pt/, (_m, w) => `${Math.round(Number(w) * 4) / 4}pt`);
|
||||
else if (k === 'latex' && fmt === 'docx') a.latex = String(v).replace(/\\text\{(.)\}/g, '$1');
|
||||
else if ((k === 'listStyleName' || k === 'listStyleType') && fmt === 'docx') delete a[k];
|
||||
else if ((k === 'styleName' || k === 'name') && typeof v === 'string') a[k] = v.replace(/ \(WW\)$/, '');
|
||||
}
|
||||
// A note's label and a citation's fields are recomputed on its side; a table style is
|
||||
// its look baked into the cells there, so a styled table compares by its text alone.
|
||||
if (node.type === 'noteRef' || node.type === 'note') delete a.text;
|
||||
// A note's text opens with the tab LibreOffice's DOCX reader puts after the number; a
|
||||
// figure or table index is a paragraph to it.
|
||||
if (node.type === 'note' && fmt === 'docx') {
|
||||
// A custom mark comes back as the note's first run, in the symbol style.
|
||||
if (node.content?.[0]?.marks?.some((m: N) => /Symbol$/.test(m.attrs?.name ?? ''))) node.content.shift();
|
||||
if (node.content?.[0]?.text) node.content[0].text = node.content[0].text.replace(/^\t/, '');
|
||||
}
|
||||
// Its DOCX reader spells a heading paragraph's zero spacing out.
|
||||
if (fmt === 'docx' && node.type === 'paragraph') { if (a.spaceAfter === 0) delete a.spaceAfter; if (a.spaceBefore === 0) delete a.spaceBefore; }
|
||||
if (node.type === 'tableOfContents' && fmt === 'docx' && ['tables', 'figures'].includes(a.index)) { node.attrs = undefined; delete node.content; node.type = 'paragraph'; }
|
||||
if (node.type === 'bibliographyEntry' && fmt === 'docx') node.attrs = {};
|
||||
// Word centres a display formula's paragraph; LibreOffice reads that as its alignment.
|
||||
if (fmt === 'docx' && node.type === 'paragraph' && node.content?.length === 1 && node.content[0].type === 'formula' && node.content[0].attrs?.display && a.textAlign === 'center') delete a.textAlign;
|
||||
if (node.attrs && !Object.keys(a).length) delete node.attrs;
|
||||
// A comment's date comes back in LibreOffice's local time; a tracked deletion wears
|
||||
// its review look (strike, underline) as formatting; a run in a character style has
|
||||
// that style's font written out beside it.
|
||||
for (const m of node.marks ?? []) {
|
||||
if (m.type === 'comment' && m.attrs) delete m.attrs.date;
|
||||
// A style name LibreOffice's DOCX reader reserves for a Word name comes back "… (WW)".
|
||||
if (m.type === 'charStyle' && m.attrs?.name) m.attrs.name = m.attrs.name.replace(/ \(WW\)$/, '');
|
||||
}
|
||||
// A tracked deletion comes back in the look of the run beside it; a raised run's
|
||||
// percentage is rounded on the way.
|
||||
if (node.marks?.some((m: N) => m.type === 'deletion' || m.type === 'insertion')) {
|
||||
node.marks = node.marks.filter((m: N) => ['deletion', 'insertion', 'link'].includes(m.type));
|
||||
}
|
||||
for (const m of node.marks ?? []) if (typeof m.attrs?.textPosition === 'number') m.attrs.textPosition = Math.round(m.attrs.textPosition * 2) / 2;
|
||||
if (node.marks?.some((m: N) => m.type === 'charStyle')) for (const m of node.marks) if (m.type === 'textStyle' && m.attrs) delete m.attrs.fontFamily;
|
||||
if (node.marks) {
|
||||
node.marks = node.marks.filter((m: N) => m.type !== 'textStyle' || Object.keys(m.attrs ?? {}).length);
|
||||
if (!node.marks.length) delete node.marks;
|
||||
}
|
||||
if (node.marks?.some((m: N) => m.type === 'link')) {
|
||||
node.marks = node.marks.filter((m: N) => m.type !== 'underline');
|
||||
for (const m of node.marks) if (m.type === 'textStyle' && m.attrs) delete m.attrs.color;
|
||||
node.marks = node.marks.filter((m: N) => m.type !== 'textStyle' || Object.keys(m.attrs ?? {}).length);
|
||||
if (!node.marks.length) delete node.marks;
|
||||
}
|
||||
// A paragraph in a text box loses its own field and rule lines on the way.
|
||||
if (node.type === 'textBox') for (const p of node.content ?? []) for (const k of Object.keys(p.attrs ?? {})) if (k === 'backgroundColor' || k.startsWith('border')) delete p.attrs[k];
|
||||
if (node.content) {
|
||||
// Its DOCX reader drops an index mark; beside a ruby it moves one and flattens the
|
||||
// ruby to its base text, so both compare as that.
|
||||
const ruby = node.content.some((c: N) => c.type === 'ruby'), mark = node.content.some((c: N) => c.type === 'indexEntry');
|
||||
node.content = node.content.filter((c: N) => !((fmt === 'docx' || ruby) && c.type === 'indexEntry'))
|
||||
.map((c: N) => (fmt === 'odt' && ruby && mark && c.type === 'ruby' ? { type: 'text', text: c.attrs.base } : loNoise(c, fmt)));
|
||||
// A document may not end on a table in Writer, so one gets a paragraph after it
|
||||
// (ahead of the notes, which the model keeps last).
|
||||
const end = node.content.length - (node.content[node.content.length - 1]?.type === 'noteSection' ? 2 : 1);
|
||||
const last = node.content[end], prev = node.content[end - 1];
|
||||
if (node.type === 'doc' && last?.type === 'paragraph' && !last.content && !last.attrs && prev?.type === 'table') node.content.splice(end, 1);
|
||||
if (!node.content.length) delete node.content;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
// A table style is its look baked into the cells on LibreOffice's side, and a formula
|
||||
// cell shows the value it recomputes in its locale's format: the authored document's
|
||||
// styled tables compare by their text alone, its formula cells by nothing.
|
||||
type Marked = { styled: Set<number>; formulas: Set<string> };
|
||||
function markedTables(doc: N): Marked {
|
||||
const styled = new Set<number>(), formulas = new Set<string>();
|
||||
(doc.content ?? []).forEach((b: N, i: number) => {
|
||||
if (b.type !== 'table') return;
|
||||
if (b.attrs?.tableStyle) styled.add(i);
|
||||
b.content?.forEach((row: N, r: number) => row.content?.forEach((cell: N, c: number) => { if (cell.attrs?.formula) formulas.add(`${i}/${r}/${c}`); }));
|
||||
});
|
||||
return { styled, formulas };
|
||||
}
|
||||
function bareTables(doc: N, marked: Marked, fmt: Fmt): N {
|
||||
doc.content?.forEach((b: N, i: number) => {
|
||||
if (b.type !== 'table') return;
|
||||
if (marked.styled.has(i)) {
|
||||
delete b.attrs?.tableStyle; delete b.attrs?.tableLook;
|
||||
(function bare(n: N) { delete n.marks; if (n.attrs) { delete n.attrs.region; if (!Object.keys(n.attrs).length) delete n.attrs; } for (const c of n.content ?? []) bare(c); })(b);
|
||||
}
|
||||
b.content?.forEach((row: N, r: number) => row.content?.forEach((cell: N, c: number) => {
|
||||
if (!marked.formulas.has(`${i}/${r}/${c}`)) return;
|
||||
delete cell.content;
|
||||
if (fmt === 'docx' && cell.attrs) { delete cell.attrs.formula; delete cell.attrs.cellFormat; if (!Object.keys(cell.attrs).length) delete cell.attrs; }
|
||||
}));
|
||||
});
|
||||
return doc;
|
||||
}
|
||||
const loContent = (doc: N, fmt: Fmt, marked: Marked) => bareTables(loNoise(stripFontHoist(normalize(unhoist(structuredClone(doc)))), fmt), marked, fmt);
|
||||
|
||||
// The canonical options as LibreOffice hands them back: a chapter field's cached name is
|
||||
// recomputed (and via DOCX, a STYLEREF field is not read at all); a watermark rides a
|
||||
// header band it floors at 0.499cm, so a section without a header of its own grows by it.
|
||||
function loOptions(canon: N, fmt: Fmt, opts: FuzzOptions, authored: boolean): N {
|
||||
// Via DOCX a zone's runs come back in the look LibreOffice gives fields and headings,
|
||||
// so there a zone compares by its text and fields alone.
|
||||
const zoneFix = (zone: N): void => {
|
||||
for (const p of zone?.content ?? []) {
|
||||
if (fmt === 'docx') p.content = p.content?.filter((c: N) => c.type !== 'chapterField').map((c: N) => (c.type === 'text' ? { type: 'text', text: c.text } : { type: c.type }));
|
||||
for (const c of p.content ?? []) if (c.type === 'chapterField') delete c.attrs.text;
|
||||
if (p.content && !p.content.length) delete p.content;
|
||||
}
|
||||
};
|
||||
// A style naming itself as next names the default; a rule's width is rounded through cm.
|
||||
for (const [name, st] of Object.entries(canon.styles ?? {}) as [string, N][]) {
|
||||
if (!st) continue;
|
||||
if (st.next === name) st.next = null;
|
||||
for (const [k, v] of Object.entries(st.para ?? {})) if (k.startsWith('border') && typeof v === 'string') st.para[k] = (v as string).replace(/^([\d.]+)pt/, (_m, w) => `${Math.round(Number(w) * 4) / 4}pt`);
|
||||
}
|
||||
for (const s of canon.sections) {
|
||||
for (const k of ['header', 'footer', 'headerFirst', 'footerFirst', 'headerEven', 'footerEven']) zoneFix(s[k]);
|
||||
if (authored && fmt === 'odt' && (opts.decor.watermark || opts.foldMarks) && !s.header && !s.headerFirst && !s.headerEven && s.margins) {
|
||||
s.margins.top = Math.round((s.margins.top + 0.499) * 1000) / 1000;
|
||||
}
|
||||
}
|
||||
return fmt === 'docx' && authored ? docWideOddEven(canon) : canon;
|
||||
}
|
||||
|
||||
// Every leaf that differs, for the LO_DUMP triage file (firstDiff stops at the first).
|
||||
function allDiffs(a: N, b: N, path = '$', out: string[] = []): string[] {
|
||||
if (out.length >= 12 || (a == null && b == null)) return out;
|
||||
if (typeof a === 'number' && typeof b === 'number') { if (Math.abs(a - b) > 0.02) out.push(`${path}: ${a} vs ${b}`); return out; }
|
||||
if (typeof a !== typeof b || a == null || b == null || typeof a !== 'object') {
|
||||
if (!Object.is(a, b)) out.push(`${path}: ${JSON.stringify(a)?.slice(0, 300)} vs ${JSON.stringify(b)?.slice(0, 300)}`);
|
||||
return out;
|
||||
}
|
||||
for (const k of new Set([...Object.keys(a), ...Object.keys(b)])) allDiffs(a[k], b[k], `${path}.${k}`, out);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Writer joins two tables that touch, as its DOCX reader finds ours; that reader also
|
||||
// swallows the blocks after an index field with no entries, and restructures a list in a
|
||||
// table cell, a floating picture in a list item and a list three deep: nothing to compare.
|
||||
const adjacentTables = (doc: N) => (doc.content ?? []).some((b: N, i: number) => b.type === 'table' && doc.content[i + 1]?.type === 'table');
|
||||
const emptyIndex = (doc: N) => (doc.content ?? []).some((b: N) => b.type === 'tableOfContents' && !(b.attrs?.entries ?? []).length);
|
||||
const has = (n: N, test: (x: N) => boolean): boolean => test(n) || (n.content ?? []).some((c: N) => has(c, test));
|
||||
const listInCell = (doc: N) => has(doc, (n) => n.type === 'tableCell' && has(n, (x) => /List$/.test(x.type)));
|
||||
const floating = (n: N) => has(n, (x) => x.type === 'image' && x.attrs?.wrap && x.attrs.wrap !== 'inline');
|
||||
const floatInList = (doc: N) => has(doc, (n) => n.type === 'listItem' && floating(n))
|
||||
|| (doc.content ?? []).some((b: N, i: number) => isList(b) && doc.content[i + 1] && floating(doc.content[i + 1]));
|
||||
const isList = (n: N) => /List$/.test(n.type);
|
||||
const deepList = (doc: N) => has(doc, (n) => isList(n) && (n.content ?? []).some((li: N) => (li.content ?? []).some((x: N) => isList(x) && (x.content ?? []).some((li2: N) => (li2.content ?? []).some(isList)))));
|
||||
const hardForDocx = (doc: N) => adjacentTables(doc) || emptyIndex(doc) || listInCell(doc) || floatInList(doc) || deepList(doc);
|
||||
|
||||
describe.skipIf(!SOFFICE)('fuzz through LibreOffice (needs soffice on PATH)', () => {
|
||||
const work = mkdtempSync(join(tmpdir(), 'lo-fuzz-'));
|
||||
const cases = Array.from({ length: SEEDS }, (_, i) => ({ seed: i + 1, ...genCase(mulberry32(i + 1)) }));
|
||||
const dump: string[] = [];
|
||||
|
||||
for (const [fmt, build] of [['odt', buildOdt], ['docx', buildDocx]] as const) {
|
||||
it(`${SEEDS} seeds as ${fmt.toUpperCase()}: LibreOffice reads them as they were written`, async () => {
|
||||
const dir = join(work, fmt);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const files: string[] = [];
|
||||
for (const c of cases) {
|
||||
const f = join(dir, `seed-${c.seed}.${fmt}`);
|
||||
writeFileSync(f, await build(c.doc, ...exportArgs(c.opts)));
|
||||
files.push(f);
|
||||
}
|
||||
convertAll(files, join(dir, 'out'));
|
||||
for (const c of cases) {
|
||||
const f = join(dir, 'out', `seed-${c.seed}.odt`);
|
||||
if (!existsSync(f)) { expect.soft(f, `seed ${c.seed}: LibreOffice wrote no file`).toBe(''); continue; }
|
||||
const res = importOdt(new Uint8Array(readFileSync(f)));
|
||||
const marked = markedTables(c.doc);
|
||||
const [want, got] = [loContent(c.doc, fmt, marked), loContent(res.content, fmt, marked)];
|
||||
const skipContent = fmt === 'docx' && hardForDocx(c.doc);
|
||||
// Loose on numbers: a size or padding comes back a thousandth off through cm.
|
||||
const diff = skipContent ? null : diffLoose(want, got);
|
||||
expect.soft(diff, `seed ${c.seed}: content via ${fmt}`).toBeNull();
|
||||
const exp = omit(loOptions(expectedOptions(c.opts), fmt, c.opts, true), LOSSY[fmt]);
|
||||
const imp = omit(loOptions(importedOptions(res, c.opts), fmt, c.opts, false), LOSSY[fmt]);
|
||||
// A file naming no language gets LibreOffice's own; via DOCX a page-number start
|
||||
// needs a leading paragraph to ride on.
|
||||
if (exp.language == null) { delete exp.language; delete imp.language; }
|
||||
if (fmt === 'docx' && !['paragraph', 'heading'].includes(c.doc.content?.[0]?.type)) { delete exp.pageNumberStart; delete imp.pageNumberStart; }
|
||||
const optDiff = diffLoose(exp, imp);
|
||||
expect.soft(optDiff, `seed ${c.seed}: options via ${fmt}`).toBeNull();
|
||||
if (DUMP) dump.push(`## seed ${c.seed} ${fmt}: warnings=${JSON.stringify(res.warnings)}`,
|
||||
...(skipContent ? [' C (not compared)'] : allDiffs(want, got).map((d) => ` C ${d}`)), ...allDiffs(exp, imp).map((d) => ` O ${d}`));
|
||||
}
|
||||
if (DUMP) writeFileSync(DUMP, dump.join('\n') + '\n');
|
||||
}, 900_000);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,222 @@
|
||||
// Monkey run: random editing in the real browser under the invariants any sequence of
|
||||
// commands must keep — no uncaught error, a document its schema accepts, undo back to the
|
||||
// opened file and redo forward again, and a saved file the schema validates and the
|
||||
// importer reads back as the editor holds it. A failure names the seed and the last ops.
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { unzipSync } from 'fflate';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { ROOT, MOD, checker, devServer, openApp } from '../browser.mjs';
|
||||
import { hasXmllint, validateOdt, validateDocx } from '../schemaValidate.ts';
|
||||
|
||||
// The schema helper strips foreign markup with the browser's DOM parser.
|
||||
Object.assign(globalThis, (({ DOMParser, XMLSerializer }) => ({ DOMParser, XMLSerializer }))(new JSDOM('').window));
|
||||
|
||||
const PORT = +(process.env.MONKEY_PORT ?? 4189);
|
||||
const SEED = +(process.env.MONKEY_SEED ?? 1);
|
||||
const RUNS = +(process.env.MONKEY_RUNS ?? 3);
|
||||
const OPS = +(process.env.MONKEY_OPS ?? 60);
|
||||
const DOCS = process.env.MONKEY_DOC ? [process.env.MONKEY_DOC] : ['02-blocks.odt', '04-table.odt', '08-lists.odt', '12-notes.odt', '15-chapters.odt'];
|
||||
const PNG = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
||||
const { check, failures } = checker();
|
||||
if (!hasXmllint) console.log('xmllint missing: the saved files are not validated against the schemas');
|
||||
|
||||
// fuzzDoc.ts's generator, so a seed replays.
|
||||
function mulberry32(seed) {
|
||||
let a = seed >>> 0;
|
||||
return () => {
|
||||
a = (a + 0x6d2b79f5) >>> 0;
|
||||
let t = a;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
const pick = (r, arr) => arr[Math.floor(r() * arr.length)];
|
||||
const int = (r, min, max) => min + Math.floor(r() * (max - min + 1));
|
||||
|
||||
const WORDS = ['lorem ', 'Ipsum', ' dolor sit ', 'äöü ', '42', '. ', 'A', ' — ', 'x'];
|
||||
const KEYS = ['Enter', 'Enter', 'Backspace', 'Backspace', 'Delete', 'Tab', 'Shift+Tab', 'ArrowLeft', 'ArrowRight', 'ArrowUp', 'ArrowDown',
|
||||
'Shift+ArrowLeft', 'Shift+ArrowRight', 'Shift+ArrowDown', 'Home', 'End', `${MOD}+b`, `${MOD}+i`, `${MOD}+u`, `${MOD}+Shift+7`, `${MOD}+Shift+8`,
|
||||
`${MOD}+Enter`, `${MOD}+z`, `${MOD}+Shift+z`];
|
||||
const STYLES = ['Standard', 'Heading 1', 'Heading 2', 'Heading 3', 'Title', 'Quotations'];
|
||||
// Each entry: a weight and a draw of the op — a key, typed text, or an editor command.
|
||||
const OPS_TABLE = [
|
||||
[24, (r) => ({ type: pick(r, WORDS) + (r() < 0.3 ? pick(r, WORDS) : '') })],
|
||||
[20, (r) => ({ key: pick(r, KEYS) })],
|
||||
[10, (r, size) => ({ cmd: 'focus', args: [int(r, 1, Math.max(1, size - 2))] })],
|
||||
[6, (r, size) => { const a = int(r, 1, Math.max(1, size - 2)), b = int(r, 1, Math.max(1, size - 2)); return { cmd: 'setTextSelection', args: [{ from: Math.min(a, b), to: Math.max(a, b) }] }; }],
|
||||
[8, (r) => pick(r, [
|
||||
{ cmd: 'toggleBold' }, { cmd: 'toggleItalic' }, { cmd: 'toggleUnderline' }, { cmd: 'toggleStrike' },
|
||||
{ cmd: 'toggleSubscript' }, { cmd: 'toggleSuperscript' }, { cmd: 'setColor', args: ['#c00000'] },
|
||||
{ cmd: 'setFontSize', args: ['14pt'] }, { cmd: 'setFontFamily', args: ['Liberation Sans'] }, { cmd: 'unsetAllMarks' },
|
||||
])],
|
||||
[12, (r) => pick(r, [
|
||||
{ cmd: 'setParagraphStyle', args: [pick(r, STYLES)] }, { cmd: 'setTextAlign', args: [pick(r, ['left', 'center', 'right', 'justify'])] },
|
||||
{ cmd: 'indentMore' }, { cmd: 'indentLess' }, { cmd: 'setLineHeight', args: ['1.5'] }, { cmd: 'setSpaceBefore', args: [6] },
|
||||
{ cmd: 'toggleBulletList' }, { cmd: 'toggleOrderedList' }, { cmd: 'sinkListItem', args: ['listItem'] }, { cmd: 'liftListItem', args: ['listItem'] },
|
||||
{ cmd: 'insertPageBreak' }, { cmd: 'setColumns', args: [2] }, { cmd: 'setColumns', args: [1] }, { cmd: 'clearDirectFormatting' },
|
||||
])],
|
||||
[8, (r) => pick(r, [
|
||||
{ cmd: 'insertTable', args: [{ rows: int(r, 2, 3), cols: int(r, 2, 3), withHeaderRow: r() < 0.5 }] },
|
||||
{ cmd: 'addRowAfter' }, { cmd: 'addColumnAfter' }, { cmd: 'deleteRow' }, { cmd: 'deleteColumn' },
|
||||
{ cmd: 'mergeCells' }, { cmd: 'splitCell' }, { cmd: 'deleteTable' }, { cmd: 'goToNextCell' },
|
||||
])],
|
||||
[8, (r) => pick(r, [
|
||||
{ cmd: 'insertNote', args: ['footnote'] }, { cmd: 'insertNote', args: ['endnote'] },
|
||||
{ cmd: 'insertFormula', args: [{ latex: 'x^{2}+1', display: false }] }, { cmd: 'insertTextBox' },
|
||||
{ cmd: 'setImage', args: [{ src: PNG, width: 80, height: 60 }] }, { cmd: 'setHardBreak' },
|
||||
{ cmd: 'addComment', args: [{ author: 'Monkey', text: 'hm' }] }, { cmd: 'insertContent', args: ['—'] },
|
||||
])],
|
||||
[3, (r) => ({ cmd: pick(r, ['undo', 'redo']) })],
|
||||
];
|
||||
const TOTAL = OPS_TABLE.reduce((n, [w]) => n + w, 0);
|
||||
const drawOp = (r, size) => {
|
||||
let x = r() * TOTAL;
|
||||
for (const [w, draw] of OPS_TABLE) { if ((x -= w) < 0) return draw(r, size); }
|
||||
return { key: 'ArrowRight' };
|
||||
};
|
||||
|
||||
const server = await devServer(PORT);
|
||||
const { browser, page, pageErrors } = await openApp(PORT);
|
||||
const consoleErrors = [];
|
||||
page.on('console', (m) => { if (m.type() === 'error') consoleErrors.push(m.text()); });
|
||||
|
||||
const ed = (fn, arg) => page.evaluate(fn, arg);
|
||||
const focused = async () => {
|
||||
const err = await ed(() => { try { document.querySelector('.tiptap').editor.commands.focus(); return null; } catch (e) { return String(e); } });
|
||||
await page.waitForFunction(() => document.activeElement === document.querySelector('.tiptap'), null, { timeout: 2000 }).catch(() => {});
|
||||
return err;
|
||||
};
|
||||
const runCmd = (op) => ed(({ cmd, args }) => {
|
||||
const editor = document.querySelector('.tiptap').editor;
|
||||
try { return { ok: editor.commands[cmd](...(args ?? [])) }; } catch (e) { return { threw: String(e?.stack ?? e) }; }
|
||||
}, op);
|
||||
// The schema's own check of the document after every op — a command may leave a node
|
||||
// with content its spec rejects, which nothing else reports.
|
||||
const docCheck = () => ed(() => {
|
||||
try { document.querySelector('.tiptap').editor.state.doc.check(); return null; } catch (e) { return String(e); }
|
||||
});
|
||||
const docJson = () => ed(() => JSON.stringify(document.querySelector('.tiptap').editor.getJSON()));
|
||||
const docSize = () => ed(() => document.querySelector('.tiptap').editor.state.doc.content.size);
|
||||
|
||||
const saveAs = async (ext) => {
|
||||
await page.click('.ribbon-tab-file');
|
||||
const [dl] = await Promise.all([
|
||||
page.waitForEvent('download', { timeout: 60_000 }),
|
||||
page.locator('.ribbon-menu button', { hasText: `(.${ext})` }).first().click(),
|
||||
]);
|
||||
await page.keyboard.press('Escape');
|
||||
return new Uint8Array(await readFile(await dl.path()));
|
||||
};
|
||||
// The file the app saved, read back by the app's importer against the editor's document
|
||||
// (its paragraphs joined across a column or page merged, as the export merges them).
|
||||
const roundTrip = (bytes, ext) => ed(async ({ b64, ext }) => {
|
||||
const [{ importOdt }, { importDocx }, { mergeJoinedParagraphsJson }, { normalize, firstDiff, stripFontHoist }] = await Promise.all([
|
||||
import('/src/lib/import/odt.ts'), import('/src/lib/import/docx.ts'), import('/src/lib/export/odt.ts'), import('/tests/normalize.ts')]);
|
||||
const bin = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0));
|
||||
const res = ext === 'odt' ? importOdt(bin) : importDocx(bin);
|
||||
const editor = document.querySelector('.tiptap').editor;
|
||||
const json = editor.getJSON();
|
||||
// What the importer suppresses as the style's own: a block attribute equal to what its
|
||||
// named style resolves to, and bold in a header cell.
|
||||
const { resolveStyle } = await import('/src/lib/styles/styleSheet.ts');
|
||||
const sheet = editor.extensionManager.extensions.find((e) => e.name === 'paragraphStyle')?.options.sheet();
|
||||
const styled = (n, header = false) => {
|
||||
if (n.attrs?.styleName && sheet) {
|
||||
const own = resolveStyle(sheet, n.attrs.styleName, 'paragraph').para ?? {};
|
||||
for (const [k, v] of Object.entries(n.attrs)) if (k !== 'styleName' && own[k] === v) delete n.attrs[k];
|
||||
}
|
||||
if (header && n.type === 'text' && n.marks) n.marks = n.marks.filter((m) => m.type !== 'bold');
|
||||
// A formula alone in its paragraph is a display formula to both files.
|
||||
if (n.type === 'paragraph' && n.content?.length === 1 && n.content[0].type === 'formula') n.content[0].attrs = { ...n.content[0].attrs, display: true };
|
||||
for (const c of n.content ?? []) styled(c, header || n.type === 'tableHeader');
|
||||
return n;
|
||||
};
|
||||
const held = styled({ ...json, content: mergeJoinedParagraphsJson(json.content ?? []) });
|
||||
return { diff: firstDiff(stripFontHoist(normalize(held)), stripFontHoist(normalize(styled(res.content)))), warnings: res.warnings, held: JSON.stringify(held), read: JSON.stringify(res.content) };
|
||||
}, { b64: Buffer.from(bytes).toString('base64'), ext });
|
||||
|
||||
try {
|
||||
await page.goto(`http://localhost:${PORT}/`, { waitUntil: 'load' });
|
||||
await page.evaluate(() => { localStorage.clear(); window.showSaveFilePicker = undefined; });
|
||||
await page.reload({ waitUntil: 'load' });
|
||||
await page.waitForSelector('.tiptap', { timeout: 30_000 });
|
||||
await page.evaluate(() => { window.showSaveFilePicker = undefined; });
|
||||
|
||||
for (let run = 0; run < RUNS; run++) {
|
||||
const seed = SEED + run;
|
||||
const doc = DOCS[run % DOCS.length];
|
||||
const r = mulberry32(seed);
|
||||
const before = await docJson();
|
||||
await page.setInputFiles('input.file-input', join(ROOT, 'tests/corpus', doc));
|
||||
await page.waitForFunction((b) => JSON.stringify(document.querySelector('.tiptap').editor.getJSON()) !== b, before, { timeout: 30_000 });
|
||||
await page.waitForTimeout(1500);
|
||||
const initial = await docJson();
|
||||
const log = [];
|
||||
let broken = null;
|
||||
for (let i = 0; i < OPS && !broken; i++) {
|
||||
const op = drawOp(r, await docSize());
|
||||
log.push(op);
|
||||
if (op.type || op.key) {
|
||||
const err = await focused();
|
||||
if (err) broken = `focus threw: ${err}`;
|
||||
else if (op.type) await page.keyboard.type(op.type);
|
||||
else await page.keyboard.press(op.key);
|
||||
} else {
|
||||
const res = await runCmd(op);
|
||||
if (res.threw) broken = `${op.cmd} threw: ${res.threw}`;
|
||||
}
|
||||
const invalid = await docCheck();
|
||||
if (invalid) broken = `schema-invalid document: ${invalid}`;
|
||||
if (pageErrors.length) broken = `uncaught: ${pageErrors.join(' | ')}`;
|
||||
}
|
||||
const final = await docJson();
|
||||
await writeFile(join(tmpdir(), `monkey-${seed}-ops.json`), JSON.stringify(log));
|
||||
// Undo until the opened document is back, redo as often, and the edited one is back.
|
||||
const undone = broken ? -1 : await ed((initial) => {
|
||||
const editor = document.querySelector('.tiptap').editor;
|
||||
for (let n = 0; n < 500; n++) {
|
||||
if (JSON.stringify(editor.getJSON()) === initial) return n;
|
||||
if (!editor.can().undo()) return -1;
|
||||
try { editor.commands.undo(); } catch (e) { return `undo threw: ${String(e).slice(0, 300)}`; }
|
||||
}
|
||||
return -1;
|
||||
}, initial);
|
||||
const redone = typeof undone !== 'number' || undone < 0 ? null : await ed((n) => {
|
||||
const editor = document.querySelector('.tiptap').editor;
|
||||
for (let i = 0; i < n; i++) editor.commands.redo();
|
||||
return JSON.stringify(editor.getJSON());
|
||||
}, undone);
|
||||
const history = broken ?? (typeof undone === 'string' ? undone : undone < 0 ? 'undo does not reach the opened document' : redone !== final ? 'redo does not restore the edited document' : null);
|
||||
const label = `seed ${seed} on ${doc}: ${log.length} ops, ${undone} undo steps`;
|
||||
check(!history, history ? `${label} — ${history}` : label);
|
||||
if (history) for (const op of log.slice(-12)) console.log(' ', JSON.stringify(op));
|
||||
if (broken) break;
|
||||
|
||||
for (const ext of ['odt', 'docx']) {
|
||||
const bytes = await saveAs(ext);
|
||||
const schema = hasXmllint ? (ext === 'odt' ? validateOdt : validateDocx)(unzipSync(bytes)) : [];
|
||||
const rt = await roundTrip(bytes, ext).catch((err) => ({ diff: null, warnings: [], threw: String(err.message ?? err).split('\n')[0] }));
|
||||
const bad = rt.threw ? `import threw: ${rt.threw}` : schema.length ? `schema: ${schema[0].slice(0, 300)}`
|
||||
: rt.diff ? `round trip: ${rt.diff}` : rt.warnings.length ? `warnings: ${rt.warnings.join(' | ')}` : null;
|
||||
check(!bad, `seed ${seed}: the saved .${ext} validates and reads back${bad ? ` — ${bad}` : ''}`);
|
||||
if (bad) {
|
||||
// The file and both documents, for the repro.
|
||||
const stem = join(tmpdir(), `monkey-${seed}`);
|
||||
await writeFile(`${stem}.${ext}`, bytes);
|
||||
if (rt.held) await Promise.all([writeFile(`${stem}-${ext}-held.json`, rt.held), writeFile(`${stem}-${ext}-read.json`, rt.read)]);
|
||||
console.log(` kept: ${stem}.${ext}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
check(false, `monkey run threw: ${err.stack ?? err}`);
|
||||
} finally {
|
||||
if (consoleErrors.length) console.log(`console errors (${consoleErrors.length}): ${[...new Set(consoleErrors)].slice(0, 5).join(' | ')}`);
|
||||
check(pageErrors.length === 0, pageErrors.length ? `no uncaught page errors — got: ${pageErrors.join(' | ')}` : 'no uncaught page errors');
|
||||
await browser.close();
|
||||
if (server) process.kill(-server.pid);
|
||||
}
|
||||
process.exit(failures.length ? 1 : 0);
|
||||
+7
-2
@@ -10,6 +10,9 @@ const ORDERED_DEFAULTS: Record<string, unknown> = {
|
||||
type: null, level: undefined, rotation: 0, wrap: 'inline',
|
||||
shapeKind: 'textbox', fillColor: '#FFFFFF', strokeColor: '#000000', strokeWidthPt: 1,
|
||||
fixed: false, key1: '',
|
||||
joinPrev: false, // pageBreaks.ts's split marker; the export merges the halves
|
||||
marginLeft: 0, marginRight: 0, marginTop: 0, marginBottom: 0, // a table's, once a command has touched them
|
||||
alt: '', inFront: false, paddingCm: 0.15, flipV: false, textVertical: false, // the editor's own picture and box defaults
|
||||
// index attrs the DOCX TOC field has no switch for come back at their defaults
|
||||
maxLevel: MAX_HEADING_LEVEL, leader: '.', citationStyle: 'key',
|
||||
};
|
||||
@@ -56,12 +59,14 @@ export function normalize(node: N): N {
|
||||
if (node.type === 'doc') canonNoteIds(node);
|
||||
const out: N = { type: node.type };
|
||||
if (node.text != null) out.text = node.text;
|
||||
if (node.marks?.length) {
|
||||
// A formula adopts the caret's marks for its font; neither file carries them.
|
||||
if (node.marks?.length && node.type !== 'formula') {
|
||||
out.marks = node.marks
|
||||
.map((m: N) => {
|
||||
const mm: N = { type: m.type };
|
||||
const attrs = Object.fromEntries(Object.entries(m.attrs ?? {})
|
||||
.filter(([k, v]) => v != null && MARK_DEFAULTS[k] !== v));
|
||||
.filter(([k, v]) => v != null && MARK_DEFAULTS[k] !== v)
|
||||
.map(([k, v]) => [k, typeof v === 'string' && /^#[0-9a-f]{6}$/i.test(v) ? v.toUpperCase() : v]));
|
||||
// ODF keeps the authored (naive local) comment/revision date, DOCX re-serializes
|
||||
// the same instant as UTC — compare the instant.
|
||||
if ((m.type === 'comment' || m.type === 'insertion' || m.type === 'deletion')
|
||||
|
||||
+2
-129
@@ -2,7 +2,6 @@
|
||||
// compares the resulting text layout (pages, lines, positions in mm).
|
||||
// Usage and prerequisites — including the mandatory font setup — are in README.md.
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { readFileSync, writeFileSync, copyFileSync, mkdirSync, mkdtempSync, readdirSync, rmSync, existsSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { tmpdir } from 'node:os';
|
||||
@@ -10,6 +9,7 @@ import { join, basename, extname, resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { chromium } from 'playwright-core';
|
||||
import { compare, toLines } from './compare.mjs';
|
||||
import { devServer, settle, extractLayout } from '../browser.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = resolve(HERE, '../..');
|
||||
@@ -91,135 +91,8 @@ async function editorRender(browser, file) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Pagination (and the columns flow) runs over several frames; wait for the page
|
||||
// count and document height to stop moving. An image still decoding has no height
|
||||
// yet and would settle the layout at a page count it leaves again a frame later.
|
||||
async function settle(page) {
|
||||
await page.waitForFunction(() => {
|
||||
const el = document.querySelector('.tiptap');
|
||||
// Importing a large file takes seconds; an editor still empty is not "settled",
|
||||
// it is the blank document the file has not replaced yet.
|
||||
if (!el || !el.textContent.trim()) return false;
|
||||
const imgs = Array.from(document.querySelectorAll('.paper img'));
|
||||
if (imgs.some((i) => !i.complete)) return false;
|
||||
// The spacers' heights too, not just their count: they go on settling for a while
|
||||
// after the last one is placed, and the page a line lands on moves with them.
|
||||
const spacers = Array.from(document.querySelectorAll('[data-page-break-spacer]'));
|
||||
// The index's page numbers settle after the spacers do — a stale one is a line of
|
||||
// text that still changes.
|
||||
const toc = Array.from(document.querySelectorAll('.toc-page')).map((t) => t.textContent).join(',');
|
||||
const key = el.style.minHeight + '|' + el.children.length + '|' + spacers.length + '|'
|
||||
+ spacers.reduce((sum, s) => sum + s.offsetHeight, 0) + '|' + toc;
|
||||
const w = window;
|
||||
if (w.__parityKey !== key) { w.__parityKey = key; w.__paritySince = performance.now(); return false; }
|
||||
return performance.now() - (w.__paritySince ?? 0) > 3000;
|
||||
// Polled, not per frame: the predicate walks every spacer and index row, and on a
|
||||
// 60-page document doing that each frame starves the layout it is waiting for.
|
||||
}, null, { timeout: 180_000, polling: 500 });
|
||||
}
|
||||
|
||||
// Runs in the browser: every rendered word with its page and mm position.
|
||||
function extractLayout() {
|
||||
const PAGE_GAP = 20, PX_MM = 25.4 / 96;
|
||||
const paper = document.querySelector('.paper');
|
||||
const cs = getComputedStyle(document.documentElement);
|
||||
const pageH = parseFloat(cs.getPropertyValue('--user-page-height'));
|
||||
const pageW = parseFloat(cs.getPropertyValue('--user-page-width'));
|
||||
const cycle = pageH + PAGE_GAP;
|
||||
const origin = paper.getBoundingClientRect();
|
||||
|
||||
const skip = (node) => {
|
||||
for (let e = node.parentElement; e && e !== paper; e = e.parentElement) {
|
||||
if (e.hasAttribute('data-page-break-spacer') || e.classList.contains('band-layer')) return true;
|
||||
const s = getComputedStyle(e);
|
||||
if (s.display === 'none' || s.visibility === 'hidden') return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
// A word broken mid-word — a token wider than its line, or one carrying a hyphen or a
|
||||
// slash — has a client rect per line, and pdftotext reads those as separate words. So
|
||||
// split it the same way, by the line each character lands on.
|
||||
const fragments = (node, from, to) => {
|
||||
const r = document.createRange();
|
||||
r.setStart(node, from); r.setEnd(node, to);
|
||||
const rects = r.getClientRects();
|
||||
const whole = (rect) => ({ top: rect.top, left: rect.left, width: rect.width, height: rect.height, text: node.nodeValue.slice(from, to) });
|
||||
if (rects.length < 2) return rects.length ? [whole(rects[0])] : [];
|
||||
const out = [];
|
||||
for (let i = from; i < to; i++) {
|
||||
const c = document.createRange();
|
||||
c.setStart(node, i); c.setEnd(node, i + 1);
|
||||
const rect = c.getClientRects()[0];
|
||||
if (!rect) continue;
|
||||
const last = out[out.length - 1];
|
||||
if (last && Math.abs(last.top - rect.top) < 1) { last.text += node.nodeValue[i]; last.right = rect.right; }
|
||||
else out.push({ top: rect.top, left: rect.left, right: rect.right, height: rect.height, text: node.nodeValue[i] });
|
||||
}
|
||||
return out.map((f) => ({ top: f.top, left: f.left, width: f.right - f.left, height: f.height, text: f.text }));
|
||||
};
|
||||
|
||||
// What the page paints, not what the node holds: a style's fo:text-transform /
|
||||
// w:caps is a CSS transform here and real uppercase in the reference's PDF.
|
||||
const painted = (text, el) => {
|
||||
const t = getComputedStyle(el).textTransform;
|
||||
if (t === 'uppercase') return text.toUpperCase();
|
||||
if (t === 'lowercase') return text.toLowerCase();
|
||||
if (t === 'capitalize') return text.replace(/^\p{L}/u, (c) => c.toUpperCase());
|
||||
return text;
|
||||
};
|
||||
|
||||
const words = [];
|
||||
const walker = document.createTreeWalker(paper, NodeFilter.SHOW_TEXT);
|
||||
for (let n = walker.nextNode(); n; n = walker.nextNode()) {
|
||||
const text = n.nodeValue;
|
||||
if (!text || !text.trim() || skip(n)) continue;
|
||||
for (const m of text.matchAll(/\S+/g)) {
|
||||
for (const f of fragments(n, m.index, m.index + m[0].length)) {
|
||||
if (!f.width) continue;
|
||||
const y = f.top - origin.top;
|
||||
const page = Math.floor(y / cycle);
|
||||
words.push({
|
||||
text: painted(f.text, n.parentElement),
|
||||
page,
|
||||
x: (f.left - origin.left) * PX_MM,
|
||||
y: (y - page * cycle) * PX_MM,
|
||||
w: f.width * PX_MM,
|
||||
h: f.height * PX_MM,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Folded, not spread: a several-hundred-page document has more words than a call
|
||||
// takes arguments, and Math.max(...words) then blows the stack instead of measuring.
|
||||
const numPages = words.reduce((m, w) => (w.page > m ? w.page : m), 0) + 1;
|
||||
const pages = Array.from({ length: numPages }, (_, i) => ({
|
||||
words: words.filter((w) => w.page === i).map(({ page, ...r }) => r),
|
||||
width: pageW * PX_MM,
|
||||
height: pageH * PX_MM,
|
||||
}));
|
||||
const mm = (v) => parseFloat(cs.getPropertyValue(v)) * PX_MM;
|
||||
return {
|
||||
pages,
|
||||
margins: { top: mm('--user-margin-top'), bottom: mm('--user-margin-bottom'),
|
||||
left: mm('--user-margin-left'), right: mm('--user-margin-right') },
|
||||
};
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------- driver
|
||||
|
||||
async function devServer() {
|
||||
const up = await fetch(`http://localhost:${PORT}/`).then(() => true).catch(() => false);
|
||||
if (up) return null;
|
||||
const proc = spawn('npm', ['run', 'dev', '--', '--port', String(PORT), '--strictPort'],
|
||||
{ cwd: ROOT, stdio: 'ignore', detached: true });
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
if (await fetch(`http://localhost:${PORT}/`).then(() => true).catch(() => false)) return proc;
|
||||
}
|
||||
throw new Error('dev server did not start');
|
||||
}
|
||||
|
||||
function corpus(args) {
|
||||
const paths = args.length ? args : [join(HERE, '..', 'corpus'), join(HERE, 'fixtures')];
|
||||
const out = [];
|
||||
@@ -256,7 +129,7 @@ if (args.includes('--quick')) {
|
||||
}
|
||||
if (!files.length) { console.error('no .docx/.odt files found'); process.exit(2); }
|
||||
|
||||
const server = await devServer();
|
||||
const server = await devServer(PORT);
|
||||
const browser = await chromium.launch({ executablePath: CHROME, args: ['--no-sandbox'] });
|
||||
const work = mkdtempSync(join(tmpdir(), 'parity-'));
|
||||
const report = [];
|
||||
|
||||
@@ -5,10 +5,11 @@ import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { strFromU8 } from 'fflate';
|
||||
|
||||
export const hasXmllint = spawnSync('xmllint', ['--version']).status === 0;
|
||||
const SCHEMAS = resolve(__dirname, 'schemas');
|
||||
const SCHEMAS = resolve(fileURLToPath(import.meta.url), '..', 'schemas');
|
||||
const MC_NS = 'http://schemas.openxmlformats.org/markup-compatibility/2006';
|
||||
// A DrawingML text box (wps:wsp inside a:graphicData) is post-ECMA-376 Microsoft
|
||||
// markup Word accepts natively; no ECMA schema covers it, so it is stripped like an
|
||||
|
||||
Reference in new issue
Block a user