From 6a313e56926cf53ac015fdc0b3daa29142eb853c Mon Sep 17 00:00:00 2001 From: Steffen Becker Date: Fri, 14 Aug 2026 08:17:06 +0200 Subject: [PATCH] feat: pictures are autosaved to IndexedDB, not localStorage A handful of photos is the whole ~5 MB quota, and past it the document stopped saving. The autosaved JSON now keeps an `idb:` per picture and the bytes live in their own store; the key is a sampled content hash, so a picture used twice is stored once and each save sweeps what the document no longer names. loadDocument is async as a result and Editor.svelte awaits it. Where IndexedDB is unavailable the data-URIs stay inline exactly as before. --- CHANGELOG.md | 2 +- src/lib/components/Editor.svelte | 6 +- src/lib/i18n/locales/de.ts | 2 + src/lib/i18n/locales/en.ts | 2 + src/lib/storage/CLAUDE.md | 2 +- src/lib/storage/autosave.ts | 13 +++- src/lib/storage/embeddedFontStore.ts | 26 ++----- src/lib/storage/idb.ts | 22 ++++++ src/lib/storage/imageStore.ts | 108 +++++++++++++++++++++++++++ tests/unit/autosave-recovery.test.ts | 25 ++++--- tests/unit/image-store.test.ts | 47 ++++++++++++ 11 files changed, 217 insertions(+), 38 deletions(-) create mode 100644 src/lib/storage/idb.ts create mode 100644 src/lib/storage/imageStore.ts create mode 100644 tests/unit/image-store.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f3cf9..9b5a92b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ - ODT export via odf-kit with extensive content.xml / styles.xml post-processing (custom node types, tables with borders, images, fonts and colors); filename derived from the first heading (fallback `document.odt`) - ODT import / open existing file: parses content.xml / styles.xml directly, adopts the file's margins and orientation, and reports graceful-degradation warnings - Opens `.ott` templates (OpenDocument Text Template): read like an `.odt`, but as a new untitled document — the first Save writes a fresh `.odt` and never overwrites the template (Word/LibreOffice behavior) +- Pictures are autosaved to IndexedDB, not localStorage: the document JSON keeps a short key per picture, so an image-heavy document no longer runs into the ~5 MB quota that used to stop it saving. Each save sweeps the pictures the document has dropped, a picture used twice is stored once, and where IndexedDB is unavailable everything stays inline as before - Embedded font loading: fonts embedded in an opened `.odt`/`.docx` (Word `.odttf` de-obfuscated) are registered via the FontFace API so text renders in its real face even when the font isn't installed; persisted per-document in IndexedDB so it survives a reload. Fonts the document only names but neither embeds nor installs are still flagged as substituted - List of figures and list of tables (References ▸ Table of contents ▸ the two caption indexes): the same generated index the table of contents is, over the captions of one category instead of the headings — single-level, with live page numbers and leader dots. Round-trips as ODF `text:illustration-index` / `text:table-index` (found by the counter's name, as LibreOffice finds them) and Word's `TOC \c "Figure"` field - Line numbering (Layout ▸ Line numbers, LibreOffice's Tools ▸ Line Numbering): numbers in the left margin, continuous or restarting each page, every nth line, with or without empty lines. Measured off the rendered lines, so the preview and the saved file agree; round-trips as ODF `text:linenumbering-configuration` and Word's `w:lnNumType` @@ -143,7 +144,6 @@ The gap against Word/LibreOffice, most valuable first. Reviewed 2026-08-08. - Section-level page **size and orientation**: only the margins are per section. A file whose sections disagree on the page size keeps the last one's - Table extras: sort, sum/formula in a cell, number recognition - A list level's own hanging indent: the marker sits at the 0.635 cm both exports write (`LIST_HANGING_CM`), so a wider *left*-set marker overflows where Word moves the text to the next list tab (a **right**-set one — `w:lvlJc`, which is what the built-in Roman numberings use — grows into the margin and is fine). Reading the value back naively also moves the markers of our own ODT exports — LibreOffice draws its own flat hanging at exactly the value odf-kit writes for level 2; see `tests/render-parity/README.md` before building on it -- Images live as data-URIs in the autosaved JSON, so an image-heavy document can exceed the ~5 MB localStorage quota; `storage/autosave.ts` warns once and a reload restores the last version that still fit. IndexedDB (as `embeddedFontStore.ts` already uses) would lift the ceiling - Linked / chained text frames - Find & Replace by formatting or by style (the regular-expression half is implemented) - Save as template (`.ott` / `.dotx`), multi-document management, recent-files list diff --git a/src/lib/components/Editor.svelte b/src/lib/components/Editor.svelte index 4e314ae..9e7d508 100644 --- a/src/lib/components/Editor.svelte +++ b/src/lib/components/Editor.svelte @@ -857,10 +857,12 @@ import { EMPTY_PAGE_DECOR, type PageDecor } from '../storage/pageDecor'; waitForSettled(); }); - onMount(() => { + onMount(async () => { // Start with empty history lists — loaded content is not an undoable edit. resetHistoryLog(); - const saved = loadDocument(); + // Awaited: the document's pictures live in IndexedDB, and the editor is built + // from the whole document or the first pagination pass measures the wrong one. + const saved = await loadDocument(); editor = new Editor({ element, diff --git a/src/lib/i18n/locales/de.ts b/src/lib/i18n/locales/de.ts index 007f4da..27d0685 100644 --- a/src/lib/i18n/locales/de.ts +++ b/src/lib/i18n/locales/de.ts @@ -777,6 +777,8 @@ const de: Messages = { 'Das Dokument ist zu groß, um automatisch gespeichert zu werden (Speicherlimit des Browsers erreicht). Das passiert meist bei großen eingebetteten Bildern. Speichere es als .odt-Datei, um deine Arbeit zu sichern.', documentNotLoaded: 'Das zuletzt bearbeitete Dokument hat den Start des Editors blockiert und wurde deshalb nicht geladen. Du arbeitest in einem neuen, leeren Dokument; das alte bleibt im Browser-Speicher unter „edentext-doc-broken" erhalten.', + picturesNotRestored: (n) => + `${n} ${n === 1 ? 'Bild konnte' : 'Bilder konnten'} nicht aus dem Browser-Speicher wiederhergestellt werden und ${n === 1 ? 'wird' : 'werden'} als defekt angezeigt. Öffne die gespeicherte Datei des Dokuments, um ${n === 1 ? 'es' : 'sie'} zurückzuholen.`, }, importWarn: { imagesSkipped: 'Einige Bilder konnten nicht gelesen werden und wurden übersprungen', diff --git a/src/lib/i18n/locales/en.ts b/src/lib/i18n/locales/en.ts index 5da97f0..72a1380 100644 --- a/src/lib/i18n/locales/en.ts +++ b/src/lib/i18n/locales/en.ts @@ -781,6 +781,8 @@ const en = { 'The document is too large to save automatically (browser storage limit reached). This usually happens with large embedded images. Save it as an .odt file to keep your work.', documentNotLoaded: 'The last document stopped the editor from starting, so it was not loaded. You are working in a new, empty document; the old one is kept in the browser storage under "edentext-doc-broken".', + picturesNotRestored: (n: number) => + `${n} picture${n === 1 ? '' : 's'} could not be restored from the browser storage and ${n === 1 ? 'is' : 'are'} shown as broken. Open the document's saved file to get ${n === 1 ? 'it' : 'them'} back.`, }, // Values mirror the canonical English messages the import parsers emit; the UI // maps a parser message to its localized form at display time (importMessages.ts). diff --git a/src/lib/storage/CLAUDE.md b/src/lib/storage/CLAUDE.md index c29dc80..5a400fe 100644 --- a/src/lib/storage/CLAUDE.md +++ b/src/lib/storage/CLAUDE.md @@ -24,7 +24,7 @@ Both margins and orientation are passed into the ODT export so the exported docu All keys share the `edentext-` prefix — a namespace, not a description: on GitHub Pages every repo of an account shares one origin, so a generic prefix would collide. -- **Document:** `edentext-doc` — TipTap JSON, debounced 1 s on every `onUpdate` (`autosave.ts`). Loading it raises `edentext-doc-loading`, which only a completed startup clears (`markDocumentLoaded`, two frames after `Editor.svelte`'s mount). Still set at the next load = that document hung or crashed the editor: it moves to `edentext-doc-broken` and the app starts empty, so a reload escapes the freeze instead of repeating it. +- **Document:** `edentext-doc` — TipTap JSON, debounced 1 s on every `onUpdate` (`autosave.ts`). Loading it raises `edentext-doc-loading`, which only a completed startup clears (`markDocumentLoaded`, two frames after `Editor.svelte`'s mount). Still set at the next load = that document hung or crashed the editor: it moves to `edentext-doc-broken` and the app starts empty, so a reload escapes the freeze instead of repeating it. **Pictures do not live in it:** `imageStore.ts` swaps every data-URI past 4 KB for an `idb:` before the write and keeps the bytes in the `edentext-images` IndexedDB store (a handful of photos is the whole localStorage quota). The key is a content hash of the data-URI's head, tail and length — sampled, because this runs on every autosave — so a picture used twice is stored once, and each save sweeps the keys the document no longer names. Where IndexedDB is unavailable the data-URIs stay inline, exactly as before; a key the store has lost keeps its `idb:` src and is reported, rather than quietly dropping the reader's content. `loadDocument` is therefore **async**, and `Editor.svelte` awaits it before building the editor. - **Theme:** `edentext-theme` — `'light' | 'dark' | 'allBlack' | 'auto'`. - **Toolbar expanded:** `edentext-toolbar-expanded` — boolean string. - **Chrome:** `edentext-chrome` — `'classic' | 'ribbon'`; absent = classic. diff --git a/src/lib/storage/autosave.ts b/src/lib/storage/autosave.ts index 2ebe017..b613bfd 100644 --- a/src/lib/storage/autosave.ts +++ b/src/lib/storage/autosave.ts @@ -1,4 +1,5 @@ import { t } from '../i18n/i18n.svelte'; +import { stashImages, putImages, restoreImages } from './imageStore'; const STORAGE_KEY = 'edentext-doc'; // Set while a stored document is being handed to the editor, cleared once the editor @@ -18,9 +19,13 @@ let quotaWarned = false; export function saveDocument(json: object): void { if (timeout) clearTimeout(timeout); - timeout = setTimeout(() => { + timeout = setTimeout(async () => { + // Pictures go to IndexedDB and the JSON keeps a key; where that fails they stay + // inline, which is the only thing localStorage ever held. + const { json: slim, blobs } = stashImages(json); + const stashed = await putImages(blobs); try { - localStorage.setItem(STORAGE_KEY, JSON.stringify(json)); + localStorage.setItem(STORAGE_KEY, JSON.stringify(stashed ? slim : json)); } catch (err) { console.error('[autosave] Could not save the document:', err); if (!quotaWarned) { @@ -31,7 +36,7 @@ export function saveDocument(json: object): void { }, DEBOUNCE_MS); } -export function loadDocument(): object | null { +export async function loadDocument(): Promise { const raw = localStorage.getItem(STORAGE_KEY); if (localStorage.getItem(BOOT_KEY)) { localStorage.removeItem(BOOT_KEY); @@ -51,6 +56,8 @@ export function loadDocument(): object | null { return null; } localStorage.setItem(BOOT_KEY, '1'); + const missing = await restoreImages(doc); + if (missing) requestAnimationFrame(() => alert(t().dialogs.picturesNotRestored(missing))); return doc; } diff --git a/src/lib/storage/embeddedFontStore.ts b/src/lib/storage/embeddedFontStore.ts index 661e950..18e5f14 100644 --- a/src/lib/storage/embeddedFontStore.ts +++ b/src/lib/storage/embeddedFontStore.ts @@ -1,4 +1,5 @@ import type { EmbeddedFont } from '../fonts/embeddedFonts'; +import { openDb, idbRequest } from './idb'; // The current document's embedded fonts, persisted in IndexedDB (not localStorage, whose // ~5 MB quota is already tight for images). One record, replaced on each open, so storage @@ -7,27 +8,14 @@ const DB_NAME = 'edentext-fonts'; const STORE = 'fonts'; const KEY = 'current'; -function openDb(): Promise { - return new Promise((resolve, reject) => { - const req = indexedDB.open(DB_NAME, 1); - req.onupgradeneeded = () => req.result.createObjectStore(STORE); - req.onsuccess = () => resolve(req.result); - req.onerror = () => reject(req.error); - }); -} - -function request(db: IDBDatabase, mode: IDBTransactionMode, run: (s: IDBObjectStore) => IDBRequest): Promise { - return new Promise((resolve, reject) => { - const req = run(db.transaction(STORE, mode).objectStore(STORE)); - req.onsuccess = () => resolve(req.result as T); - req.onerror = () => reject(req.error); - }); -} +const db_ = () => openDb(DB_NAME, STORE); +const request = (db: IDBDatabase, mode: IDBTransactionMode, run: (s: IDBObjectStore) => IDBRequest) => + idbRequest(db, STORE, mode, run); export async function saveEmbeddedFonts(fonts: EmbeddedFont[]): Promise { if (typeof indexedDB === 'undefined') return; try { - const db = await openDb(); + const db = await db_(); if (fonts.length) await request(db, 'readwrite', (s) => s.put(fonts, KEY)); else await request(db, 'readwrite', (s) => s.delete(KEY)); db.close(); @@ -37,7 +25,7 @@ export async function saveEmbeddedFonts(fonts: EmbeddedFont[]): Promise { export async function loadEmbeddedFonts(): Promise { if (typeof indexedDB === 'undefined') return []; try { - const db = await openDb(); + const db = await db_(); const fonts = await request(db, 'readonly', (s) => s.get(KEY)); db.close(); return fonts ?? []; @@ -47,7 +35,7 @@ export async function loadEmbeddedFonts(): Promise { export async function clearEmbeddedFontStore(): Promise { if (typeof indexedDB === 'undefined') return; try { - const db = await openDb(); + const db = await db_(); await request(db, 'readwrite', (s) => s.delete(KEY)); db.close(); } catch { /* nothing to clear */ } diff --git a/src/lib/storage/idb.ts b/src/lib/storage/idb.ts new file mode 100644 index 0000000..1e43d1e --- /dev/null +++ b/src/lib/storage/idb.ts @@ -0,0 +1,22 @@ +// The two-call IndexedDB wrapper both stores here need: one object store per database, +// opened per operation and closed again. No versioning beyond the first schema — a +// store that ever needs a second one gets its own database name instead. + +export function openDb(name: string, store: string): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(name, 1); + req.onupgradeneeded = () => req.result.createObjectStore(store); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); +} + +export function idbRequest( + db: IDBDatabase, store: string, mode: IDBTransactionMode, run: (s: IDBObjectStore) => IDBRequest, +): Promise { + return new Promise((resolve, reject) => { + const req = run(db.transaction(store, mode).objectStore(store)); + req.onsuccess = () => resolve(req.result as T); + req.onerror = () => reject(req.error); + }); +} diff --git a/src/lib/storage/imageStore.ts b/src/lib/storage/imageStore.ts new file mode 100644 index 0000000..26e6f4a --- /dev/null +++ b/src/lib/storage/imageStore.ts @@ -0,0 +1,108 @@ +import { openDb, idbRequest } from './idb'; + +// The document's pictures, out of the autosaved JSON and into IndexedDB. localStorage's +// ~5 MB quota is a handful of photos, and a document that exceeds it stops saving; the +// JSON keeps a short key in each `src` instead, and only that is small enough to fit. +// Where IndexedDB is unavailable the data-URIs stay inline, exactly as they were. +const DB_NAME = 'edentext-images'; +const STORE = 'images'; +export const IDB_SRC = 'idb:'; + +// Below this a picture is an icon, and one round trip through IndexedDB costs more +// than the bytes save. +const MIN_STASH_CHARS = 4096; + +// Content key: FNV-1a over the head, the tail and the length, not the whole data-URI — +// a photo is megabytes, and this runs on every autosave. Two different pictures +// agreeing on all three is not something a document produces. +function keyOf(src: string): string { + const sample = src.length <= 8192 ? src : src.slice(0, 4096) + src.slice(-4096); + let h = 0x811c9dc5; + for (let i = 0; i < sample.length; i++) { + h ^= sample.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + return `${(h >>> 0).toString(36)}${src.length.toString(36)}`; +} + +type Json = { attrs?: Record; content?: Json[] } & Record; + +// Every node with a `src`, in document order. Images ride paragraphs, cells and text +// boxes alike, so the walk is over the whole tree rather than a known depth. +function srcNodes(node: Json, out: Json[] = []): Json[] { + if (typeof node?.attrs?.src === 'string') out.push(node); + for (const child of node?.content ?? []) srcNodes(child, out); + return out; +} + +/** + * The document with its large pictures replaced by `idb:` keys, plus those pictures. + * A structural copy: only the nodes on the path to a picture are rebuilt, so the + * caller's JSON is untouched. + */ +export function stashImages(json: object): { json: object; blobs: Map } { + const blobs = new Map(); + const walk = (node: Json): Json => { + const src = node?.attrs?.src; + const content = node?.content?.map(walk); + let attrs = node?.attrs; + if (typeof src === 'string' && src.startsWith('data:') && src.length >= MIN_STASH_CHARS) { + const key = keyOf(src); + blobs.set(key, src); + attrs = { ...attrs, src: IDB_SRC + key }; + } + return content || attrs !== node?.attrs ? { ...node, ...(attrs ? { attrs } : {}), ...(content ? { content } : {}) } : node; + }; + return { json: walk(json as Json), blobs }; +} + +// Whether this session has anything in the store. Without it a document whose last +// picture was just deleted would skip the sweep below and leave its bytes behind. +let inUse = false; + +/** Write the pictures and drop every key the document no longer references. */ +export async function putImages(blobs: Map): Promise { + if (!blobs.size && !inUse) return true; + if (typeof indexedDB === 'undefined') return false; + inUse = blobs.size > 0; + try { + const db = await openDb(DB_NAME, STORE); + const known = await idbRequest(db, STORE, 'readonly', (s) => s.getAllKeys()); + for (const [key, src] of blobs) { + if (!known.includes(key)) await idbRequest(db, STORE, 'readwrite', (s) => s.put(src, key)); + } + for (const key of known) { + if (!blobs.has(String(key))) await idbRequest(db, STORE, 'readwrite', (s) => s.delete(key)); + } + db.close(); + return true; + } catch { + return false; + } +} + +/** + * Put the pictures back. Returns how many keys the store no longer had — their nodes + * keep the key as their `src`, which renders as a broken picture rather than quietly + * dropping the reader's content. + */ +export async function restoreImages(json: object): Promise { + const nodes = srcNodes(json as Json).filter((n) => String(n.attrs!.src).startsWith(IDB_SRC)); + if (!nodes.length) return 0; + inUse = true; + if (typeof indexedDB === 'undefined') return nodes.length; + let missing = 0; + try { + const db = await openDb(DB_NAME, STORE); + for (const node of nodes) { + const key = String(node.attrs!.src).slice(IDB_SRC.length); + const src = await idbRequest(db, STORE, 'readonly', (s) => s.get(key)); + if (src) node.attrs!.src = src; + else missing++; + } + db.close(); + } catch { + return nodes.length; + } + return missing; +} diff --git a/tests/unit/autosave-recovery.test.ts b/tests/unit/autosave-recovery.test.ts index 510ab55..ebe3d5f 100644 --- a/tests/unit/autosave-recovery.test.ts +++ b/tests/unit/autosave-recovery.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { loadDocument, markDocumentLoaded, saveDocument } from '../../src/lib/storage/autosave'; // A document that freezes the editor would be reloaded from localStorage forever, so -// loading raises a boot flag that only a completed startup clears. +// loading raises a boot flag that only a completed startup clears. Loading is async +// because the document's pictures come back out of IndexedDB (imageStore.ts). describe('autosave crash recovery', () => { beforeEach(() => { localStorage.clear(); @@ -10,29 +11,29 @@ describe('autosave crash recovery', () => { vi.stubGlobal('alert', () => {}); }); - it('loads the document again after a startup that completed', () => { + it('loads the document again after a startup that completed', async () => { localStorage.setItem('edentext-doc', '{"type":"doc"}'); - expect(loadDocument()).toEqual({ type: 'doc' }); + expect(await loadDocument()).toEqual({ type: 'doc' }); markDocumentLoaded(); - expect(loadDocument()).toEqual({ type: 'doc' }); + expect(await loadDocument()).toEqual({ type: 'doc' }); }); - it('parks the document and starts empty when the last startup never finished', () => { + it('parks the document and starts empty when the last startup never finished', async () => { localStorage.setItem('edentext-doc', '{"type":"doc"}'); - loadDocument(); // raises the boot flag, then the "editor hangs" - expect(loadDocument()).toBeNull(); + await loadDocument(); // raises the boot flag, then the "editor hangs" + expect(await loadDocument()).toBeNull(); expect(localStorage.getItem('edentext-doc-broken')).toBe('{"type":"doc"}'); expect(localStorage.getItem('edentext-doc')).toBeNull(); }); - it('keeps the next document after a recovery', () => { + it('keeps the next document after a recovery', async () => { localStorage.setItem('edentext-doc', '{"type":"doc","content":[]}'); - loadDocument(); - expect(loadDocument()).toBeNull(); // recovered: boot flag cleared, doc parked + await loadDocument(); + expect(await loadDocument()).toBeNull(); // recovered: boot flag cleared, doc parked vi.useFakeTimers(); saveDocument({ type: 'doc' }); - vi.runAllTimers(); + await vi.runAllTimersAsync(); vi.useRealTimers(); - expect(loadDocument()).toEqual({ type: 'doc' }); + expect(await loadDocument()).toEqual({ type: 'doc' }); }); }); diff --git a/tests/unit/image-store.test.ts b/tests/unit/image-store.test.ts new file mode 100644 index 0000000..2f6e8a6 --- /dev/null +++ b/tests/unit/image-store.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from 'vitest'; +import { stashImages, IDB_SRC } from '../../src/lib/storage/imageStore'; + +const big = (fill: string) => `data:image/png;base64,${fill.repeat(5000)}`; + +const doc = (...content: any[]) => ({ type: 'doc', content }); +const img = (src: string) => ({ type: 'image', attrs: { src, width: 100 } }); +const para = (...content: any[]) => ({ type: 'paragraph', content }); + +describe('stashImages', () => { + it('replaces a large picture and keeps its other attrs', () => { + const src = big('A'); + const json: any = doc(para(img(src))); + const { json: slim, blobs } = stashImages(json); + const out = (slim as any).content[0].content[0]; + expect(out.attrs.src.startsWith(IDB_SRC)).toBe(true); + expect(out.attrs.width).toBe(100); + expect([...blobs.values()]).toEqual([src]); + // The caller's JSON is what the editor is still holding. + expect(json.content[0].content[0].attrs.src).toBe(src); + }); + + it('leaves an icon and a non-data src inline', () => { + const { json: slim, blobs } = stashImages(doc(para(img('data:image/png;base64,AAAA'), img('https://x/y.png')))); + expect(blobs.size).toBe(0); + expect((slim as any).content[0].content[0].attrs.src).toBe('data:image/png;base64,AAAA'); + expect((slim as any).content[0].content[1].attrs.src).toBe('https://x/y.png'); + }); + + it('reaches a picture at any depth and dedupes by content', () => { + const src = big('B'); + const cell = (...c: any[]) => ({ type: 'tableCell', content: c }); + const { json: slim, blobs } = stashImages(doc( + { type: 'table', content: [{ type: 'tableRow', content: [cell(para(img(src)))] }] }, + { type: 'textBox', content: [para(img(src))] }, + )); + expect(blobs.size).toBe(1); + const keys = JSON.stringify(slim).match(new RegExp(`${IDB_SRC}[a-z0-9]+`, 'g')); + expect(keys?.length).toBe(2); + expect(new Set(keys).size).toBe(1); + }); + + it('gives two different pictures two keys', () => { + const { blobs } = stashImages(doc(para(img(big('C')), img(big('D'))))); + expect(blobs.size).toBe(2); + }); +});