diff --git a/docs/integrations/document-import.md b/docs/integrations/document-import.md index 793ab2acb..595bc964c 100644 --- a/docs/integrations/document-import.md +++ b/docs/integrations/document-import.md @@ -60,7 +60,8 @@ change the type in HealthLog. Tagging a document again, or editing it in Paperless, fires the workflow again. That is harmless: HealthLog recognises the Paperless document id and -answers "already stored". If you delete the document in HealthLog, a later +answers "already stored", though each re-send still counts towards the +token's hourly limit. If you delete the document in HealthLog, a later re-send does not bring it back. **What the workflow is not for.** Paperless gives up on a webhook after @@ -150,7 +151,8 @@ What it does: - When HealthLog asks it to slow down, it waits as long as HealthLog says and carries on. By default a document token may upload 120 documents an hour, so an archive of a thousand documents takes an evening. Documents - HealthLog already has do not count towards that. Uploads you make yourself + HealthLog already has do not count towards that, because the script asks + before sending them. Uploads you make yourself in the web app or on your phone are counted separately and are not held up. - At the end it prints how many documents were imported, how many were @@ -169,11 +171,11 @@ thousand AI requests at once. Imported documents still get a preview and are searchable by their text where the file has a text layer. HealthLog remembers that these documents were held back: turning on -automatic AI reading later does not send them to your AI provider either. -To have documents read, open one and choose **Read with AI** or -**Generate summary**, or use **Index all for search** in Documents for the -ones that had no text to search. Pass `--ai-read` if you do want every -imported document read as it arrives. +automatic AI reading later does not send them to your AI provider, and +**Index all for search** in Documents leaves them out too. To have a +document read, open it and choose **Read with AI** or **Generate summary**. +Pass `--ai-read` if you do want every imported document read as it +arrives. The Paperless workflow does not hold AI reading back: new documents are treated like any other upload. @@ -213,6 +215,11 @@ per account: if you push documents from two other systems, make sure their ids cannot collide, for example by prefixing them (`nextcloud-123`, `scanner-123`). +When the same file arrives under a second id (the same scan in the source +twice, or a file you had uploaded by hand), HealthLog keeps one copy and +remembers the extra id for it. One document takes up to 20 such ids; a +21st is refused with a message, and nothing is stored. + ## For operators `DOCUMENT_UPLOAD_LIMIT_PER_HOUR` sets how many documents one document token diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts index c547945be..4117e1048 100644 --- a/scripts/__tests__/import-documents.test.ts +++ b/scripts/__tests__/import-documents.test.ts @@ -90,6 +90,8 @@ async function healthlog( noLookup?: boolean; /** The address redirects elsewhere (http → https, a moved host). */ redirect?: string; + /** Lookups are spent: every lookup answers 429 with a long Retry-After. */ + lookup429?: boolean; } = {}, ) { const stored = new Set(); @@ -110,6 +112,13 @@ async function healthlog( !opts.noLookup ) { lookups.push(queryKey); + if (opts.lookup429) { + return { + status: 429, + headers: { "retry-after": "3600" }, + json: { data: null, error: "Too many lookups." }, + }; + } const deleted = opts.deleted?.includes(queryKey) ?? false; const known = deleted || stored.has(queryKey); return { @@ -158,9 +167,17 @@ async function healthlog( size = v.size; } } - // The key rides the query string, where HealthLog reads it first. - fields.sourceSystem = url.searchParams.get("sourceSystem") ?? ""; - fields.sourceId = url.searchParams.get("sourceId") ?? ""; + // The key rides the query string, where HealthLog reads it first, and + // the form; HealthLog refuses the two disagreeing. + if ( + fields.sourceSystem !== url.searchParams.get("sourceSystem") || + fields.sourceId !== url.searchParams.get("sourceId") + ) { + return { + status: 422, + json: { data: null, error: "source key mismatch" }, + }; + } received.push({ auth: req.headers.authorization, fields, filename, size }); const key = queryKey; if (opts.tooLarge?.includes(key)) { @@ -546,6 +563,28 @@ describe("import-documents.mjs — edges", () => { expect(res.stdout).toMatch(/Imported: 2/); }); + it("does not wait on spent lookups: the keyed upload answers instead", async () => { + const hl = await healthlog({ lookup429: true }); + const pl = await paperless(); + const started = Date.now(); + const res = await runScript( + [ + "paperless", + "--paperless-url", + pl.url, + "--healthlog-url", + hl.url, + "--tag", + "HealthLog", + ], + env, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch(/Imported: 2/); + // A Retry-After of an hour was not honoured for a lookup. + expect(Date.now() - started).toBeLessThan(20_000); + }); + it("stops on a redirecting HealthLog address and names where it points", async () => { const hl = await healthlog({ redirect: "https://health.example" }); const pl = await paperless(); diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs index 06f679248..307247b35 100644 --- a/scripts/import-documents.mjs +++ b/scripts/import-documents.mjs @@ -201,7 +201,7 @@ function retryAfterSeconds(response, fallback) { * service pacing us), a 5xx or a dropped connection is retried a few times * with a growing pause, and anything else is returned to the caller. */ -async function request(label, url, init, makeBody) { +async function request(label, url, init, makeBody, { waitOn429 = true } = {}) { let transient = 0; for (;;) { let response; @@ -218,7 +218,7 @@ async function request(label, url, init, makeBody) { await wait(TRANSIENT_BACKOFF_MS[transient++], `${label} did not answer`); continue; } - if (response.status === 429) { + if (response.status === 429 && waitOn429) { await response.arrayBuffer().catch(() => {}); const seconds = retryAfterSeconds(response, 60); await wait(seconds * 1000, `${label} asked to slow down`); @@ -569,12 +569,20 @@ function stopOnRefusal(response, body, url) { */ async function lookup(options, source, doc) { const url = `${options.healthlogUrl}/api/documents/inbound/source?${keyQuery(source, doc)}`; - const response = await request("HealthLog", url, { - headers: { - Authorization: `Bearer ${options.healthlogToken}`, - Accept: "application/json", + // Not worth waiting for: if the lookups are spent, the upload with the key + // answers the same question. + const response = await request( + "HealthLog", + url, + { + headers: { + Authorization: `Bearer ${options.healthlogToken}`, + Accept: "application/json", + }, }, - }); + undefined, + { waitOn429: false }, + ); const body = await readBody(response); stopOnRefusal(response, body, url); if (!response.ok || !body?.data?.known) return null; @@ -603,6 +611,9 @@ async function upload(options, source, doc, file) { if (doc.date) form.append("documentDate", doc.date); if (kind) form.append("kind", kind); if (!options.aiRead) form.append("aiRead", "defer"); + // In the form as well as the address; HealthLog checks they agree. + form.append("sourceSystem", source.system); + form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX)); return form; }, ); @@ -789,8 +800,7 @@ function printSummary(options, tally, listed, totalBytes, unknownSizes) { if (!options.dryRun && tally.imported > 0 && !options.aiRead) { out( `${tally.imported} document(s) imported without automatic AI reading. ` + - "To have one read, open it in HealthLog and choose Read with AI or Generate summary; " + - "Documents > Index all for search reads the ones that had no text to search.", + "To have one read, open it in HealthLog and choose Read with AI or Generate summary.", ); } }