From 0257f2207aae9c8bd452babaa3ccebaf4887aee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 25 Sep 2026 21:48:12 +0200 Subject: [PATCH] feat(scripts): import documents from Paperless-ngx and Papra A single dependency-free Node 22 script that copies tagged documents from Paperless-ngx (REST, original files) or Papra (API key, /file) into the vault through a document token. Serial, one file in memory, waits on 429 Retry-After, and resumable through the server's source-key dedup. Supports --since, --kind / --kind-map, --dry-run with sizes, and holds back automatic AI reading unless --ai-read is given. Refs #1038 --- scripts/__tests__/import-documents.test.ts | 531 +++++++++++++++ scripts/import-documents.mjs | 723 +++++++++++++++++++++ 2 files changed, 1254 insertions(+) create mode 100644 scripts/__tests__/import-documents.test.ts create mode 100644 scripts/import-documents.mjs diff --git a/scripts/__tests__/import-documents.test.ts b/scripts/__tests__/import-documents.test.ts new file mode 100644 index 000000000..4a8af6d45 --- /dev/null +++ b/scripts/__tests__/import-documents.test.ts @@ -0,0 +1,531 @@ +/** + * `scripts/import-documents.mjs` against recorded-shape fixtures of the two + * source APIs and a stand-in HealthLog, all on local HTTP servers. + * + * The fixtures are the shapes the script depends on (Paperless-ngx API v10 + * pagination, tag and document-type lookups, `?original=true` downloads; + * Papra's organization-scoped document list, tags and `/file`), so a change + * on either side shows up here as a fixture to refresh rather than as a user + * report. The HealthLog stand-in answers the upload contract the real route + * speaks: 201, 200 duplicate, 200 deleted, 413, 429 with Retry-After, 403 + * `module.disabled`. + */ +import { spawn } from "node:child_process"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const SCRIPT = join(process.cwd(), "scripts/import-documents.mjs"); +const PDF = Buffer.from("%PDF-1.7\n%fixture\n%%EOF\n"); + +const servers: Server[] = []; +afterEach(async () => { + await Promise.all( + servers.splice(0).map((s) => new Promise((r) => s.close(() => r(null)))), + ); +}); + +type Handler = ( + req: IncomingMessage, + url: URL, + body: Buffer, +) => Promise<{ + status: number; + json?: unknown; + bytes?: Buffer; + headers?: Record; +}>; + +async function serve(handler: Handler): Promise { + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const c of req) chunks.push(c as Buffer); + const url = new URL(req.url ?? "/", "http://localhost"); + const r = await handler(req, url, Buffer.concat(chunks)); + const headers: Record = { ...(r.headers ?? {}) }; + if (r.json !== undefined) headers["content-type"] = "application/json"; + res.writeHead(r.status, headers); + res.end(r.json !== undefined ? JSON.stringify(r.json) : (r.bytes ?? "")); + }); + servers.push(server); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +function runScript( + args: string[], + env: Record, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(process.execPath, [SCRIPT, ...args], { + env: { PATH: process.env.PATH ?? "", ...env }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => (stdout += d)); + child.stderr.on("data", (d) => (stderr += d)); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +interface Received { + auth: string | undefined; + fields: Record; + filename: string | null; + size: number; +} + +/** + * A HealthLog stand-in. Keys stored documents by source key the way the real + * route does, answers the first upload with a 429, and refuses keys listed as + * deleted or too large. + */ +async function healthlog( + opts: { + deleted?: string[]; + tooLarge?: string[]; + moduleOff?: boolean; + } = {}, +) { + const stored = new Set(); + const received: Received[] = []; + let throttled = false; + const url = await serve(async (req, url, body) => { + if (url.pathname !== "/api/documents/inbound" || req.method !== "POST") { + return { status: 404, json: { data: null, error: "Not found" } }; + } + if (opts.moduleOff) { + return { + status: 403, + json: { + data: null, + error: 'Module "inboundDocuments" is not enabled', + meta: { errorCode: "module.disabled" }, + }, + }; + } + if (!throttled) { + throttled = true; + return { + status: 429, + headers: { "retry-after": "0" }, + json: { + data: null, + error: "Too many uploads.", + meta: { errorCode: "documents.inbound.rateLimited" }, + }, + }; + } + const form = await new Response(body, { + headers: { "content-type": req.headers["content-type"] ?? "" }, + }).formData(); + const fields: Record = {}; + let filename: string | null = null; + let size = 0; + for (const [k, v] of form.entries()) { + if (typeof v === "string") fields[k] = v; + else { + filename = v.name; + size = v.size; + } + } + received.push({ auth: req.headers.authorization, fields, filename, size }); + const key = `${fields.sourceSystem}:${fields.sourceId}`; + if (opts.tooLarge?.includes(key)) { + return { + status: 413, + json: { + data: null, + error: "File is too large.", + meta: { reason: "fileTooLarge", maxFileBytes: 26_214_400 }, + }, + }; + } + if (opts.deleted?.includes(key)) { + return { + status: 200, + json: { + data: { id: null, duplicate: true, deleted: true }, + error: null, + meta: { duplicate: true, deleted: true }, + }, + }; + } + if (stored.has(key)) { + return { + status: 200, + json: { + data: { id: key, duplicate: true }, + error: null, + meta: { duplicate: true }, + }, + }; + } + stored.add(key); + return { + status: 201, + json: { data: { id: key, duplicate: false }, error: null }, + }; + }); + return { url, received }; +} + +/** Paperless-ngx v10 stand-in: two pages of documents, one outside the tag. */ +async function paperless() { + const seen: { path: string; auth?: string; accept?: string }[] = []; + const url = await serve(async (req, url) => { + seen.push({ + path: url.pathname + url.search, + auth: req.headers.authorization, + accept: req.headers.accept, + }); + const base = `http://${req.headers.host}`; + if (url.pathname === "/api/tags/") { + return { + status: 200, + json: { + count: 1, + next: null, + results: [{ id: 7, name: "HealthLog" }], + }, + }; + } + if (url.pathname === "/api/document_types/") { + return { + status: 200, + json: { count: 1, next: null, results: [{ id: 3, name: "Labor" }] }, + }; + } + if (url.pathname === "/api/documents/") { + if (url.searchParams.get("page") === "2") { + return { + status: 200, + json: { + count: 3, + next: null, + results: [ + { + id: 3, + title: "Car insurance", + created: "2024-05-01", + added: "2024-05-02T10:00:00Z", + tags: [9], + document_type: null, + original_file_name: "car.pdf", + }, + ], + }, + }; + } + const next = new URL(url.toString(), base); + next.searchParams.set("page", "2"); + return { + status: 200, + json: { + count: 3, + // Deliberately another origin, as behind a reverse proxy. + next: `https://paperless.internal${next.pathname}${next.search}`, + results: [ + { + id: 1, + title: "Blutbild März", + created: "2024-03-05", + added: "2024-03-06T08:00:00Z", + tags: [7], + document_type: 3, + original_file_name: "blutbild.pdf", + }, + { + id: 2, + title: "Arztbrief", + created: null, + added: "2024-04-01T09:30:00Z", + tags: [7], + document_type: null, + original_file_name: "brief.pdf", + }, + ], + }, + }; + } + const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(url.pathname); + if (download) { + return { + status: 200, + bytes: PDF, + headers: { + "content-type": "application/pdf", + "content-disposition": `attachment; filename="doc-${download[1]}.pdf"`, + }, + }; + } + const metadata = /^\/api\/documents\/(\d+)\/metadata\/$/.exec(url.pathname); + if (metadata) { + return { status: 200, json: { original_size: 1_048_576 } }; + } + return { status: 404, json: { detail: "Not found." } }; + }); + return { url, seen }; +} + +/** Papra stand-in: one organization, a tag, three documents. */ +async function papra() { + const seen: { path: string; auth?: string }[] = []; + const url = await serve(async (req, url) => { + seen.push({ + path: url.pathname + url.search, + auth: req.headers.authorization, + }); + const org = "/api/organizations/org_1"; + if (url.pathname === `${org}/tags`) { + return { + status: 200, + json: { + tags: [ + { id: "tag_h", name: "Health" }, + { id: "tag_l", name: "Lab" }, + ], + }, + }; + } + if (url.pathname === `${org}/documents`) { + return { + status: 200, + json: { + documentsCount: 3, + documents: [ + { + id: "doc_a", + name: "Impfpass.pdf", + originalName: "impfpass.pdf", + originalSize: 2048, + createdAt: "2025-01-10T12:00:00.000Z", + documentDate: "2019-06-01", + tags: [{ id: "tag_h", name: "Health" }], + }, + { + id: "doc_b", + name: "Labor.pdf", + originalName: "labor.pdf", + originalSize: 4096, + createdAt: "2025-02-10T12:00:00.000Z", + tags: [ + { id: "tag_h", name: "Health" }, + { id: "tag_l", name: "Lab" }, + ], + }, + { + id: "doc_old", + name: "Old.pdf", + originalName: "old.pdf", + originalSize: 100, + createdAt: "2015-02-10T12:00:00.000Z", + documentDate: "2015-02-10", + tags: [{ id: "tag_h", name: "Health" }], + }, + ], + }, + }; + } + if ( + /^\/api\/organizations\/org_1\/documents\/[\w]+\/file$/.test(url.pathname) + ) { + return { + status: 200, + bytes: PDF, + headers: { "content-type": "application/pdf" }, + }; + } + return { status: 404, json: { error: "not found" } }; + }); + return { url, seen }; +} + +describe("import-documents.mjs — Paperless-ngx", () => { + it("imports the tagged documents with their title, date, kind and source key", async () => { + const hl = await healthlog(); + const pl = await paperless(); + const env = { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" }; + const args = [ + "paperless", + "--paperless-url", + pl.url, + "--healthlog-url", + hl.url, + "--tag", + "HealthLog", + "--kind-map", + "Labor=LAB_RESULT", + ]; + const first = await runScript(args, env); + + expect(first.stderr).toBe(""); + expect(first.code).toBe(0); + // The first upload met a 429 and was sent again, not dropped. + expect(hl.received).toHaveLength(2); + expect(hl.received[0]).toMatchObject({ + auth: "Bearer hlk_test", + filename: "blutbild.pdf", + size: PDF.byteLength, + fields: { + title: "Blutbild März", + documentDate: "2024-03-05", + kind: "LAB_RESULT", + sourceSystem: "PAPERLESS", + sourceId: "1", + aiRead: "defer", + }, + }); + // No created date: filed under the day it was added, and said so. + expect(hl.received[1].fields.documentDate).toBe("2024-04-01"); + expect(hl.received[1].fields.kind).toBeUndefined(); + expect(first.stdout).toMatch(/Imported: 2/); + expect(first.stdout).toMatch(/filed under the day it was added/); + expect(first.stdout).toMatch(/without automatic AI reading/); + + // The API contract the script leans on. + const docs = pl.seen.filter((s) => s.path.startsWith("/api/documents/?")); + expect(docs[0].path).toContain("tags__id__all=7"); + expect(docs[0].accept).toBe("application/json; version=10"); + expect(docs[0].auth).toBe("Token pl_test"); + // `next` was followed on the source's own origin. + expect(docs[1].path).toContain("page=2"); + expect( + pl.seen.filter((s) => s.path.includes("/download/")).map((s) => s.path), + ).toEqual([ + "/api/documents/1/download/?original=true", + "/api/documents/2/download/?original=true", + ]); + + // A second run changes nothing: HealthLog answers duplicate for both. + const second = await runScript(args, env); + expect(second.code).toBe(0); + expect(second.stdout).toMatch(/Imported: 0/); + expect(second.stdout).toMatch(/Already in HealthLog: 2/); + }); + + it("leaves a document deleted in HealthLog alone", async () => { + const hl = await healthlog({ deleted: ["PAPERLESS:2"] }); + const pl = await paperless(); + const res = await runScript( + [ + "paperless", + "--paperless-url", + pl.url, + "--healthlog-url", + hl.url, + "--tag", + "healthlog", + ], + { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch(/Imported: 1/); + expect(res.stdout).toMatch(/Deleted in HealthLog, left alone: 1/); + }); + + it("stops with a plain instruction when the vault is switched off", async () => { + const hl = await healthlog({ moduleOff: true }); + const pl = await paperless(); + const res = await runScript( + ["paperless", "--paperless-url", pl.url, "--healthlog-url", hl.url], + { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(1); + expect(res.stderr).toMatch(/Turn on Documents in Settings/); + }); + + it("lists sizes on a dry run and sends nothing", async () => { + const pl = await paperless(); + const res = await runScript( + [ + "paperless", + "--paperless-url", + pl.url, + "--tag", + "HealthLog", + "--dry-run", + ], + { PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(0); + expect(res.stdout).toMatch( + /2 document\(s\) would be imported, about 2\.0 MB/, + ); + expect(pl.seen.some((s) => s.path.includes("/download/"))).toBe(false); + }); + + it("refuses to start without a HealthLog token", async () => { + const res = await runScript( + [ + "paperless", + "--paperless-url", + "http://x", + "--healthlog-url", + "http://y", + ], + { PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(2); + expect(res.stderr).toMatch(/HEALTHLOG_TOKEN is not set/); + }); + + it("names a missing tag instead of importing everything", async () => { + const pl = await paperless(); + const res = await runScript( + ["paperless", "--paperless-url", pl.url, "--tag", "Nope", "--dry-run"], + { PAPERLESS_TOKEN: "pl_test" }, + ); + expect(res.code).toBe(1); + }); +}); + +describe("import-documents.mjs — Papra", () => { + it("imports by tag since a date, and skips what HealthLog refuses as too large", async () => { + const hl = await healthlog({ tooLarge: ["PAPRA:doc_b"] }); + const pp = await papra(); + const res = await runScript( + [ + "papra", + "--papra-url", + pp.url, + "--papra-org", + "org_1", + "--healthlog-url", + hl.url, + "--tag", + "Health", + "--since", + "2016-01-01", + "--kind-map", + "Lab=LAB_RESULT", + "--ai-read", + ], + { HEALTHLOG_TOKEN: "hlk_test", PAPRA_TOKEN: "pp_test" }, + ); + + // Exit 3: finished, with at least one document skipped. + expect(res.code).toBe(3); + const sent = hl.received.map((r) => r.fields); + expect(sent).toHaveLength(2); + expect(sent[0]).toMatchObject({ + title: "Impfpass", + documentDate: "2019-06-01", + sourceSystem: "PAPRA", + sourceId: "doc_a", + }); + // --ai-read: no deferral field at all. + expect(sent[0].aiRead).toBeUndefined(); + expect(sent[1]).toMatchObject({ sourceId: "doc_b", kind: "LAB_RESULT" }); + // doc_old is before --since and never downloaded. + expect(pp.seen.some((s) => s.path.includes("doc_old"))).toBe(false); + expect(pp.seen.find((s) => s.path.includes("/documents?"))?.path).toContain( + "searchQuery=tag%3AHealth", + ); + expect(pp.seen[0].auth).toBe("Bearer pp_test"); + expect(res.stdout).toMatch(/Imported: 1/); + expect(res.stdout).toMatch(/Skipped: 1/); + expect(res.stdout).toMatch(/larger than HealthLog accepts/); + }); +}); diff --git a/scripts/import-documents.mjs b/scripts/import-documents.mjs new file mode 100644 index 000000000..305c60b4a --- /dev/null +++ b/scripts/import-documents.mjs @@ -0,0 +1,723 @@ +#!/usr/bin/env node +/** + * Copy documents from Paperless-ngx or Papra into the HealthLog document vault. + * + * Runs wherever you are, not inside the HealthLog container: it talks to two + * HTTP APIs and needs nothing but Node 22 (no dependencies, no install). + * + * node import-documents.mjs paperless --paperless-url https://paperless.lan \ + * --healthlog-url https://health.example --tag HealthLog + * + * node import-documents.mjs papra --papra-url https://papra.lan \ + * --papra-org --healthlog-url https://health.example \ + * --tag Health + * + * Credentials come from the environment, never from flags (flags end up in + * shell history): + * + * HEALTHLOG_TOKEN a document token from HealthLog Settings > API & Tokens + * PAPERLESS_TOKEN a Paperless-ngx API token (My Profile) + * PAPRA_TOKEN a Papra API key with documents:read and tags:read + * + * Options: + * --tag only documents carrying this tag (recommended) + * --since YYYY-MM-DD only documents dated on or after this day + * --kind document type for everything imported + * --kind-map "A=KIND,B=KIND" + * type by Paperless document type or Papra tag name; + * anything unmatched gets --kind, or HealthLog's OTHER + * --dry-run list what would be imported, with sizes; send nothing + * --ai-read let HealthLog read the documents with AI as they + * arrive (by default automatic reading is held back) + * --help this text + * + * One document at a time, one file in memory at a time. A document that is + * already in HealthLog, or that you deleted there, is skipped by HealthLog + * itself, so running the script again is safe and simply picks up where it + * stopped. HealthLog's rate limit answers 429 with Retry-After; the script + * waits and carries on. + */ +import { parseArgs } from "node:util"; +import { setTimeout as sleep } from "node:timers/promises"; + +const KINDS = [ + "DOCTOR_REPORT", + "DISCHARGE_LETTER", + "LAB_RESULT", + "IMAGING", + "PRESCRIPTION", + "REFERRAL", + "INSURANCE", + "VACCINATION", + "OTHER", +]; +const TITLE_MAX = 200; +const SOURCE_ID_MAX = 128; +const PAGE_SIZE = 100; +/** Attempts for a server error or a dropped connection, per request. */ +const TRANSIENT_ATTEMPTS = 3; +const TRANSIENT_BACKOFF_MS = [5_000, 15_000, 45_000]; +/** Longest single wait we accept from a Retry-After header, in seconds. */ +const MAX_RETRY_AFTER_S = 3_600; + +class StopError extends Error {} + +const out = (line = "") => process.stdout.write(`${line}\n`); +const err = (line) => process.stderr.write(`${line}\n`); + +function usage() { + const text = []; + let inside = false; + for (const line of SELF_DOC.split("\n")) { + if (line.startsWith("/**")) inside = true; + else if (line.startsWith(" */")) break; + else if (inside) text.push(line.replace(/^ \* ?/, "")); + } + return text.join("\n"); +} + +// ── Arguments ────────────────────────────────────────────────────────────── + +function readOptions(argv, env) { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + options: { + "healthlog-url": { type: "string" }, + "paperless-url": { type: "string" }, + "papra-url": { type: "string" }, + "papra-org": { type: "string" }, + tag: { type: "string" }, + since: { type: "string" }, + kind: { type: "string" }, + "kind-map": { type: "string" }, + "dry-run": { type: "boolean", default: false }, + "ai-read": { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, + }); + if (values.help) return { help: true }; + + const source = positionals[0]; + if (source !== "paperless" && source !== "papra") { + throw new StopError( + 'Name the source first: "paperless" or "papra". Run with --help for the options.', + ); + } + const dryRun = values["dry-run"]; + const healthlogUrl = trimSlash(values["healthlog-url"] ?? env.HEALTHLOG_URL); + if (!dryRun && !healthlogUrl) { + throw new StopError("--healthlog-url is required (or HEALTHLOG_URL)."); + } + if (!dryRun && !env.HEALTHLOG_TOKEN) { + throw new StopError( + "HEALTHLOG_TOKEN is not set. Create a document token in HealthLog under Settings > API & Tokens.", + ); + } + if (values.since && !/^\d{4}-\d{2}-\d{2}$/.test(values.since)) { + throw new StopError("--since expects a date as YYYY-MM-DD."); + } + const kind = values.kind ? checkKind(values.kind, "--kind") : null; + const kindMap = new Map(); + if (values["kind-map"]) { + for (const pair of values["kind-map"].split(",")) { + const at = pair.lastIndexOf("="); + if (at <= 0) + throw new StopError(`--kind-map: "${pair}" is not NAME=KIND.`); + kindMap.set( + pair.slice(0, at).trim().toLowerCase(), + checkKind(pair.slice(at + 1).trim(), "--kind-map"), + ); + } + } + + const options = { + source, + healthlogUrl, + healthlogToken: env.HEALTHLOG_TOKEN ?? "", + tag: values.tag ?? null, + since: values.since ?? null, + kind, + kindMap, + dryRun, + aiRead: values["ai-read"], + }; + if (source === "paperless") { + options.baseUrl = trimSlash(values["paperless-url"] ?? env.PAPERLESS_URL); + options.sourceToken = env.PAPERLESS_TOKEN; + if (!options.baseUrl) throw new StopError("--paperless-url is required."); + if (!options.sourceToken) + throw new StopError("PAPERLESS_TOKEN is not set."); + } else { + options.baseUrl = trimSlash(values["papra-url"] ?? env.PAPRA_URL); + options.organizationId = values["papra-org"] ?? env.PAPRA_ORG; + options.sourceToken = env.PAPRA_TOKEN; + if (!options.baseUrl) throw new StopError("--papra-url is required."); + if (!options.organizationId) { + throw new StopError( + "--papra-org is required: the organization id from the Papra address bar.", + ); + } + if (!options.sourceToken) throw new StopError("PAPRA_TOKEN is not set."); + } + return options; +} + +function checkKind(value, flag) { + const upper = value.toUpperCase(); + if (!KINDS.includes(upper)) { + throw new StopError( + `${flag}: "${value}" is not one of ${KINDS.join(", ")}.`, + ); + } + return upper; +} + +function trimSlash(url) { + return url ? url.replace(/\/+$/, "") : url; +} + +// ── HTTP ─────────────────────────────────────────────────────────────────── + +function retryAfterSeconds(response, fallback) { + const raw = response.headers.get("retry-after"); + if (!raw) return fallback; + const seconds = Number(raw); + if (Number.isFinite(seconds)) + return Math.min(Math.max(0, seconds), MAX_RETRY_AFTER_S); + const at = Date.parse(raw); + if (Number.isFinite(at)) { + return Math.min( + Math.max(0, Math.ceil((at - Date.now()) / 1000)), + MAX_RETRY_AFTER_S, + ); + } + return fallback; +} + +/** + * Fetch with the waiting rules shared by all three services: a 429 waits for + * Retry-After and tries again (as often as it takes, since that is the + * service pacing us), a 5xx or a dropped connection is retried a few times + * with a growing pause, and anything else is returned to the caller. + */ +async function request(label, url, init, makeBody) { + let transient = 0; + for (;;) { + let response; + try { + response = await fetch(url, { + ...init, + body: makeBody ? makeBody() : undefined, + redirect: "manual", + }); + } catch (cause) { + if (transient >= TRANSIENT_ATTEMPTS) { + throw new Error(`${label}: could not connect (${cause.message})`); + } + await wait(TRANSIENT_BACKOFF_MS[transient++], `${label} did not answer`); + continue; + } + if (response.status === 429) { + await response.arrayBuffer().catch(() => {}); + const seconds = retryAfterSeconds(response, 60); + await wait(seconds * 1000, `${label} asked to slow down`); + continue; + } + if (response.status >= 500 || response.status === 408) { + await response.arrayBuffer().catch(() => {}); + if (transient >= TRANSIENT_ATTEMPTS) return response; + await wait( + TRANSIENT_BACKOFF_MS[transient++], + `${label} answered ${response.status}`, + ); + continue; + } + return response; + } +} + +async function wait(ms, why) { + if (ms > 0) out(` … ${why}, waiting ${Math.round(ms / 1000)} s`); + await sleep(ms); +} + +async function sourceJson(label, url, headers) { + const response = await request(label, url, { headers }); + if (response.status === 401 || response.status === 403) { + throw new StopError( + `${label} refused the token (${response.status}). Check the token and its permissions.`, + ); + } + if (!response.ok) { + throw new StopError(`${label} answered ${response.status} for ${url}`); + } + return response.json(); +} + +/** Keep the source's own origin: a `next` link behind a proxy may name another. */ +function sameOrigin(base, link) { + const next = new URL(link, base); + const origin = new URL(base); + next.protocol = origin.protocol; + next.host = origin.host; + return next.toString(); +} + +function filenameFrom(response, fallback) { + const header = response.headers.get("content-disposition") ?? ""; + const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header); + if (star) { + try { + return decodeURIComponent(star[1].trim().replace(/^"|"$/g, "")); + } catch { + /* fall through */ + } + } + const plain = /filename="?([^";]+)"?/i.exec(header); + return plain ? plain[1] : fallback; +} + +// ── Paperless-ngx ────────────────────────────────────────────────────────── + +function paperless(options) { + const headers = { + Authorization: `Token ${options.sourceToken}`, + Accept: "application/json; version=10", + }; + const base = options.baseUrl; + const label = "Paperless-ngx"; + + async function* pages(firstUrl) { + let url = firstUrl; + while (url) { + const page = await sourceJson(label, url, headers); + for (const item of page.results ?? []) yield item; + url = page.next ? sameOrigin(base, page.next) : null; + } + } + + return { + system: "PAPERLESS", + label, + async *documents() { + const query = new URLSearchParams({ + page_size: String(PAGE_SIZE), + ordering: "id", + }); + let tagId = null; + if (options.tag) { + const found = []; + for await (const tag of pages( + `${base}/api/tags/?${new URLSearchParams({ name__iexact: options.tag, page_size: String(PAGE_SIZE) })}`, + )) { + if (String(tag.name).toLowerCase() === options.tag.toLowerCase()) + found.push(tag); + } + if (found.length === 0) { + throw new StopError( + `Paperless-ngx has no tag named "${options.tag}".`, + ); + } + tagId = found[0].id; + query.set("tags__id__all", String(tagId)); + } + if (options.since) query.set("created__date__gte", options.since); + + let typeNames = new Map(); + if (options.kindMap.size > 0) { + for await (const type of pages( + `${base}/api/document_types/?page_size=${PAGE_SIZE}`, + )) { + typeNames.set(type.id, String(type.name)); + } + } + + for await (const doc of pages(`${base}/api/documents/?${query}`)) { + // The server filters too; checking again keeps an older Paperless + // that ignores a parameter from importing the whole archive. + if ( + tagId !== null && + Array.isArray(doc.tags) && + !doc.tags.includes(tagId) + ) { + continue; + } + const created = dayOf(doc.created); + const added = dayOf(doc.added); + yield { + sourceId: String(doc.id), + title: doc.title || doc.original_file_name || `Paperless ${doc.id}`, + date: created ?? added, + dateFallback: created ? null : added ? "added" : "none", + filename: doc.original_file_name ?? null, + kindKeys: + doc.document_type != null && typeNames.has(doc.document_type) + ? [typeNames.get(doc.document_type)] + : [], + size: null, + }; + } + }, + async size(doc) { + const meta = await sourceJson( + label, + `${base}/api/documents/${encodeURIComponent(doc.sourceId)}/metadata/`, + headers, + ); + return typeof meta.original_size === "number" ? meta.original_size : null; + }, + async download(doc) { + const response = await request( + label, + `${base}/api/documents/${encodeURIComponent(doc.sourceId)}/download/?original=true`, + { headers: { Authorization: headers.Authorization } }, + ); + return readDownload(label, response, doc); + }, + }; +} + +// ── Papra ────────────────────────────────────────────────────────────────── + +function papra(options) { + const headers = { + Authorization: `Bearer ${options.sourceToken}`, + Accept: "application/json", + }; + const org = `${options.baseUrl}/api/organizations/${encodeURIComponent(options.organizationId)}`; + const label = "Papra"; + + return { + system: "PAPRA", + label, + async *documents() { + let tagId = null; + if (options.tag) { + const body = await sourceJson(label, `${org}/tags`, headers); + const tag = (body.tags ?? []).find( + (t) => String(t.name).toLowerCase() === options.tag.toLowerCase(), + ); + if (!tag) + throw new StopError(`Papra has no tag named "${options.tag}".`); + tagId = tag.id; + } + const search = options.tag + ? `tag:${/\s/.test(options.tag) ? JSON.stringify(options.tag) : options.tag}` + : ""; + for (let pageIndex = 0; ; pageIndex++) { + const query = new URLSearchParams({ + pageIndex: String(pageIndex), + pageSize: String(PAGE_SIZE), + }); + if (search) query.set("searchQuery", search); + const body = await sourceJson( + label, + `${org}/documents?${query}`, + headers, + ); + const documents = body.documents ?? []; + for (const doc of documents) { + const tags = Array.isArray(doc.tags) ? doc.tags : null; + if (tagId !== null && tags && !tags.some((t) => t.id === tagId)) + continue; + const documentDate = dayOf(doc.documentDate); + const created = dayOf(doc.createdAt); + const date = documentDate ?? created; + // Papra's search dates differ between versions; the date is + // checked here instead of trusting a query syntax. + if (options.since && date && date < options.since) continue; + yield { + sourceId: String(doc.id), + title: stripExtension(doc.name) || `Papra ${doc.id}`, + date, + dateFallback: documentDate ? null : created ? "added" : "none", + filename: doc.originalName ?? doc.name ?? null, + kindKeys: (tags ?? []).map((t) => String(t.name)), + size: + typeof doc.originalSize === "number" ? doc.originalSize : null, + }; + } + const total = + typeof body.documentsCount === "number" ? body.documentsCount : null; + if (documents.length < PAGE_SIZE) break; + if (total !== null && (pageIndex + 1) * PAGE_SIZE >= total) break; + } + }, + async size(doc) { + return doc.size; + }, + async download(doc) { + const response = await request( + label, + `${org}/documents/${encodeURIComponent(doc.sourceId)}/file`, + { headers: { Authorization: headers.Authorization } }, + ); + return readDownload(label, response, doc); + }, + }; +} + +async function readDownload(label, response, doc) { + if (response.status === 401 || response.status === 403) { + throw new StopError(`${label} refused the download (${response.status}).`); + } + if (!response.ok) { + return { skip: `${label} could not deliver the file (${response.status})` }; + } + const bytes = new Uint8Array(await response.arrayBuffer()); + return { + bytes, + filename: + doc.filename ?? filenameFrom(response, `document-${doc.sourceId}`), + type: response.headers.get("content-type") ?? "application/octet-stream", + }; +} + +function dayOf(value) { + if (typeof value !== "string") return null; + const day = value.slice(0, 10); + return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null; +} + +function stripExtension(name) { + if (typeof name !== "string") return ""; + return name.replace(/\.[A-Za-z0-9]{1,5}$/, ""); +} + +// ── HealthLog ────────────────────────────────────────────────────────────── + +function kindFor(options, doc) { + for (const key of doc.kindKeys) { + const mapped = options.kindMap.get(key.toLowerCase()); + if (mapped) return mapped; + } + return options.kind; +} + +async function upload(options, source, doc, file) { + const url = `${options.healthlogUrl}/api/documents/inbound`; + const kind = kindFor(options, doc); + const response = await request( + "HealthLog", + url, + { + method: "POST", + headers: { Authorization: `Bearer ${options.healthlogToken}` }, + }, + // Rebuilt per attempt: a body stream is spent once it has been sent. + () => { + const form = new FormData(); + form.append( + "file", + new Blob([file.bytes], { type: file.type }), + file.filename, + ); + form.append("title", doc.title.trim().slice(0, TITLE_MAX)); + if (doc.date) form.append("documentDate", doc.date); + if (kind) form.append("kind", kind); + form.append("sourceSystem", source.system); + form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX)); + if (!options.aiRead) form.append("aiRead", "defer"); + return form; + }, + ); + + let body = null; + try { + body = await response.json(); + } catch { + body = null; + } + const meta = body?.meta ?? {}; + switch (response.status) { + case 201: + return { outcome: "imported" }; + case 200: + return body?.data?.deleted + ? { outcome: "deleted" } + : { outcome: "duplicate" }; + case 401: + throw new StopError( + "HealthLog refused the token (401). It may be expired or revoked; create a new document token.", + ); + case 403: + if (meta.errorCode === "module.disabled") { + throw new StopError( + "The document vault is switched off in HealthLog. Turn on Documents in Settings, then run the script again.", + ); + } + throw new StopError( + "HealthLog refused the upload (403). Use a document token from Settings > API & Tokens; other tokens cannot upload documents.", + ); + case 413: + if (meta.reason === "quotaExceeded") { + throw new StopError( + `Your HealthLog storage is full (${mib(meta.usedBytes)} of ${mib(meta.quotaBytes)} used). An admin can raise the limit in the admin area, for everyone or just for your account.`, + ); + } + return { + outcome: "skipped", + reason: `larger than HealthLog accepts${meta.maxFileBytes ? ` (${mib(meta.maxFileBytes)})` : ""}`, + }; + case 415: + return { + outcome: "skipped", + reason: "HealthLog does not accept this file type", + }; + case 422: + return { + outcome: "skipped", + reason: `HealthLog refused the details: ${body?.error ?? "invalid"}`, + }; + default: + return { + outcome: "skipped", + reason: `HealthLog answered ${response.status}${body?.error ? `: ${body.error}` : ""}`, + }; + } +} + +function mib(bytes) { + if (typeof bytes !== "number") return "?"; + return bytes >= 1024 * 1024 * 1024 + ? `${(bytes / 1024 ** 3).toFixed(1)} GB` + : `${(bytes / 1024 ** 2).toFixed(1)} MB`; +} + +// ── Run ──────────────────────────────────────────────────────────────────── + +async function run(argv, env = process.env) { + let options; + try { + options = readOptions(argv, env); + } catch (error) { + if ( + error instanceof StopError || + error?.code === "ERR_PARSE_ARGS_UNKNOWN_OPTION" + ) { + err(error.message); + return 2; + } + throw error; + } + if (options.help) { + out(usage()); + return 0; + } + + const source = + options.source === "paperless" ? paperless(options) : papra(options); + const tally = { + imported: 0, + duplicate: 0, + deleted: 0, + skipped: [], + fallback: [], + }; + let listed = 0; + let totalBytes = 0; + let unknownSizes = 0; + + out( + `${options.dryRun ? "Dry run: listing" : "Importing"} documents from ${source.label}` + + `${options.tag ? ` tagged "${options.tag}"` : ""}${options.since ? ` since ${options.since}` : ""}.`, + ); + + try { + for await (const doc of source.documents()) { + listed++; + const where = `${source.label} ${doc.sourceId} "${doc.title}"`; + if (doc.dateFallback) + tally.fallback.push({ where, dateFallback: doc.dateFallback }); + + if (options.dryRun) { + const size = await source.size(doc); + if (size === null) unknownSizes++; + else totalBytes += size; + const kind = kindFor(options, doc) ?? "OTHER"; + out( + ` ${doc.date ?? "no date"} ${kind.padEnd(16)} ${size === null ? "?" : mib(size)} ${where}`, + ); + continue; + } + + const file = await source.download(doc); + if (file.skip) { + tally.skipped.push({ where, reason: file.skip }); + out(` skipped ${where}: ${file.skip}`); + continue; + } + const result = await upload(options, source, doc, file); + if (result.outcome === "imported") { + tally.imported++; + out(` imported ${where}`); + } else if (result.outcome === "duplicate") { + tally.duplicate++; + out(` present ${where}`); + } else if (result.outcome === "deleted") { + tally.deleted++; + out(` deleted ${where} (you deleted it in HealthLog; left alone)`); + } else { + tally.skipped.push({ where, reason: result.reason }); + out(` skipped ${where}: ${result.reason}`); + } + } + } catch (error) { + printSummary(options, tally, listed, totalBytes, unknownSizes); + err(""); + err(`Stopped: ${error.message}`); + return 1; + } + + printSummary(options, tally, listed, totalBytes, unknownSizes); + return tally.skipped.length > 0 ? 3 : 0; +} + +function printSummary(options, tally, listed, totalBytes, unknownSizes) { + out(""); + if (options.dryRun) { + out( + `${listed} document(s) would be imported, about ${mib(totalBytes)}` + + `${unknownSizes > 0 ? ` (size unknown for ${unknownSizes})` : ""}.`, + ); + out( + "HealthLog's default storage limit is 1 GB per person; an admin can raise it.", + ); + out( + "Documents already in HealthLog, or deleted there, will be skipped by HealthLog.", + ); + } else { + out(`Imported: ${tally.imported}`); + out(`Already in HealthLog: ${tally.duplicate}`); + out(`Deleted in HealthLog, left alone: ${tally.deleted}`); + out(`Skipped: ${tally.skipped.length}`); + for (const { where, reason } of tally.skipped) out(` ${where}: ${reason}`); + } + if (tally.fallback.length > 0) { + out( + `No document date in the source for ${tally.fallback.length} document(s):`, + ); + for (const { where, dateFallback } of tally.fallback) { + out( + dateFallback === "added" + ? ` ${where}: filed under the day it was added to the source` + : ` ${where}: filed under the day of import`, + ); + } + } + if (!options.dryRun && tally.imported > 0 && !options.aiRead) { + out( + `${tally.imported} document(s) imported without automatic AI reading. ` + + "To have one read, open it in HealthLog and choose Read with AI or Generate summary; " + + "Documents > Index all for search reads the ones that had no text to search.", + ); + } +} + +const SELF_DOC = await (async () => { + const { readFile } = await import("node:fs/promises"); + return readFile(new URL(import.meta.url), "utf8"); +})(); + +process.exitCode = await run(process.argv.slice(2));