mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
feat(scripts): import documents from Paperless-ngx and Papra
A single dependency-free Node 22 script that copies tagged documents from Paperless-ngx (REST, original files) or Papra (API key, /file) into the vault through a document token. Serial, one file in memory, waits on 429 Retry-After, and resumable through the server's source-key dedup. Supports --since, --kind / --kind-map, --dry-run with sizes, and holds back automatic AI reading unless --ai-read is given. Refs #1038
This commit is contained in:
Marc-André Bombeck committed
1 parent
55a32ff7d1
commit
0257f2207a
2 files changed
+1254
No files matched your search
@@ -0,0 +1,531 @@
|
||||
/**
|
||||
* `scripts/import-documents.mjs` against recorded-shape fixtures of the two
|
||||
* source APIs and a stand-in HealthLog, all on local HTTP servers.
|
||||
*
|
||||
* The fixtures are the shapes the script depends on (Paperless-ngx API v10
|
||||
* pagination, tag and document-type lookups, `?original=true` downloads;
|
||||
* Papra's organization-scoped document list, tags and `/file`), so a change
|
||||
* on either side shows up here as a fixture to refresh rather than as a user
|
||||
* report. The HealthLog stand-in answers the upload contract the real route
|
||||
* speaks: 201, 200 duplicate, 200 deleted, 413, 429 with Retry-After, 403
|
||||
* `module.disabled`.
|
||||
*/
|
||||
import { spawn } from "node:child_process";
|
||||
import { createServer, type IncomingMessage, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
const SCRIPT = join(process.cwd(), "scripts/import-documents.mjs");
|
||||
const PDF = Buffer.from("%PDF-1.7\n%fixture\n%%EOF\n");
|
||||
|
||||
const servers: Server[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
servers.splice(0).map((s) => new Promise((r) => s.close(() => r(null)))),
|
||||
);
|
||||
});
|
||||
|
||||
type Handler = (
|
||||
req: IncomingMessage,
|
||||
url: URL,
|
||||
body: Buffer,
|
||||
) => Promise<{
|
||||
status: number;
|
||||
json?: unknown;
|
||||
bytes?: Buffer;
|
||||
headers?: Record<string, string>;
|
||||
}>;
|
||||
|
||||
async function serve(handler: Handler): Promise<string> {
|
||||
const server = createServer(async (req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const c of req) chunks.push(c as Buffer);
|
||||
const url = new URL(req.url ?? "/", "http://localhost");
|
||||
const r = await handler(req, url, Buffer.concat(chunks));
|
||||
const headers: Record<string, string> = { ...(r.headers ?? {}) };
|
||||
if (r.json !== undefined) headers["content-type"] = "application/json";
|
||||
res.writeHead(r.status, headers);
|
||||
res.end(r.json !== undefined ? JSON.stringify(r.json) : (r.bytes ?? ""));
|
||||
});
|
||||
servers.push(server);
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||
return `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
}
|
||||
|
||||
function runScript(
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
): Promise<{ code: number | null; stdout: string; stderr: string }> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, [SCRIPT, ...args], {
|
||||
env: { PATH: process.env.PATH ?? "", ...env },
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (d) => (stdout += d));
|
||||
child.stderr.on("data", (d) => (stderr += d));
|
||||
child.on("close", (code) => resolve({ code, stdout, stderr }));
|
||||
});
|
||||
}
|
||||
|
||||
interface Received {
|
||||
auth: string | undefined;
|
||||
fields: Record<string, string>;
|
||||
filename: string | null;
|
||||
size: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A HealthLog stand-in. Keys stored documents by source key the way the real
|
||||
* route does, answers the first upload with a 429, and refuses keys listed as
|
||||
* deleted or too large.
|
||||
*/
|
||||
async function healthlog(
|
||||
opts: {
|
||||
deleted?: string[];
|
||||
tooLarge?: string[];
|
||||
moduleOff?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const stored = new Set<string>();
|
||||
const received: Received[] = [];
|
||||
let throttled = false;
|
||||
const url = await serve(async (req, url, body) => {
|
||||
if (url.pathname !== "/api/documents/inbound" || req.method !== "POST") {
|
||||
return { status: 404, json: { data: null, error: "Not found" } };
|
||||
}
|
||||
if (opts.moduleOff) {
|
||||
return {
|
||||
status: 403,
|
||||
json: {
|
||||
data: null,
|
||||
error: 'Module "inboundDocuments" is not enabled',
|
||||
meta: { errorCode: "module.disabled" },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (!throttled) {
|
||||
throttled = true;
|
||||
return {
|
||||
status: 429,
|
||||
headers: { "retry-after": "0" },
|
||||
json: {
|
||||
data: null,
|
||||
error: "Too many uploads.",
|
||||
meta: { errorCode: "documents.inbound.rateLimited" },
|
||||
},
|
||||
};
|
||||
}
|
||||
const form = await new Response(body, {
|
||||
headers: { "content-type": req.headers["content-type"] ?? "" },
|
||||
}).formData();
|
||||
const fields: Record<string, string> = {};
|
||||
let filename: string | null = null;
|
||||
let size = 0;
|
||||
for (const [k, v] of form.entries()) {
|
||||
if (typeof v === "string") fields[k] = v;
|
||||
else {
|
||||
filename = v.name;
|
||||
size = v.size;
|
||||
}
|
||||
}
|
||||
received.push({ auth: req.headers.authorization, fields, filename, size });
|
||||
const key = `${fields.sourceSystem}:${fields.sourceId}`;
|
||||
if (opts.tooLarge?.includes(key)) {
|
||||
return {
|
||||
status: 413,
|
||||
json: {
|
||||
data: null,
|
||||
error: "File is too large.",
|
||||
meta: { reason: "fileTooLarge", maxFileBytes: 26_214_400 },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (opts.deleted?.includes(key)) {
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
data: { id: null, duplicate: true, deleted: true },
|
||||
error: null,
|
||||
meta: { duplicate: true, deleted: true },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (stored.has(key)) {
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
data: { id: key, duplicate: true },
|
||||
error: null,
|
||||
meta: { duplicate: true },
|
||||
},
|
||||
};
|
||||
}
|
||||
stored.add(key);
|
||||
return {
|
||||
status: 201,
|
||||
json: { data: { id: key, duplicate: false }, error: null },
|
||||
};
|
||||
});
|
||||
return { url, received };
|
||||
}
|
||||
|
||||
/** Paperless-ngx v10 stand-in: two pages of documents, one outside the tag. */
|
||||
async function paperless() {
|
||||
const seen: { path: string; auth?: string; accept?: string }[] = [];
|
||||
const url = await serve(async (req, url) => {
|
||||
seen.push({
|
||||
path: url.pathname + url.search,
|
||||
auth: req.headers.authorization,
|
||||
accept: req.headers.accept,
|
||||
});
|
||||
const base = `http://${req.headers.host}`;
|
||||
if (url.pathname === "/api/tags/") {
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
count: 1,
|
||||
next: null,
|
||||
results: [{ id: 7, name: "HealthLog" }],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (url.pathname === "/api/document_types/") {
|
||||
return {
|
||||
status: 200,
|
||||
json: { count: 1, next: null, results: [{ id: 3, name: "Labor" }] },
|
||||
};
|
||||
}
|
||||
if (url.pathname === "/api/documents/") {
|
||||
if (url.searchParams.get("page") === "2") {
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
count: 3,
|
||||
next: null,
|
||||
results: [
|
||||
{
|
||||
id: 3,
|
||||
title: "Car insurance",
|
||||
created: "2024-05-01",
|
||||
added: "2024-05-02T10:00:00Z",
|
||||
tags: [9],
|
||||
document_type: null,
|
||||
original_file_name: "car.pdf",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
const next = new URL(url.toString(), base);
|
||||
next.searchParams.set("page", "2");
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
count: 3,
|
||||
// Deliberately another origin, as behind a reverse proxy.
|
||||
next: `https://paperless.internal${next.pathname}${next.search}`,
|
||||
results: [
|
||||
{
|
||||
id: 1,
|
||||
title: "Blutbild März",
|
||||
created: "2024-03-05",
|
||||
added: "2024-03-06T08:00:00Z",
|
||||
tags: [7],
|
||||
document_type: 3,
|
||||
original_file_name: "blutbild.pdf",
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
title: "Arztbrief",
|
||||
created: null,
|
||||
added: "2024-04-01T09:30:00Z",
|
||||
tags: [7],
|
||||
document_type: null,
|
||||
original_file_name: "brief.pdf",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
const download = /^\/api\/documents\/(\d+)\/download\/$/.exec(url.pathname);
|
||||
if (download) {
|
||||
return {
|
||||
status: 200,
|
||||
bytes: PDF,
|
||||
headers: {
|
||||
"content-type": "application/pdf",
|
||||
"content-disposition": `attachment; filename="doc-${download[1]}.pdf"`,
|
||||
},
|
||||
};
|
||||
}
|
||||
const metadata = /^\/api\/documents\/(\d+)\/metadata\/$/.exec(url.pathname);
|
||||
if (metadata) {
|
||||
return { status: 200, json: { original_size: 1_048_576 } };
|
||||
}
|
||||
return { status: 404, json: { detail: "Not found." } };
|
||||
});
|
||||
return { url, seen };
|
||||
}
|
||||
|
||||
/** Papra stand-in: one organization, a tag, three documents. */
|
||||
async function papra() {
|
||||
const seen: { path: string; auth?: string }[] = [];
|
||||
const url = await serve(async (req, url) => {
|
||||
seen.push({
|
||||
path: url.pathname + url.search,
|
||||
auth: req.headers.authorization,
|
||||
});
|
||||
const org = "/api/organizations/org_1";
|
||||
if (url.pathname === `${org}/tags`) {
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
tags: [
|
||||
{ id: "tag_h", name: "Health" },
|
||||
{ id: "tag_l", name: "Lab" },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (url.pathname === `${org}/documents`) {
|
||||
return {
|
||||
status: 200,
|
||||
json: {
|
||||
documentsCount: 3,
|
||||
documents: [
|
||||
{
|
||||
id: "doc_a",
|
||||
name: "Impfpass.pdf",
|
||||
originalName: "impfpass.pdf",
|
||||
originalSize: 2048,
|
||||
createdAt: "2025-01-10T12:00:00.000Z",
|
||||
documentDate: "2019-06-01",
|
||||
tags: [{ id: "tag_h", name: "Health" }],
|
||||
},
|
||||
{
|
||||
id: "doc_b",
|
||||
name: "Labor.pdf",
|
||||
originalName: "labor.pdf",
|
||||
originalSize: 4096,
|
||||
createdAt: "2025-02-10T12:00:00.000Z",
|
||||
tags: [
|
||||
{ id: "tag_h", name: "Health" },
|
||||
{ id: "tag_l", name: "Lab" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "doc_old",
|
||||
name: "Old.pdf",
|
||||
originalName: "old.pdf",
|
||||
originalSize: 100,
|
||||
createdAt: "2015-02-10T12:00:00.000Z",
|
||||
documentDate: "2015-02-10",
|
||||
tags: [{ id: "tag_h", name: "Health" }],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (
|
||||
/^\/api\/organizations\/org_1\/documents\/[\w]+\/file$/.test(url.pathname)
|
||||
) {
|
||||
return {
|
||||
status: 200,
|
||||
bytes: PDF,
|
||||
headers: { "content-type": "application/pdf" },
|
||||
};
|
||||
}
|
||||
return { status: 404, json: { error: "not found" } };
|
||||
});
|
||||
return { url, seen };
|
||||
}
|
||||
|
||||
describe("import-documents.mjs — Paperless-ngx", () => {
|
||||
it("imports the tagged documents with their title, date, kind and source key", async () => {
|
||||
const hl = await healthlog();
|
||||
const pl = await paperless();
|
||||
const env = { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" };
|
||||
const args = [
|
||||
"paperless",
|
||||
"--paperless-url",
|
||||
pl.url,
|
||||
"--healthlog-url",
|
||||
hl.url,
|
||||
"--tag",
|
||||
"HealthLog",
|
||||
"--kind-map",
|
||||
"Labor=LAB_RESULT",
|
||||
];
|
||||
const first = await runScript(args, env);
|
||||
|
||||
expect(first.stderr).toBe("");
|
||||
expect(first.code).toBe(0);
|
||||
// The first upload met a 429 and was sent again, not dropped.
|
||||
expect(hl.received).toHaveLength(2);
|
||||
expect(hl.received[0]).toMatchObject({
|
||||
auth: "Bearer hlk_test",
|
||||
filename: "blutbild.pdf",
|
||||
size: PDF.byteLength,
|
||||
fields: {
|
||||
title: "Blutbild März",
|
||||
documentDate: "2024-03-05",
|
||||
kind: "LAB_RESULT",
|
||||
sourceSystem: "PAPERLESS",
|
||||
sourceId: "1",
|
||||
aiRead: "defer",
|
||||
},
|
||||
});
|
||||
// No created date: filed under the day it was added, and said so.
|
||||
expect(hl.received[1].fields.documentDate).toBe("2024-04-01");
|
||||
expect(hl.received[1].fields.kind).toBeUndefined();
|
||||
expect(first.stdout).toMatch(/Imported: 2/);
|
||||
expect(first.stdout).toMatch(/filed under the day it was added/);
|
||||
expect(first.stdout).toMatch(/without automatic AI reading/);
|
||||
|
||||
// The API contract the script leans on.
|
||||
const docs = pl.seen.filter((s) => s.path.startsWith("/api/documents/?"));
|
||||
expect(docs[0].path).toContain("tags__id__all=7");
|
||||
expect(docs[0].accept).toBe("application/json; version=10");
|
||||
expect(docs[0].auth).toBe("Token pl_test");
|
||||
// `next` was followed on the source's own origin.
|
||||
expect(docs[1].path).toContain("page=2");
|
||||
expect(
|
||||
pl.seen.filter((s) => s.path.includes("/download/")).map((s) => s.path),
|
||||
).toEqual([
|
||||
"/api/documents/1/download/?original=true",
|
||||
"/api/documents/2/download/?original=true",
|
||||
]);
|
||||
|
||||
// A second run changes nothing: HealthLog answers duplicate for both.
|
||||
const second = await runScript(args, env);
|
||||
expect(second.code).toBe(0);
|
||||
expect(second.stdout).toMatch(/Imported: 0/);
|
||||
expect(second.stdout).toMatch(/Already in HealthLog: 2/);
|
||||
});
|
||||
|
||||
it("leaves a document deleted in HealthLog alone", async () => {
|
||||
const hl = await healthlog({ deleted: ["PAPERLESS:2"] });
|
||||
const pl = await paperless();
|
||||
const res = await runScript(
|
||||
[
|
||||
"paperless",
|
||||
"--paperless-url",
|
||||
pl.url,
|
||||
"--healthlog-url",
|
||||
hl.url,
|
||||
"--tag",
|
||||
"healthlog",
|
||||
],
|
||||
{ HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" },
|
||||
);
|
||||
expect(res.code).toBe(0);
|
||||
expect(res.stdout).toMatch(/Imported: 1/);
|
||||
expect(res.stdout).toMatch(/Deleted in HealthLog, left alone: 1/);
|
||||
});
|
||||
|
||||
it("stops with a plain instruction when the vault is switched off", async () => {
|
||||
const hl = await healthlog({ moduleOff: true });
|
||||
const pl = await paperless();
|
||||
const res = await runScript(
|
||||
["paperless", "--paperless-url", pl.url, "--healthlog-url", hl.url],
|
||||
{ HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" },
|
||||
);
|
||||
expect(res.code).toBe(1);
|
||||
expect(res.stderr).toMatch(/Turn on Documents in Settings/);
|
||||
});
|
||||
|
||||
it("lists sizes on a dry run and sends nothing", async () => {
|
||||
const pl = await paperless();
|
||||
const res = await runScript(
|
||||
[
|
||||
"paperless",
|
||||
"--paperless-url",
|
||||
pl.url,
|
||||
"--tag",
|
||||
"HealthLog",
|
||||
"--dry-run",
|
||||
],
|
||||
{ PAPERLESS_TOKEN: "pl_test" },
|
||||
);
|
||||
expect(res.code).toBe(0);
|
||||
expect(res.stdout).toMatch(
|
||||
/2 document\(s\) would be imported, about 2\.0 MB/,
|
||||
);
|
||||
expect(pl.seen.some((s) => s.path.includes("/download/"))).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses to start without a HealthLog token", async () => {
|
||||
const res = await runScript(
|
||||
[
|
||||
"paperless",
|
||||
"--paperless-url",
|
||||
"http://x",
|
||||
"--healthlog-url",
|
||||
"http://y",
|
||||
],
|
||||
{ PAPERLESS_TOKEN: "pl_test" },
|
||||
);
|
||||
expect(res.code).toBe(2);
|
||||
expect(res.stderr).toMatch(/HEALTHLOG_TOKEN is not set/);
|
||||
});
|
||||
|
||||
it("names a missing tag instead of importing everything", async () => {
|
||||
const pl = await paperless();
|
||||
const res = await runScript(
|
||||
["paperless", "--paperless-url", pl.url, "--tag", "Nope", "--dry-run"],
|
||||
{ PAPERLESS_TOKEN: "pl_test" },
|
||||
);
|
||||
expect(res.code).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("import-documents.mjs — Papra", () => {
|
||||
it("imports by tag since a date, and skips what HealthLog refuses as too large", async () => {
|
||||
const hl = await healthlog({ tooLarge: ["PAPRA:doc_b"] });
|
||||
const pp = await papra();
|
||||
const res = await runScript(
|
||||
[
|
||||
"papra",
|
||||
"--papra-url",
|
||||
pp.url,
|
||||
"--papra-org",
|
||||
"org_1",
|
||||
"--healthlog-url",
|
||||
hl.url,
|
||||
"--tag",
|
||||
"Health",
|
||||
"--since",
|
||||
"2016-01-01",
|
||||
"--kind-map",
|
||||
"Lab=LAB_RESULT",
|
||||
"--ai-read",
|
||||
],
|
||||
{ HEALTHLOG_TOKEN: "hlk_test", PAPRA_TOKEN: "pp_test" },
|
||||
);
|
||||
|
||||
// Exit 3: finished, with at least one document skipped.
|
||||
expect(res.code).toBe(3);
|
||||
const sent = hl.received.map((r) => r.fields);
|
||||
expect(sent).toHaveLength(2);
|
||||
expect(sent[0]).toMatchObject({
|
||||
title: "Impfpass",
|
||||
documentDate: "2019-06-01",
|
||||
sourceSystem: "PAPRA",
|
||||
sourceId: "doc_a",
|
||||
});
|
||||
// --ai-read: no deferral field at all.
|
||||
expect(sent[0].aiRead).toBeUndefined();
|
||||
expect(sent[1]).toMatchObject({ sourceId: "doc_b", kind: "LAB_RESULT" });
|
||||
// doc_old is before --since and never downloaded.
|
||||
expect(pp.seen.some((s) => s.path.includes("doc_old"))).toBe(false);
|
||||
expect(pp.seen.find((s) => s.path.includes("/documents?"))?.path).toContain(
|
||||
"searchQuery=tag%3AHealth",
|
||||
);
|
||||
expect(pp.seen[0].auth).toBe("Bearer pp_test");
|
||||
expect(res.stdout).toMatch(/Imported: 1/);
|
||||
expect(res.stdout).toMatch(/Skipped: 1/);
|
||||
expect(res.stdout).toMatch(/larger than HealthLog accepts/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,723 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Copy documents from Paperless-ngx or Papra into the HealthLog document vault.
|
||||
*
|
||||
* Runs wherever you are, not inside the HealthLog container: it talks to two
|
||||
* HTTP APIs and needs nothing but Node 22 (no dependencies, no install).
|
||||
*
|
||||
* node import-documents.mjs paperless --paperless-url https://paperless.lan \
|
||||
* --healthlog-url https://health.example --tag HealthLog
|
||||
*
|
||||
* node import-documents.mjs papra --papra-url https://papra.lan \
|
||||
* --papra-org <organization id> --healthlog-url https://health.example \
|
||||
* --tag Health
|
||||
*
|
||||
* Credentials come from the environment, never from flags (flags end up in
|
||||
* shell history):
|
||||
*
|
||||
* HEALTHLOG_TOKEN a document token from HealthLog Settings > API & Tokens
|
||||
* PAPERLESS_TOKEN a Paperless-ngx API token (My Profile)
|
||||
* PAPRA_TOKEN a Papra API key with documents:read and tags:read
|
||||
*
|
||||
* Options:
|
||||
* --tag <name> only documents carrying this tag (recommended)
|
||||
* --since YYYY-MM-DD only documents dated on or after this day
|
||||
* --kind <KIND> document type for everything imported
|
||||
* --kind-map "A=KIND,B=KIND"
|
||||
* type by Paperless document type or Papra tag name;
|
||||
* anything unmatched gets --kind, or HealthLog's OTHER
|
||||
* --dry-run list what would be imported, with sizes; send nothing
|
||||
* --ai-read let HealthLog read the documents with AI as they
|
||||
* arrive (by default automatic reading is held back)
|
||||
* --help this text
|
||||
*
|
||||
* One document at a time, one file in memory at a time. A document that is
|
||||
* already in HealthLog, or that you deleted there, is skipped by HealthLog
|
||||
* itself, so running the script again is safe and simply picks up where it
|
||||
* stopped. HealthLog's rate limit answers 429 with Retry-After; the script
|
||||
* waits and carries on.
|
||||
*/
|
||||
import { parseArgs } from "node:util";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
|
||||
const KINDS = [
|
||||
"DOCTOR_REPORT",
|
||||
"DISCHARGE_LETTER",
|
||||
"LAB_RESULT",
|
||||
"IMAGING",
|
||||
"PRESCRIPTION",
|
||||
"REFERRAL",
|
||||
"INSURANCE",
|
||||
"VACCINATION",
|
||||
"OTHER",
|
||||
];
|
||||
const TITLE_MAX = 200;
|
||||
const SOURCE_ID_MAX = 128;
|
||||
const PAGE_SIZE = 100;
|
||||
/** Attempts for a server error or a dropped connection, per request. */
|
||||
const TRANSIENT_ATTEMPTS = 3;
|
||||
const TRANSIENT_BACKOFF_MS = [5_000, 15_000, 45_000];
|
||||
/** Longest single wait we accept from a Retry-After header, in seconds. */
|
||||
const MAX_RETRY_AFTER_S = 3_600;
|
||||
|
||||
class StopError extends Error {}
|
||||
|
||||
const out = (line = "") => process.stdout.write(`${line}\n`);
|
||||
const err = (line) => process.stderr.write(`${line}\n`);
|
||||
|
||||
function usage() {
|
||||
const text = [];
|
||||
let inside = false;
|
||||
for (const line of SELF_DOC.split("\n")) {
|
||||
if (line.startsWith("/**")) inside = true;
|
||||
else if (line.startsWith(" */")) break;
|
||||
else if (inside) text.push(line.replace(/^ \* ?/, ""));
|
||||
}
|
||||
return text.join("\n");
|
||||
}
|
||||
|
||||
// ── Arguments ──────────────────────────────────────────────────────────────
|
||||
|
||||
function readOptions(argv, env) {
|
||||
const { values, positionals } = parseArgs({
|
||||
args: argv,
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
"healthlog-url": { type: "string" },
|
||||
"paperless-url": { type: "string" },
|
||||
"papra-url": { type: "string" },
|
||||
"papra-org": { type: "string" },
|
||||
tag: { type: "string" },
|
||||
since: { type: "string" },
|
||||
kind: { type: "string" },
|
||||
"kind-map": { type: "string" },
|
||||
"dry-run": { type: "boolean", default: false },
|
||||
"ai-read": { type: "boolean", default: false },
|
||||
help: { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
if (values.help) return { help: true };
|
||||
|
||||
const source = positionals[0];
|
||||
if (source !== "paperless" && source !== "papra") {
|
||||
throw new StopError(
|
||||
'Name the source first: "paperless" or "papra". Run with --help for the options.',
|
||||
);
|
||||
}
|
||||
const dryRun = values["dry-run"];
|
||||
const healthlogUrl = trimSlash(values["healthlog-url"] ?? env.HEALTHLOG_URL);
|
||||
if (!dryRun && !healthlogUrl) {
|
||||
throw new StopError("--healthlog-url is required (or HEALTHLOG_URL).");
|
||||
}
|
||||
if (!dryRun && !env.HEALTHLOG_TOKEN) {
|
||||
throw new StopError(
|
||||
"HEALTHLOG_TOKEN is not set. Create a document token in HealthLog under Settings > API & Tokens.",
|
||||
);
|
||||
}
|
||||
if (values.since && !/^\d{4}-\d{2}-\d{2}$/.test(values.since)) {
|
||||
throw new StopError("--since expects a date as YYYY-MM-DD.");
|
||||
}
|
||||
const kind = values.kind ? checkKind(values.kind, "--kind") : null;
|
||||
const kindMap = new Map();
|
||||
if (values["kind-map"]) {
|
||||
for (const pair of values["kind-map"].split(",")) {
|
||||
const at = pair.lastIndexOf("=");
|
||||
if (at <= 0)
|
||||
throw new StopError(`--kind-map: "${pair}" is not NAME=KIND.`);
|
||||
kindMap.set(
|
||||
pair.slice(0, at).trim().toLowerCase(),
|
||||
checkKind(pair.slice(at + 1).trim(), "--kind-map"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const options = {
|
||||
source,
|
||||
healthlogUrl,
|
||||
healthlogToken: env.HEALTHLOG_TOKEN ?? "",
|
||||
tag: values.tag ?? null,
|
||||
since: values.since ?? null,
|
||||
kind,
|
||||
kindMap,
|
||||
dryRun,
|
||||
aiRead: values["ai-read"],
|
||||
};
|
||||
if (source === "paperless") {
|
||||
options.baseUrl = trimSlash(values["paperless-url"] ?? env.PAPERLESS_URL);
|
||||
options.sourceToken = env.PAPERLESS_TOKEN;
|
||||
if (!options.baseUrl) throw new StopError("--paperless-url is required.");
|
||||
if (!options.sourceToken)
|
||||
throw new StopError("PAPERLESS_TOKEN is not set.");
|
||||
} else {
|
||||
options.baseUrl = trimSlash(values["papra-url"] ?? env.PAPRA_URL);
|
||||
options.organizationId = values["papra-org"] ?? env.PAPRA_ORG;
|
||||
options.sourceToken = env.PAPRA_TOKEN;
|
||||
if (!options.baseUrl) throw new StopError("--papra-url is required.");
|
||||
if (!options.organizationId) {
|
||||
throw new StopError(
|
||||
"--papra-org is required: the organization id from the Papra address bar.",
|
||||
);
|
||||
}
|
||||
if (!options.sourceToken) throw new StopError("PAPRA_TOKEN is not set.");
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function checkKind(value, flag) {
|
||||
const upper = value.toUpperCase();
|
||||
if (!KINDS.includes(upper)) {
|
||||
throw new StopError(
|
||||
`${flag}: "${value}" is not one of ${KINDS.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
return upper;
|
||||
}
|
||||
|
||||
function trimSlash(url) {
|
||||
return url ? url.replace(/\/+$/, "") : url;
|
||||
}
|
||||
|
||||
// ── HTTP ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function retryAfterSeconds(response, fallback) {
|
||||
const raw = response.headers.get("retry-after");
|
||||
if (!raw) return fallback;
|
||||
const seconds = Number(raw);
|
||||
if (Number.isFinite(seconds))
|
||||
return Math.min(Math.max(0, seconds), MAX_RETRY_AFTER_S);
|
||||
const at = Date.parse(raw);
|
||||
if (Number.isFinite(at)) {
|
||||
return Math.min(
|
||||
Math.max(0, Math.ceil((at - Date.now()) / 1000)),
|
||||
MAX_RETRY_AFTER_S,
|
||||
);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch with the waiting rules shared by all three services: a 429 waits for
|
||||
* Retry-After and tries again (as often as it takes, since that is the
|
||||
* service pacing us), a 5xx or a dropped connection is retried a few times
|
||||
* with a growing pause, and anything else is returned to the caller.
|
||||
*/
|
||||
async function request(label, url, init, makeBody) {
|
||||
let transient = 0;
|
||||
for (;;) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(url, {
|
||||
...init,
|
||||
body: makeBody ? makeBody() : undefined,
|
||||
redirect: "manual",
|
||||
});
|
||||
} catch (cause) {
|
||||
if (transient >= TRANSIENT_ATTEMPTS) {
|
||||
throw new Error(`${label}: could not connect (${cause.message})`);
|
||||
}
|
||||
await wait(TRANSIENT_BACKOFF_MS[transient++], `${label} did not answer`);
|
||||
continue;
|
||||
}
|
||||
if (response.status === 429) {
|
||||
await response.arrayBuffer().catch(() => {});
|
||||
const seconds = retryAfterSeconds(response, 60);
|
||||
await wait(seconds * 1000, `${label} asked to slow down`);
|
||||
continue;
|
||||
}
|
||||
if (response.status >= 500 || response.status === 408) {
|
||||
await response.arrayBuffer().catch(() => {});
|
||||
if (transient >= TRANSIENT_ATTEMPTS) return response;
|
||||
await wait(
|
||||
TRANSIENT_BACKOFF_MS[transient++],
|
||||
`${label} answered ${response.status}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
async function wait(ms, why) {
|
||||
if (ms > 0) out(` … ${why}, waiting ${Math.round(ms / 1000)} s`);
|
||||
await sleep(ms);
|
||||
}
|
||||
|
||||
async function sourceJson(label, url, headers) {
|
||||
const response = await request(label, url, { headers });
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new StopError(
|
||||
`${label} refused the token (${response.status}). Check the token and its permissions.`,
|
||||
);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new StopError(`${label} answered ${response.status} for ${url}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/** Keep the source's own origin: a `next` link behind a proxy may name another. */
|
||||
function sameOrigin(base, link) {
|
||||
const next = new URL(link, base);
|
||||
const origin = new URL(base);
|
||||
next.protocol = origin.protocol;
|
||||
next.host = origin.host;
|
||||
return next.toString();
|
||||
}
|
||||
|
||||
function filenameFrom(response, fallback) {
|
||||
const header = response.headers.get("content-disposition") ?? "";
|
||||
const star = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header);
|
||||
if (star) {
|
||||
try {
|
||||
return decodeURIComponent(star[1].trim().replace(/^"|"$/g, ""));
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
}
|
||||
const plain = /filename="?([^";]+)"?/i.exec(header);
|
||||
return plain ? plain[1] : fallback;
|
||||
}
|
||||
|
||||
// ── Paperless-ngx ──────────────────────────────────────────────────────────
|
||||
|
||||
function paperless(options) {
|
||||
const headers = {
|
||||
Authorization: `Token ${options.sourceToken}`,
|
||||
Accept: "application/json; version=10",
|
||||
};
|
||||
const base = options.baseUrl;
|
||||
const label = "Paperless-ngx";
|
||||
|
||||
async function* pages(firstUrl) {
|
||||
let url = firstUrl;
|
||||
while (url) {
|
||||
const page = await sourceJson(label, url, headers);
|
||||
for (const item of page.results ?? []) yield item;
|
||||
url = page.next ? sameOrigin(base, page.next) : null;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
system: "PAPERLESS",
|
||||
label,
|
||||
async *documents() {
|
||||
const query = new URLSearchParams({
|
||||
page_size: String(PAGE_SIZE),
|
||||
ordering: "id",
|
||||
});
|
||||
let tagId = null;
|
||||
if (options.tag) {
|
||||
const found = [];
|
||||
for await (const tag of pages(
|
||||
`${base}/api/tags/?${new URLSearchParams({ name__iexact: options.tag, page_size: String(PAGE_SIZE) })}`,
|
||||
)) {
|
||||
if (String(tag.name).toLowerCase() === options.tag.toLowerCase())
|
||||
found.push(tag);
|
||||
}
|
||||
if (found.length === 0) {
|
||||
throw new StopError(
|
||||
`Paperless-ngx has no tag named "${options.tag}".`,
|
||||
);
|
||||
}
|
||||
tagId = found[0].id;
|
||||
query.set("tags__id__all", String(tagId));
|
||||
}
|
||||
if (options.since) query.set("created__date__gte", options.since);
|
||||
|
||||
let typeNames = new Map();
|
||||
if (options.kindMap.size > 0) {
|
||||
for await (const type of pages(
|
||||
`${base}/api/document_types/?page_size=${PAGE_SIZE}`,
|
||||
)) {
|
||||
typeNames.set(type.id, String(type.name));
|
||||
}
|
||||
}
|
||||
|
||||
for await (const doc of pages(`${base}/api/documents/?${query}`)) {
|
||||
// The server filters too; checking again keeps an older Paperless
|
||||
// that ignores a parameter from importing the whole archive.
|
||||
if (
|
||||
tagId !== null &&
|
||||
Array.isArray(doc.tags) &&
|
||||
!doc.tags.includes(tagId)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const created = dayOf(doc.created);
|
||||
const added = dayOf(doc.added);
|
||||
yield {
|
||||
sourceId: String(doc.id),
|
||||
title: doc.title || doc.original_file_name || `Paperless ${doc.id}`,
|
||||
date: created ?? added,
|
||||
dateFallback: created ? null : added ? "added" : "none",
|
||||
filename: doc.original_file_name ?? null,
|
||||
kindKeys:
|
||||
doc.document_type != null && typeNames.has(doc.document_type)
|
||||
? [typeNames.get(doc.document_type)]
|
||||
: [],
|
||||
size: null,
|
||||
};
|
||||
}
|
||||
},
|
||||
async size(doc) {
|
||||
const meta = await sourceJson(
|
||||
label,
|
||||
`${base}/api/documents/${encodeURIComponent(doc.sourceId)}/metadata/`,
|
||||
headers,
|
||||
);
|
||||
return typeof meta.original_size === "number" ? meta.original_size : null;
|
||||
},
|
||||
async download(doc) {
|
||||
const response = await request(
|
||||
label,
|
||||
`${base}/api/documents/${encodeURIComponent(doc.sourceId)}/download/?original=true`,
|
||||
{ headers: { Authorization: headers.Authorization } },
|
||||
);
|
||||
return readDownload(label, response, doc);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Papra ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function papra(options) {
|
||||
const headers = {
|
||||
Authorization: `Bearer ${options.sourceToken}`,
|
||||
Accept: "application/json",
|
||||
};
|
||||
const org = `${options.baseUrl}/api/organizations/${encodeURIComponent(options.organizationId)}`;
|
||||
const label = "Papra";
|
||||
|
||||
return {
|
||||
system: "PAPRA",
|
||||
label,
|
||||
async *documents() {
|
||||
let tagId = null;
|
||||
if (options.tag) {
|
||||
const body = await sourceJson(label, `${org}/tags`, headers);
|
||||
const tag = (body.tags ?? []).find(
|
||||
(t) => String(t.name).toLowerCase() === options.tag.toLowerCase(),
|
||||
);
|
||||
if (!tag)
|
||||
throw new StopError(`Papra has no tag named "${options.tag}".`);
|
||||
tagId = tag.id;
|
||||
}
|
||||
const search = options.tag
|
||||
? `tag:${/\s/.test(options.tag) ? JSON.stringify(options.tag) : options.tag}`
|
||||
: "";
|
||||
for (let pageIndex = 0; ; pageIndex++) {
|
||||
const query = new URLSearchParams({
|
||||
pageIndex: String(pageIndex),
|
||||
pageSize: String(PAGE_SIZE),
|
||||
});
|
||||
if (search) query.set("searchQuery", search);
|
||||
const body = await sourceJson(
|
||||
label,
|
||||
`${org}/documents?${query}`,
|
||||
headers,
|
||||
);
|
||||
const documents = body.documents ?? [];
|
||||
for (const doc of documents) {
|
||||
const tags = Array.isArray(doc.tags) ? doc.tags : null;
|
||||
if (tagId !== null && tags && !tags.some((t) => t.id === tagId))
|
||||
continue;
|
||||
const documentDate = dayOf(doc.documentDate);
|
||||
const created = dayOf(doc.createdAt);
|
||||
const date = documentDate ?? created;
|
||||
// Papra's search dates differ between versions; the date is
|
||||
// checked here instead of trusting a query syntax.
|
||||
if (options.since && date && date < options.since) continue;
|
||||
yield {
|
||||
sourceId: String(doc.id),
|
||||
title: stripExtension(doc.name) || `Papra ${doc.id}`,
|
||||
date,
|
||||
dateFallback: documentDate ? null : created ? "added" : "none",
|
||||
filename: doc.originalName ?? doc.name ?? null,
|
||||
kindKeys: (tags ?? []).map((t) => String(t.name)),
|
||||
size:
|
||||
typeof doc.originalSize === "number" ? doc.originalSize : null,
|
||||
};
|
||||
}
|
||||
const total =
|
||||
typeof body.documentsCount === "number" ? body.documentsCount : null;
|
||||
if (documents.length < PAGE_SIZE) break;
|
||||
if (total !== null && (pageIndex + 1) * PAGE_SIZE >= total) break;
|
||||
}
|
||||
},
|
||||
async size(doc) {
|
||||
return doc.size;
|
||||
},
|
||||
async download(doc) {
|
||||
const response = await request(
|
||||
label,
|
||||
`${org}/documents/${encodeURIComponent(doc.sourceId)}/file`,
|
||||
{ headers: { Authorization: headers.Authorization } },
|
||||
);
|
||||
return readDownload(label, response, doc);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readDownload(label, response, doc) {
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new StopError(`${label} refused the download (${response.status}).`);
|
||||
}
|
||||
if (!response.ok) {
|
||||
return { skip: `${label} could not deliver the file (${response.status})` };
|
||||
}
|
||||
const bytes = new Uint8Array(await response.arrayBuffer());
|
||||
return {
|
||||
bytes,
|
||||
filename:
|
||||
doc.filename ?? filenameFrom(response, `document-${doc.sourceId}`),
|
||||
type: response.headers.get("content-type") ?? "application/octet-stream",
|
||||
};
|
||||
}
|
||||
|
||||
function dayOf(value) {
|
||||
if (typeof value !== "string") return null;
|
||||
const day = value.slice(0, 10);
|
||||
return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null;
|
||||
}
|
||||
|
||||
function stripExtension(name) {
|
||||
if (typeof name !== "string") return "";
|
||||
return name.replace(/\.[A-Za-z0-9]{1,5}$/, "");
|
||||
}
|
||||
|
||||
// ── HealthLog ──────────────────────────────────────────────────────────────
|
||||
|
||||
function kindFor(options, doc) {
|
||||
for (const key of doc.kindKeys) {
|
||||
const mapped = options.kindMap.get(key.toLowerCase());
|
||||
if (mapped) return mapped;
|
||||
}
|
||||
return options.kind;
|
||||
}
|
||||
|
||||
async function upload(options, source, doc, file) {
|
||||
const url = `${options.healthlogUrl}/api/documents/inbound`;
|
||||
const kind = kindFor(options, doc);
|
||||
const response = await request(
|
||||
"HealthLog",
|
||||
url,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { Authorization: `Bearer ${options.healthlogToken}` },
|
||||
},
|
||||
// Rebuilt per attempt: a body stream is spent once it has been sent.
|
||||
() => {
|
||||
const form = new FormData();
|
||||
form.append(
|
||||
"file",
|
||||
new Blob([file.bytes], { type: file.type }),
|
||||
file.filename,
|
||||
);
|
||||
form.append("title", doc.title.trim().slice(0, TITLE_MAX));
|
||||
if (doc.date) form.append("documentDate", doc.date);
|
||||
if (kind) form.append("kind", kind);
|
||||
form.append("sourceSystem", source.system);
|
||||
form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX));
|
||||
if (!options.aiRead) form.append("aiRead", "defer");
|
||||
return form;
|
||||
},
|
||||
);
|
||||
|
||||
let body = null;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
body = null;
|
||||
}
|
||||
const meta = body?.meta ?? {};
|
||||
switch (response.status) {
|
||||
case 201:
|
||||
return { outcome: "imported" };
|
||||
case 200:
|
||||
return body?.data?.deleted
|
||||
? { outcome: "deleted" }
|
||||
: { outcome: "duplicate" };
|
||||
case 401:
|
||||
throw new StopError(
|
||||
"HealthLog refused the token (401). It may be expired or revoked; create a new document token.",
|
||||
);
|
||||
case 403:
|
||||
if (meta.errorCode === "module.disabled") {
|
||||
throw new StopError(
|
||||
"The document vault is switched off in HealthLog. Turn on Documents in Settings, then run the script again.",
|
||||
);
|
||||
}
|
||||
throw new StopError(
|
||||
"HealthLog refused the upload (403). Use a document token from Settings > API & Tokens; other tokens cannot upload documents.",
|
||||
);
|
||||
case 413:
|
||||
if (meta.reason === "quotaExceeded") {
|
||||
throw new StopError(
|
||||
`Your HealthLog storage is full (${mib(meta.usedBytes)} of ${mib(meta.quotaBytes)} used). An admin can raise the limit in the admin area, for everyone or just for your account.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
outcome: "skipped",
|
||||
reason: `larger than HealthLog accepts${meta.maxFileBytes ? ` (${mib(meta.maxFileBytes)})` : ""}`,
|
||||
};
|
||||
case 415:
|
||||
return {
|
||||
outcome: "skipped",
|
||||
reason: "HealthLog does not accept this file type",
|
||||
};
|
||||
case 422:
|
||||
return {
|
||||
outcome: "skipped",
|
||||
reason: `HealthLog refused the details: ${body?.error ?? "invalid"}`,
|
||||
};
|
||||
default:
|
||||
return {
|
||||
outcome: "skipped",
|
||||
reason: `HealthLog answered ${response.status}${body?.error ? `: ${body.error}` : ""}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function mib(bytes) {
|
||||
if (typeof bytes !== "number") return "?";
|
||||
return bytes >= 1024 * 1024 * 1024
|
||||
? `${(bytes / 1024 ** 3).toFixed(1)} GB`
|
||||
: `${(bytes / 1024 ** 2).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
// ── Run ────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function run(argv, env = process.env) {
|
||||
let options;
|
||||
try {
|
||||
options = readOptions(argv, env);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof StopError ||
|
||||
error?.code === "ERR_PARSE_ARGS_UNKNOWN_OPTION"
|
||||
) {
|
||||
err(error.message);
|
||||
return 2;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (options.help) {
|
||||
out(usage());
|
||||
return 0;
|
||||
}
|
||||
|
||||
const source =
|
||||
options.source === "paperless" ? paperless(options) : papra(options);
|
||||
const tally = {
|
||||
imported: 0,
|
||||
duplicate: 0,
|
||||
deleted: 0,
|
||||
skipped: [],
|
||||
fallback: [],
|
||||
};
|
||||
let listed = 0;
|
||||
let totalBytes = 0;
|
||||
let unknownSizes = 0;
|
||||
|
||||
out(
|
||||
`${options.dryRun ? "Dry run: listing" : "Importing"} documents from ${source.label}` +
|
||||
`${options.tag ? ` tagged "${options.tag}"` : ""}${options.since ? ` since ${options.since}` : ""}.`,
|
||||
);
|
||||
|
||||
try {
|
||||
for await (const doc of source.documents()) {
|
||||
listed++;
|
||||
const where = `${source.label} ${doc.sourceId} "${doc.title}"`;
|
||||
if (doc.dateFallback)
|
||||
tally.fallback.push({ where, dateFallback: doc.dateFallback });
|
||||
|
||||
if (options.dryRun) {
|
||||
const size = await source.size(doc);
|
||||
if (size === null) unknownSizes++;
|
||||
else totalBytes += size;
|
||||
const kind = kindFor(options, doc) ?? "OTHER";
|
||||
out(
|
||||
` ${doc.date ?? "no date"} ${kind.padEnd(16)} ${size === null ? "?" : mib(size)} ${where}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const file = await source.download(doc);
|
||||
if (file.skip) {
|
||||
tally.skipped.push({ where, reason: file.skip });
|
||||
out(` skipped ${where}: ${file.skip}`);
|
||||
continue;
|
||||
}
|
||||
const result = await upload(options, source, doc, file);
|
||||
if (result.outcome === "imported") {
|
||||
tally.imported++;
|
||||
out(` imported ${where}`);
|
||||
} else if (result.outcome === "duplicate") {
|
||||
tally.duplicate++;
|
||||
out(` present ${where}`);
|
||||
} else if (result.outcome === "deleted") {
|
||||
tally.deleted++;
|
||||
out(` deleted ${where} (you deleted it in HealthLog; left alone)`);
|
||||
} else {
|
||||
tally.skipped.push({ where, reason: result.reason });
|
||||
out(` skipped ${where}: ${result.reason}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
printSummary(options, tally, listed, totalBytes, unknownSizes);
|
||||
err("");
|
||||
err(`Stopped: ${error.message}`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
printSummary(options, tally, listed, totalBytes, unknownSizes);
|
||||
return tally.skipped.length > 0 ? 3 : 0;
|
||||
}
|
||||
|
||||
function printSummary(options, tally, listed, totalBytes, unknownSizes) {
|
||||
out("");
|
||||
if (options.dryRun) {
|
||||
out(
|
||||
`${listed} document(s) would be imported, about ${mib(totalBytes)}` +
|
||||
`${unknownSizes > 0 ? ` (size unknown for ${unknownSizes})` : ""}.`,
|
||||
);
|
||||
out(
|
||||
"HealthLog's default storage limit is 1 GB per person; an admin can raise it.",
|
||||
);
|
||||
out(
|
||||
"Documents already in HealthLog, or deleted there, will be skipped by HealthLog.",
|
||||
);
|
||||
} else {
|
||||
out(`Imported: ${tally.imported}`);
|
||||
out(`Already in HealthLog: ${tally.duplicate}`);
|
||||
out(`Deleted in HealthLog, left alone: ${tally.deleted}`);
|
||||
out(`Skipped: ${tally.skipped.length}`);
|
||||
for (const { where, reason } of tally.skipped) out(` ${where}: ${reason}`);
|
||||
}
|
||||
if (tally.fallback.length > 0) {
|
||||
out(
|
||||
`No document date in the source for ${tally.fallback.length} document(s):`,
|
||||
);
|
||||
for (const { where, dateFallback } of tally.fallback) {
|
||||
out(
|
||||
dateFallback === "added"
|
||||
? ` ${where}: filed under the day it was added to the source`
|
||||
: ` ${where}: filed under the day of import`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!options.dryRun && tally.imported > 0 && !options.aiRead) {
|
||||
out(
|
||||
`${tally.imported} document(s) imported without automatic AI reading. ` +
|
||||
"To have one read, open it in HealthLog and choose Read with AI or Generate summary; " +
|
||||
"Documents > Index all for search reads the ones that had no text to search.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const SELF_DOC = await (async () => {
|
||||
const { readFile } = await import("node:fs/promises");
|
||||
return readFile(new URL(import.meta.url), "utf8");
|
||||
})();
|
||||
|
||||
process.exitCode = await run(process.argv.slice(2));
|
||||
Reference in new issue
Block a user