Merge branch 'feat/document-import-token' into release/v1.39.2

scripts/__tests__/import-documents.test.ts

@@ -86,12 +86,40 @@ async function healthlog(
     deleted?: string[];
     tooLarge?: string[];
     moduleOff?: boolean;
+    /** An older HealthLog with no lookup route. */
+    noLookup?: boolean;
+    /** The address redirects elsewhere (http → https, a moved host). */
+    redirect?: string;
   } = {},
 ) {
   const stored = new Set<string>();
   const received: Received[] = [];
+  const lookups: string[] = [];
   let throttled = false;
   const url = await serve(async (req, url, body) => {
+    if (opts.redirect) {
+      return {
+        status: 301,
+        headers: { location: `${opts.redirect}${url.pathname}` },
+      };
+    }
+    const queryKey = `${url.searchParams.get("sourceSystem")}:${url.searchParams.get("sourceId")}`;
+    if (
+      url.pathname === "/api/documents/inbound/source" &&
+      req.method === "GET" &&
+      !opts.noLookup
+    ) {
+      lookups.push(queryKey);
+      const deleted = opts.deleted?.includes(queryKey) ?? false;
+      const known = deleted || stored.has(queryKey);
+      return {
+        status: 200,
+        json: {
+          data: { known, id: known ? queryKey : null, deleted },
+          error: null,
+        },
+      };
+    }
     if (url.pathname !== "/api/documents/inbound" || req.method !== "POST") {
       return { status: 404, json: { data: null, error: "Not found" } };
     }
@@ -130,8 +158,11 @@ async function healthlog(
         size = v.size;
       }
     }
+    // The key rides the query string, where HealthLog reads it first.
+    fields.sourceSystem = url.searchParams.get("sourceSystem") ?? "";
+    fields.sourceId = url.searchParams.get("sourceId") ?? "";
     received.push({ auth: req.headers.authorization, fields, filename, size });
-    const key = `${fields.sourceSystem}:${fields.sourceId}`;
+    const key = queryKey;
     if (opts.tooLarge?.includes(key)) {
       return {
         status: 413,
@@ -168,7 +199,7 @@ async function healthlog(
       json: { data: { id: key, duplicate: false }, error: null },
     };
   });
-  return { url, received };
+  return { url, received, lookups };
 }
 
 /** Paperless-ngx v10 stand-in: two pages of documents, one outside the tag. */
@@ -238,7 +269,8 @@ async function paperless() {
             },
             {
               id: 2,
-              title: "Arztbrief",
+              // Blank in the source: the file name stands in.
+              title: "   ",
               created: null,
               added: "2024-04-01T09:30:00Z",
               tags: [7],
@@ -379,6 +411,7 @@ describe("import-documents.mjs — Paperless-ngx", () => {
     });
     // No created date: filed under the day it was added, and said so.
     expect(hl.received[1].fields.documentDate).toBe("2024-04-01");
+    expect(hl.received[1].fields.title).toBe("brief");
     expect(hl.received[1].fields.kind).toBeUndefined();
     expect(first.stdout).toMatch(/Imported: 2/);
     expect(first.stdout).toMatch(/filed under the day it was added/);
@@ -387,7 +420,8 @@ describe("import-documents.mjs — Paperless-ngx", () => {
     // The API contract the script leans on.
     const docs = pl.seen.filter((s) => s.path.startsWith("/api/documents/?"));
     expect(docs[0].path).toContain("tags__id__all=7");
-    expect(docs[0].accept).toBe("application/json; version=10");
+    // API v9: the lowest the current Paperless-ngx releases accept.
+    expect(docs[0].accept).toBe("application/json; version=9");
     expect(docs[0].auth).toBe("Token pl_test");
     // `next` was followed on the source's own origin.
     expect(docs[1].path).toContain("page=2");
@@ -398,11 +432,20 @@ describe("import-documents.mjs — Paperless-ngx", () => {
       "/api/documents/2/download/?original=true",
     ]);
 
-    // A second run changes nothing: HealthLog answers duplicate for both.
+    // A second run changes nothing: HealthLog already holds both, and the
+    // lookup says so before anything is downloaded or uploaded again.
+    const downloadsBefore = pl.seen.filter((s) =>
+      s.path.includes("/download/"),
+    ).length;
     const second = await runScript(args, env);
     expect(second.code).toBe(0);
     expect(second.stdout).toMatch(/Imported: 0/);
     expect(second.stdout).toMatch(/Already in HealthLog: 2/);
+    expect(pl.seen.filter((s) => s.path.includes("/download/")).length).toBe(
+      downloadsBefore,
+    );
+    expect(hl.received).toHaveLength(2);
+    expect(hl.lookups).toContain("PAPERLESS:1");
   });
 
   it("leaves a document deleted in HealthLog alone", async () => {
@@ -481,6 +524,73 @@ describe("import-documents.mjs — Paperless-ngx", () => {
   });
 });
 
+describe("import-documents.mjs — edges", () => {
+  const env = { HEALTHLOG_TOKEN: "hlk_test", PAPERLESS_TOKEN: "pl_test" };
+
+  it("still imports against a HealthLog without the lookup", async () => {
+    const hl = await healthlog({ noLookup: true });
+    const pl = await paperless();
+    const res = await runScript(
+      [
+        "paperless",
+        "--paperless-url",
+        pl.url,
+        "--healthlog-url",
+        hl.url,
+        "--tag",
+        "HealthLog",
+      ],
+      env,
+    );
+    expect(res.code).toBe(0);
+    expect(res.stdout).toMatch(/Imported: 2/);
+  });
+
+  it("stops on a redirecting HealthLog address and names where it points", async () => {
+    const hl = await healthlog({ redirect: "https://health.example" });
+    const pl = await paperless();
+    const res = await runScript(
+      [
+        "paperless",
+        "--paperless-url",
+        pl.url,
+        "--healthlog-url",
+        hl.url,
+        "--tag",
+        "HealthLog",
+      ],
+      env,
+    );
+    expect(res.code).toBe(1);
+    expect(res.stderr).toMatch(/redirects to https:\/\/health\.example/);
+  });
+
+  it("applies --since itself when the server ignores the filter", async () => {
+    const hl = await healthlog();
+    const pl = await paperless();
+    const res = await runScript(
+      [
+        "paperless",
+        "--paperless-url",
+        pl.url,
+        "--healthlog-url",
+        hl.url,
+        "--tag",
+        "HealthLog",
+        "--since",
+        "2024-04-01",
+      ],
+      env,
+    );
+    expect(res.code).toBe(0);
+    // Document 1 (created 2024-03-05) is before the cut and never fetched.
+    expect(hl.received.map((r) => r.fields.sourceId)).toEqual(["2"]);
+    expect(pl.seen.some((s) => s.path.startsWith("/api/documents/1/"))).toBe(
+      false,
+    );
+  });
+});
+
 describe("import-documents.mjs — Papra", () => {
   it("imports by tag since a date, and skips what HealthLog refuses as too large", async () => {
     const hl = await healthlog({ tooLarge: ["PAPRA:doc_b"] });

scripts/import-documents.mjs

@@ -283,7 +283,9 @@ function filenameFrom(response, fallback) {
 function paperless(options) {
   const headers = {
     Authorization: `Token ${options.sourceToken}`,
-    Accept: "application/json; version=10",
+    // API version 9 (Paperless-ngx 2.16 and later): the lowest the current
+    // servers still accept, and the one where `created` became a plain date.
+    Accept: "application/json; version=9",
   };
   const base = options.baseUrl;
   const label = "Paperless-ngx";
@@ -345,10 +347,17 @@ function paperless(options) {
         }
         const created = dayOf(doc.created);
         const added = dayOf(doc.added);
+        const date = created ?? added;
+        // Checked here as well: an older server may ignore the date filter.
+        if (options.since && date && date < options.since) continue;
         yield {
           sourceId: String(doc.id),
-          title: doc.title || doc.original_file_name || `Paperless ${doc.id}`,
-          date: created ?? added,
+          title: titleOf(
+            doc.title,
+            doc.original_file_name,
+            `Paperless ${doc.id}`,
+          ),
+          date,
           dateFallback: created ? null : added ? "added" : "none",
           filename: doc.original_file_name ?? null,
           kindKeys:
@@ -429,7 +438,11 @@ function papra(options) {
           if (options.since && date && date < options.since) continue;
           yield {
             sourceId: String(doc.id),
-            title: stripExtension(doc.name) || `Papra ${doc.id}`,
+            title: titleOf(
+              stripExtension(doc.name),
+              doc.originalName,
+              `Papra ${doc.id}`,
+            ),
             date,
             dateFallback: documentDate ? null : created ? "added" : "none",
             filename: doc.originalName ?? doc.name ?? null,
@@ -480,6 +493,19 @@ function dayOf(value) {
   return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : null;
 }
 
+/**
+ * A title that is not blank: the source's title, else its file name without
+ * the extension, else a placeholder naming the source id. HealthLog refuses a
+ * blank title.
+ */
+function titleOf(title, filename, fallback) {
+  return (
+    (typeof title === "string" ? title.trim() : "") ||
+    stripExtension(filename).trim() ||
+    fallback
+  );
+}
+
 function stripExtension(name) {
   if (typeof name !== "string") return "";
   return name.replace(/\.[A-Za-z0-9]{1,5}$/, "");
@@ -495,8 +521,68 @@ function kindFor(options, doc) {
   return options.kind;
 }
 
+/** The source key as query parameters, which HealthLog reads before the body. */
+function keyQuery(source, doc) {
+  return new URLSearchParams({
+    sourceSystem: source.system,
+    sourceId: doc.sourceId.slice(0, SOURCE_ID_MAX),
+  });
+}
+
+async function readBody(response) {
+  try {
+    return await response.json();
+  } catch {
+    return null;
+  }
+}
+
+/** Refusals that end the run, whichever HealthLog request met them. */
+function stopOnRefusal(response, body, url) {
+  if (response.status >= 300 && response.status < 400) {
+    const location = response.headers.get("location");
+    throw new StopError(
+      `The HealthLog address redirects${location ? ` to ${new URL(location, url).origin}` : ""}. Use that address with --healthlog-url.`,
+    );
+  }
+  if (response.status === 401) {
+    throw new StopError(
+      "HealthLog refused the token (401). It may be expired or revoked; create a new document token.",
+    );
+  }
+  if (response.status === 403) {
+    if (body?.meta?.errorCode === "module.disabled") {
+      throw new StopError(
+        "The document vault is switched off in HealthLog. Turn on Documents in Settings, then run the script again.",
+      );
+    }
+    throw new StopError(
+      "HealthLog refused the request (403). Use a document token from Settings > API & Tokens; other tokens cannot upload documents.",
+    );
+  }
+}
+
+/**
+ * Ask HealthLog whether it already holds this document, before downloading
+ * it from the source. Null when the answer is "no", or when the HealthLog
+ * version has no lookup yet (the upload then decides, as before).
+ */
+async function lookup(options, source, doc) {
+  const url = `${options.healthlogUrl}/api/documents/inbound/source?${keyQuery(source, doc)}`;
+  const response = await request("HealthLog", url, {
+    headers: {
+      Authorization: `Bearer ${options.healthlogToken}`,
+      Accept: "application/json",
+    },
+  });
+  const body = await readBody(response);
+  stopOnRefusal(response, body, url);
+  if (!response.ok || !body?.data?.known) return null;
+  return body.data.deleted ? { outcome: "deleted" } : { outcome: "duplicate" };
+}
+
 async function upload(options, source, doc, file) {
-  const url = `${options.healthlogUrl}/api/documents/inbound`;
+  const url = `${options.healthlogUrl}/api/documents/inbound?${keyQuery(source, doc)}`;
   const kind = kindFor(options, doc);
   const response = await request(
     "HealthLog",
@@ -513,22 +599,16 @@ async function upload(options, source, doc, file) {
         new Blob([file.bytes], { type: file.type }),
         file.filename,
       );
-      form.append("title", doc.title.trim().slice(0, TITLE_MAX));
+      form.append("title", doc.title.slice(0, TITLE_MAX));
       if (doc.date) form.append("documentDate", doc.date);
       if (kind) form.append("kind", kind);
-      form.append("sourceSystem", source.system);
-      form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX));
       if (!options.aiRead) form.append("aiRead", "defer");
       return form;
     },
   );
 
-  let body = null;
-  try {
-    body = await response.json();
-  } catch {
-    body = null;
-  }
+  const body = await readBody(response);
+  stopOnRefusal(response, body, url);
   const meta = body?.meta ?? {};
   switch (response.status) {
     case 201:
@@ -537,23 +617,10 @@ async function upload(options, source, doc, file) {
       return body?.data?.deleted
         ? { outcome: "deleted" }
         : { outcome: "duplicate" };
-    case 401:
-      throw new StopError(
-        "HealthLog refused the token (401). It may be expired or revoked; create a new document token.",
-      );
-    case 403:
-      if (meta.errorCode === "module.disabled") {
-        throw new StopError(
-          "The document vault is switched off in HealthLog. Turn on Documents in Settings, then run the script again.",
-        );
-      }
-      throw new StopError(
-        "HealthLog refused the upload (403). Use a document token from Settings > API & Tokens; other tokens cannot upload documents.",
-      );
     case 413:
       if (meta.reason === "quotaExceeded") {
         throw new StopError(
-          `Your HealthLog storage is full (${mib(meta.usedBytes)} of ${mib(meta.quotaBytes)} used). An admin can raise the limit in the admin area, for everyone or just for your account.`,
+          "Your HealthLog storage is full. An admin can raise the limit in the admin area, for everyone or just for your account.",
         );
       }
       return {
@@ -642,6 +709,19 @@ async function run(argv, env = process.env) {
         continue;
       }
 
+      // Asked first so a document HealthLog already holds is not downloaded.
+      const known = await lookup(options, source, doc);
+      if (known) {
+        if (known.outcome === "duplicate") {
+          tally.duplicate++;
+          out(`  present   ${where}`);
+        } else {
+          tally.deleted++;
+          out(`  deleted   ${where} (you deleted it in HealthLog; left alone)`);
+        }
+        continue;
+      }
+
       const file = await source.download(doc);
       if (file.skip) {
         tally.skipped.push({ where, reason: file.skip });