Merge branch 'feat/document-import-token' into release/v1.39.2

# Conflicts:
#	docs/api/openapi.yaml
#	docs/integrations/document-import.md

scripts/__tests__/import-documents.test.ts

@@ -90,6 +90,8 @@ async function healthlog(
     noLookup?: boolean;
     /** The address redirects elsewhere (http → https, a moved host). */
     redirect?: string;
+    /** Lookups are spent: every lookup answers 429 with a long Retry-After. */
+    lookup429?: boolean;
   } = {},
 ) {
   const stored = new Set<string>();
@@ -110,6 +112,13 @@ async function healthlog(
       !opts.noLookup
     ) {
       lookups.push(queryKey);
+      if (opts.lookup429) {
+        return {
+          status: 429,
+          headers: { "retry-after": "3600" },
+          json: { data: null, error: "Too many lookups." },
+        };
+      }
       const deleted = opts.deleted?.includes(queryKey) ?? false;
       const known = deleted || stored.has(queryKey);
       return {
@@ -158,9 +167,17 @@ async function healthlog(
         size = v.size;
       }
     }
-    // The key rides the query string, where HealthLog reads it first.
-    fields.sourceSystem = url.searchParams.get("sourceSystem") ?? "";
-    fields.sourceId = url.searchParams.get("sourceId") ?? "";
+    // The key rides the query string, where HealthLog reads it first, and
+    // the form; HealthLog refuses the two disagreeing.
+    if (
+      fields.sourceSystem !== url.searchParams.get("sourceSystem") ||
+      fields.sourceId !== url.searchParams.get("sourceId")
+    ) {
+      return {
+        status: 422,
+        json: { data: null, error: "source key mismatch" },
+      };
+    }
     received.push({ auth: req.headers.authorization, fields, filename, size });
     const key = queryKey;
     if (opts.tooLarge?.includes(key)) {
@@ -546,6 +563,28 @@ describe("import-documents.mjs — edges", () => {
     expect(res.stdout).toMatch(/Imported: 2/);
   });
 
+  it("does not wait on spent lookups: the keyed upload answers instead", async () => {
+    const hl = await healthlog({ lookup429: true });
+    const pl = await paperless();
+    const started = Date.now();
+    const res = await runScript(
+      [
+        "paperless",
+        "--paperless-url",
+        pl.url,
+        "--healthlog-url",
+        hl.url,
+        "--tag",
+        "HealthLog",
+      ],
+      env,
+    );
+    expect(res.code).toBe(0);
+    expect(res.stdout).toMatch(/Imported: 2/);
+    // A Retry-After of an hour was not honoured for a lookup.
+    expect(Date.now() - started).toBeLessThan(20_000);
+  });
+
   it("stops on a redirecting HealthLog address and names where it points", async () => {
     const hl = await healthlog({ redirect: "https://health.example" });
     const pl = await paperless();

scripts/import-documents.mjs

@@ -201,7 +201,7 @@ function retryAfterSeconds(response, fallback) {
  * service pacing us), a 5xx or a dropped connection is retried a few times
  * with a growing pause, and anything else is returned to the caller.
  */
-async function request(label, url, init, makeBody) {
+async function request(label, url, init, makeBody, { waitOn429 = true } = {}) {
   let transient = 0;
   for (;;) {
     let response;
@@ -218,7 +218,7 @@ async function request(label, url, init, makeBody) {
       await wait(TRANSIENT_BACKOFF_MS[transient++], `${label} did not answer`);
       continue;
     }
-    if (response.status === 429) {
+    if (response.status === 429 && waitOn429) {
       await response.arrayBuffer().catch(() => {});
       const seconds = retryAfterSeconds(response, 60);
       await wait(seconds * 1000, `${label} asked to slow down`);
@@ -569,12 +569,20 @@ function stopOnRefusal(response, body, url) {
  */
 async function lookup(options, source, doc) {
   const url = `${options.healthlogUrl}/api/documents/inbound/source?${keyQuery(source, doc)}`;
-  const response = await request("HealthLog", url, {
-    headers: {
-      Authorization: `Bearer ${options.healthlogToken}`,
-      Accept: "application/json",
+  // Not worth waiting for: if the lookups are spent, the upload with the key
+  // answers the same question.
+  const response = await request(
+    "HealthLog",
+    url,
+    {
+      headers: {
+        Authorization: `Bearer ${options.healthlogToken}`,
+        Accept: "application/json",
+      },
     },
-  });
+    undefined,
+    { waitOn429: false },
+  );
   const body = await readBody(response);
   stopOnRefusal(response, body, url);
   if (!response.ok || !body?.data?.known) return null;
@@ -603,6 +611,9 @@ async function upload(options, source, doc, file) {
       if (doc.date) form.append("documentDate", doc.date);
       if (kind) form.append("kind", kind);
       if (!options.aiRead) form.append("aiRead", "defer");
+      // In the form as well as the address; HealthLog checks they agree.
+      form.append("sourceSystem", source.system);
+      form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX));
       return form;
     },
   );
@@ -789,8 +800,7 @@ function printSummary(options, tally, listed, totalBytes, unknownSizes) {
   if (!options.dryRun && tally.imported > 0 && !options.aiRead) {
     out(
       `${tally.imported} document(s) imported without automatic AI reading. ` +
-        "To have one read, open it in HealthLog and choose Read with AI or Generate summary; " +
-        "Documents > Index all for search reads the ones that had no text to search.",
+        "To have one read, open it in HealthLog and choose Read with AI or Generate summary.",
     );
   }
 }