fix(scripts): fall through on spent lookups, send the key in both places

A 429 on the lookup no longer waits for the Retry-After; the keyed upload
answers the same question. The source key goes in the query string and the
form. The closing summary and the guide point to Read with AI for deferred
documents instead of Index all.

Refs #1038

scripts/__tests__/import-documents.test.ts

@@ -90,6 +90,8 @@ async function healthlog(
     noLookup?: boolean;
     /** The address redirects elsewhere (http → https, a moved host). */
     redirect?: string;
+    /** Lookups are spent: every lookup answers 429 with a long Retry-After. */
+    lookup429?: boolean;
   } = {},
 ) {
   const stored = new Set<string>();
@@ -110,6 +112,13 @@ async function healthlog(
       !opts.noLookup
     ) {
       lookups.push(queryKey);
+      if (opts.lookup429) {
+        return {
+          status: 429,
+          headers: { "retry-after": "3600" },
+          json: { data: null, error: "Too many lookups." },
+        };
+      }
       const deleted = opts.deleted?.includes(queryKey) ?? false;
       const known = deleted || stored.has(queryKey);
       return {
@@ -158,9 +167,17 @@ async function healthlog(
         size = v.size;
       }
     }
-    // The key rides the query string, where HealthLog reads it first.
-    fields.sourceSystem = url.searchParams.get("sourceSystem") ?? "";
-    fields.sourceId = url.searchParams.get("sourceId") ?? "";
+    // The key rides the query string, where HealthLog reads it first, and
+    // the form; HealthLog refuses the two disagreeing.
+    if (
+      fields.sourceSystem !== url.searchParams.get("sourceSystem") ||
+      fields.sourceId !== url.searchParams.get("sourceId")
+    ) {
+      return {
+        status: 422,
+        json: { data: null, error: "source key mismatch" },
+      };
+    }
     received.push({ auth: req.headers.authorization, fields, filename, size });
     const key = queryKey;
     if (opts.tooLarge?.includes(key)) {
@@ -546,6 +563,28 @@ describe("import-documents.mjs — edges", () => {
     expect(res.stdout).toMatch(/Imported: 2/);
   });
 
+  it("does not wait on spent lookups: the keyed upload answers instead", async () => {
+    const hl = await healthlog({ lookup429: true });
+    const pl = await paperless();
+    const started = Date.now();
+    const res = await runScript(
+      [
+        "paperless",
+        "--paperless-url",
+        pl.url,
+        "--healthlog-url",
+        hl.url,
+        "--tag",
+        "HealthLog",
+      ],
+      env,
+    );
+    expect(res.code).toBe(0);
+    expect(res.stdout).toMatch(/Imported: 2/);
+    // A Retry-After of an hour was not honoured for a lookup.
+    expect(Date.now() - started).toBeLessThan(20_000);
+  });
+
   it("stops on a redirecting HealthLog address and names where it points", async () => {
     const hl = await healthlog({ redirect: "https://health.example" });
     const pl = await paperless();

scripts/import-documents.mjs

@@ -201,7 +201,7 @@ function retryAfterSeconds(response, fallback) {
  * service pacing us), a 5xx or a dropped connection is retried a few times
  * with a growing pause, and anything else is returned to the caller.
  */
-async function request(label, url, init, makeBody) {
+async function request(label, url, init, makeBody, { waitOn429 = true } = {}) {
   let transient = 0;
   for (;;) {
     let response;
@@ -218,7 +218,7 @@ async function request(label, url, init, makeBody) {
       await wait(TRANSIENT_BACKOFF_MS[transient++], `${label} did not answer`);
       continue;
     }
-    if (response.status === 429) {
+    if (response.status === 429 && waitOn429) {
       await response.arrayBuffer().catch(() => {});
       const seconds = retryAfterSeconds(response, 60);
       await wait(seconds * 1000, `${label} asked to slow down`);
@@ -569,12 +569,20 @@ function stopOnRefusal(response, body, url) {
  */
 async function lookup(options, source, doc) {
   const url = `${options.healthlogUrl}/api/documents/inbound/source?${keyQuery(source, doc)}`;
-  const response = await request("HealthLog", url, {
-    headers: {
-      Authorization: `Bearer ${options.healthlogToken}`,
-      Accept: "application/json",
+  // Not worth waiting for: if the lookups are spent, the upload with the key
+  // answers the same question.
+  const response = await request(
+    "HealthLog",
+    url,
+    {
+      headers: {
+        Authorization: `Bearer ${options.healthlogToken}`,
+        Accept: "application/json",
+      },
     },
-  });
+    undefined,
+    { waitOn429: false },
+  );
   const body = await readBody(response);
   stopOnRefusal(response, body, url);
   if (!response.ok || !body?.data?.known) return null;
@@ -603,6 +611,9 @@ async function upload(options, source, doc, file) {
       if (doc.date) form.append("documentDate", doc.date);
       if (kind) form.append("kind", kind);
       if (!options.aiRead) form.append("aiRead", "defer");
+      // In the form as well as the address; HealthLog checks they agree.
+      form.append("sourceSystem", source.system);
+      form.append("sourceId", doc.sourceId.slice(0, SOURCE_ID_MAX));
       return form;
     },
   );
@@ -789,8 +800,7 @@ function printSummary(options, tally, listed, totalBytes, unknownSizes) {
   if (!options.dryRun && tally.imported > 0 && !options.aiRead) {
     out(
       `${tally.imported} document(s) imported without automatic AI reading. ` +
-        "To have one read, open it in HealthLog and choose Read with AI or Generate summary; " +
-        "Documents > Index all for search reads the ones that had no text to search.",
+        "To have one read, open it in HealthLog and choose Read with AI or Generate summary.",
     );
   }
 }