perf(jobs): stop the geo backfill from retrying the same rows hourly

An audit row whose address no database can place stayed without a
location, so every hourly pass looked it up again for its whole 30-day
window, and finding it walked every audit row of those 30 days through
the created_at index.

Migration 0418 adds audit_logs.geo_attempted_at and a partial index on
the rows still missing a location or carrier. Every row a pass looks at
is stamped, and a row tried less than seven days ago is not picked
again, which still lets a range a later GeoLite2 release learns resolve.
The candidate read goes through the partial index, which holds only the
backlog.

src/lib/jobs/__tests__/geo-backfill.test.ts

@@ -9,6 +9,7 @@ import {
   GEO_BACKFILL_BATCH_CAP,
   GEO_BACKFILL_CRON,
   GEO_BACKFILL_QUEUE,
+  GEO_BACKFILL_RETRY_DAYS,
   GEO_BACKFILL_WINDOW_DAYS,
   runGeoBackfill,
 } from "../geo-backfill";
@@ -65,6 +66,21 @@ describe("runGeoBackfill", () => {
       createdAt: {
         gt: new Date(now.getTime() - GEO_BACKFILL_WINDOW_DAYS * 86_400_000),
       },
+      // A row tried less than the retry wait ago is not picked again.
+      AND: [
+        {
+          OR: [
+            { geoAttemptedAt: null },
+            {
+              geoAttemptedAt: {
+                lt: new Date(
+                  now.getTime() - GEO_BACKFILL_RETRY_DAYS * 86_400_000,
+                ),
+              },
+            },
+          ],
+        },
+      ],
     });
     expect(args?.take).toBe(GEO_BACKFILL_BATCH_CAP);
   });
@@ -91,7 +107,7 @@ describe("runGeoBackfill", () => {
     });
     expect(prisma.auditLog.update).toHaveBeenCalledWith({
       where: { id: "a1" },
-      data: { location: "Berlin, DE" },
+      data: { location: "Berlin, DE", geoAttemptedAt: expect.any(Date) },
     });
   });
 
@@ -117,6 +133,7 @@ describe("runGeoBackfill", () => {
         location: "München, DE",
         asn: 3320,
         carrier: "Deutsche Telekom AG",
+        geoAttemptedAt: expect.any(Date),
       },
     });
   });
@@ -136,7 +153,11 @@ describe("runGeoBackfill", () => {
     expect(summary.carrierResolved).toBe(1);
     expect(prisma.auditLog.update).toHaveBeenCalledWith({
       where: { id: "a2b" },
-      data: { location: "Bochum, DE", carrier: "Deutsche Telekom AG" },
+      data: {
+        location: "Bochum, DE",
+        carrier: "Deutsche Telekom AG",
+        geoAttemptedAt: expect.any(Date),
+      },
     });
   });
 
@@ -158,15 +179,20 @@ describe("runGeoBackfill", () => {
     });
     expect(prisma.auditLog.update).toHaveBeenCalledWith({
       where: { id: "a3" },
-      data: { carrier: "Vodafone GmbH", asn: 3209 },
+      data: {
+        carrier: "Vodafone GmbH",
+        asn: 3209,
+        geoAttemptedAt: expect.any(Date),
+      },
     });
   });
 
-  it("skips the update when the resolver misses entirely, counts as still-unresolved", async () => {
+  it("stamps only the attempt when the resolver misses entirely, counts as still-unresolved", async () => {
     const prisma = makePrismaMock([{ id: "a4", ipAddress: "192.0.2.1" }]);
     vi.mocked(lookupIpGeo).mockResolvedValueOnce(EMPTY_GEO);
+    const now = new Date("2026-05-15T12:00:00Z");
 
-    const summary = await runGeoBackfill(prisma);
+    const summary = await runGeoBackfill(prisma, now);
 
     expect(summary).toEqual({
       scanned: 1,
@@ -174,7 +200,12 @@ describe("runGeoBackfill", () => {
       carrierResolved: 0,
       stillUnresolved: 1,
     });
-    expect(prisma.auditLog.update).not.toHaveBeenCalled();
+    // The try is recorded, so the next passes leave the row alone until
+    // the retry wait has passed.
+    expect(prisma.auditLog.update).toHaveBeenCalledWith({
+      where: { id: "a4" },
+      data: { geoAttemptedAt: now },
+    });
   });
 
   it("handles a mixed batch of hits and misses", async () => {
@@ -204,7 +235,7 @@ describe("runGeoBackfill", () => {
       carrierResolved: 2,
       stillUnresolved: 1,
     });
-    expect(prisma.auditLog.update).toHaveBeenCalledTimes(2);
+    expect(prisma.auditLog.update).toHaveBeenCalledTimes(3);
   });
 
   it("treats a Prisma update throw as still-unresolved without aborting the pass", async () => {

src/lib/jobs/geo-backfill.ts

@@ -34,9 +34,16 @@
  *
  * Idempotent: a row that the resolver still cannot match (private IP
  * sneaking through CF egress, freshly-allocated range the GeoLite2
- * release hasn't picked up yet) stays NULL and the next pass tries
- * again. The cap is per-call, not per-row, so the helper can be
- * scheduled at any cadence without doubling up.
+ * release hasn't picked up yet) stays NULL. Every row a pass looks at is
+ * stamped `geoAttemptedAt`, and a row stamped less than
+ * `GEO_BACKFILL_RETRY_DAYS` ago is not picked again: an address no
+ * database will ever place used to be looked up on every hourly pass for
+ * its whole 30-day window, and finding it walked every audit row of those
+ * 30 days. The wait still lets a range a later GeoLite2 release learns
+ * resolve on a later pass. The candidates are read through the partial
+ * index `audit_logs_geo_backfill_idx` (migration 0418), which holds only
+ * the rows still missing something. The cap is per-call, not per-row, so
+ * the helper can be scheduled at any cadence without doubling up.
  *
  * No notifications, no event ladder — the maintainer can read the
  * counts off the returned summary and decide whether to keep
@@ -47,6 +54,8 @@ import { lookupIpGeo } from "@/lib/geo";
 
 export const GEO_BACKFILL_BATCH_CAP = 500;
 export const GEO_BACKFILL_WINDOW_DAYS = 30;
+/** How long a row the backfill could not complete waits for its next try. */
+export const GEO_BACKFILL_RETRY_DAYS = 7;
 
 /**
  * v1.4.37 — pg-boss queue name + cron expression for the recurring
@@ -74,6 +83,9 @@ export async function runGeoBackfill(
   const cutoff = new Date(
     now.getTime() - GEO_BACKFILL_WINDOW_DAYS * 86_400_000,
   );
+  const retryBefore = new Date(
+    now.getTime() - GEO_BACKFILL_RETRY_DAYS * 86_400_000,
+  );
 
   const rows = await prisma.auditLog.findMany({
     where: {
@@ -85,6 +97,14 @@ export async function runGeoBackfill(
       OR: [{ location: null }, { carrier: null }],
       ipAddress: { not: null },
       createdAt: { gt: cutoff },
+      AND: [
+        {
+          OR: [
+            { geoAttemptedAt: null },
+            { geoAttemptedAt: { lt: retryBefore } },
+          ],
+        },
+      ],
     },
     select: { id: true, ipAddress: true },
     orderBy: { createdAt: "desc" },
@@ -109,8 +129,14 @@ export async function runGeoBackfill(
     // ASN MMDB. The online fallback bounds itself with a request timeout.
     const { location, asn, carrier } = await lookupIpGeo(ip);
 
-    const data: { location?: string; asn?: number; carrier?: string | null } =
-      {};
+    // Stamped whatever the outcome: a row that resolved leaves the candidate
+    // set anyway, and one that did not waits for its next try.
+    const data: {
+      location?: string;
+      asn?: number;
+      carrier?: string | null;
+      geoAttemptedAt: Date;
+    } = { geoAttemptedAt: now };
     if (location) {
       data.location = location;
       summary.located += 1;
@@ -121,10 +147,8 @@ export async function runGeoBackfill(
       summary.carrierResolved += 1;
     }
 
-    if (Object.keys(data).length === 0) {
-      summary.stillUnresolved += 1;
-      continue;
-    }
+    const resolved = data.location !== undefined || data.carrier !== undefined;
+    if (!resolved) summary.stillUnresolved += 1;
 
     try {
       await prisma.auditLog.update({
@@ -135,7 +159,7 @@ export async function runGeoBackfill(
       // The row may have been deleted by the retention sweeper between
       // findMany and update. Treat as unresolved; the next pass will
       // skip it because the row no longer exists.
-      summary.stillUnresolved += 1;
+      if (resolved) summary.stillUnresolved += 1;
     }
   }