From 2bb030b656fe319b9aa7739ae0592ac0bc711b12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= <mbombeck@gmail.com>
Date: Sun, 25 Oct 2026 17:23:51 +0100
Subject: [PATCH] fix(offhost): keep the bucket's last lifecycle answer through
 a failed read

A lifecycle read that timed out once was cached as unknown for ten minutes,
which kept every key an off-host object ever needed in the recovery kit: the
kit turned due, the admins were pushed attention, and the next good read
pushed recovered. A failed read now stands on the last answer the bucket gave,
for up to a day.
---
 .../__tests__/offhost-lifecycle-probe.test.ts | 56 +++++++++++++++++++
 src/lib/jobs/offhost-backup.ts                | 20 +++++++
 2 files changed, 76 insertions(+)
 create mode 100644 src/lib/jobs/__tests__/offhost-lifecycle-probe.test.ts

diff --git a/src/lib/jobs/__tests__/offhost-lifecycle-probe.test.ts b/src/lib/jobs/__tests__/offhost-lifecycle-probe.test.ts
new file mode 100644
index 000000000..3a53b2c3d
--- /dev/null
+++ b/src/lib/jobs/__tests__/offhost-lifecycle-probe.test.ts
@@ -0,0 +1,56 @@
+/**
+ * A lifecycle read that fails once does not undo what the bucket said an
+ * hour before. Read as "unknown", it kept every key an off-host object ever
+ * needed in the recovery kit, the kit turned due and the admins were pushed,
+ * and the next good read pushed "recovered".
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+  LIFECYCLE_UNKNOWN,
+  type OffhostLifecycle,
+} from "@/lib/jobs/offhost-lifecycle";
+import { probeOffhostLifecycle, type S3Like } from "@/lib/jobs/offhost-backup";
+
+const CONFIGURED: OffhostLifecycle = {
+  state: "configured",
+  expirationDays: 30,
+  database: { state: "configured", expirationDays: 30 },
+};
+const bucket = (answer: () => Promise<OffhostLifecycle>) =>
+  ({ getLifecycle: answer }) as unknown as S3Like;
+
+const HOUR = 3_600_000;
+const T0 = Date.parse("2026-10-25T00:00:00Z");
+
+beforeEach(() => {
+  vi.stubEnv("BACKUP_S3_ENDPOINT", "https://bucket.example");
+  vi.stubEnv("BACKUP_S3_BUCKET", "b");
+  vi.stubEnv("BACKUP_S3_ACCESS_KEY", "a");
+  vi.stubEnv("BACKUP_S3_SECRET_KEY", "s");
+  vi.stubEnv("BACKUP_ENCRYPTION_KEY", "e".repeat(64));
+});
+afterEach(() => vi.unstubAllEnvs());
+
+describe("the bucket's lifecycle rule, read again", () => {
+  it("keeps the last answer the bucket gave through a failed read, for a day", async () => {
+    expect(
+      await probeOffhostLifecycle(
+        bucket(async () => CONFIGURED),
+        T0,
+      ),
+    ).toEqual(CONFIGURED);
+    const failing = bucket(async () => LIFECYCLE_UNKNOWN);
+    expect(await probeOffhostLifecycle(failing, T0 + HOUR)).toEqual(CONFIGURED);
+    const thrown = bucket(async () => {
+      throw new Error("ETIMEDOUT");
+    });
+    expect(await probeOffhostLifecycle(thrown, T0 + 2 * HOUR)).toEqual(
+      CONFIGURED,
+    );
+    // A day without a single good read is an answer of its own.
+    expect(await probeOffhostLifecycle(failing, T0 + 25 * HOUR)).toEqual(
+      LIFECYCLE_UNKNOWN,
+    );
+  });
+});
diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts
index 9899c83c2..4994d156b 100644
--- a/src/lib/jobs/offhost-backup.ts
+++ b/src/lib/jobs/offhost-backup.ts
@@ -758,6 +758,18 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
 const LIFECYCLE_CACHE_MS = 10 * 60 * 1000;
 let lifecycleCache: { at: number; value: OffhostLifecycle } | null = null;
 
+/**
+ * The last answer the bucket actually gave (`configured` or `missing`), and
+ * how long it stands in for a read that failed. A probe that timed out once
+ * read as "unknown", which keeps every key an off-host object ever needed
+ * in the recovery kit: the kit turned due, the admins were pushed, and the
+ * next good read pushed "recovered". A rule does not change between two
+ * reads an hour apart; a credential that has lost the right to read it for a
+ * whole day does say something, and then "unknown" is the honest answer.
+ */
+const LAST_KNOWN_LIFECYCLE_MS = 24 * 60 * 60 * 1000;
+let lastKnownLifecycle: { at: number; value: OffhostLifecycle } | null = null;
+
 /**
  * The bucket's lifecycle rule for the admin card, cached for ten minutes so a
  * page render does not become a bucket call. Never throws.
@@ -782,6 +794,14 @@ export async function probeOffhostLifecycle(
   } catch {
     value = LIFECYCLE_UNKNOWN;
   }
+  if (value.state !== "unknown") {
+    lastKnownLifecycle = { at: now, value };
+  } else if (
+    lastKnownLifecycle &&
+    now - lastKnownLifecycle.at < LAST_KNOWN_LIFECYCLE_MS
+  ) {
+    value = lastKnownLifecycle.value;
+  }
   if (!s3Override) lifecycleCache = { at: now, value };
   return value;
 }