feat(admin): replace the key backup step with a recovery kit

The step confirmed one key, the active application key. A restore needs
more: off-host copies are sealed under BACKUP_ENCRYPTION_KEY, a copy
written before a rotation needs the retired application key it was
written under, and every session, token and share link hangs off
API_TOKEN_HMAC_KEY. An operator could do everything the step asked and
still be unable to read a single backup.

The recovery kit lists every key a restore of this server needs, by id
and fingerprint, never key material:

- each id in ENCRYPTION_KEYS, plus any retired id a stored copy
  (innerKeyIds) or an off-host object inside the bucket's retention
  (OffhostBackupKeyUse) still lists, also once the server dropped it;
- BACKUP_ENCRYPTION_KEY and each BACKUP_ENCRYPTION_PREVIOUS_KEYS entry,
  named by the id their objects record;
- API_TOKEN_HMAC_KEY.

Confirming records the set the admin was shown and refuses a stale one
with 409 recovery.kit.stale. The kit is due again as soon as a key it
lists is not in that set (a rotation, a new backup key, a re-keyed
install); a key that merely drops out does not make it due. "Check my
copy" now checks any key in the kit.

Migration 0487 carries a confirmation given under the old step forward
as a kit holding that key, so that part stays confirmed, and drops the
four app_settings columns. It also creates the two tables the readiness
evaluation uses (restore_drill_outcomes, recovery_readiness_notices).

The routes move to /api/admin/recovery-kit; the reader of the keys the
backups still need moves out of the encryption status route into
src/lib/recovery/backup-key-needs.ts so the kit and the status view
share one definition.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 13:32:29 +01:00
1 parent 953c64f7a4
commit d3c672cc3f
56 files changed
+3042 -1611

No files matched your search

@@ -1,50 +0,0 @@
import { describe, expect, it } from "vitest";
import { isKeyBackupDue, resolvePlatformHint } from "../key-backup";
const confirmed = (keyId: string, fingerprint: string) => ({
encryptionKeyBackupConfirmedAt: new Date("2026-10-03T10:00:00Z"),
encryptionKeyBackupConfirmedKeyId: keyId,
encryptionKeyBackupConfirmedFingerprint: fingerprint,
});
describe("key backup step", () => {
it("is due without any confirmation", () => {
expect(isKeyBackupDue(null, "v1", "abcdefabcdef")).toBe(true);
expect(
isKeyBackupDue(
{
encryptionKeyBackupConfirmedAt: null,
encryptionKeyBackupConfirmedKeyId: null,
encryptionKeyBackupConfirmedFingerprint: null,
},
"v1",
"abcdefabcdef",
),
).toBe(true);
});
it("is done only for the exact key that was confirmed", () => {
expect(
isKeyBackupDue(confirmed("v1", "abcdefabcdef"), "v1", "abcdefabcdef"),
).toBe(false);
});
it("is due again after a rotation (new active id)", () => {
expect(
isKeyBackupDue(confirmed("v1", "abcdefabcdef"), "v2", "abcdefabcdef"),
).toBe(true);
});
it("is due again after a re-keyed install (same id, other key)", () => {
expect(
isKeyBackupDue(confirmed("v1", "abcdefabcdef"), "v1", "123456123456"),
).toBe(true);
});
it("reads the platform hint and falls back to compose", () => {
expect(resolvePlatformHint("truenas")).toBe("truenas");
expect(resolvePlatformHint(" Unraid ")).toBe("unraid");
expect(resolvePlatformHint(undefined)).toBe("compose");
expect(resolvePlatformHint("kubernetes")).toBe("compose");
});
});
-142
View File
@@ -1,142 +0,0 @@
/**
* The "Back up your encryption key" step: instance state and its rules.
*
* The confirmation lives on the `AppSettings` singleton, bound to the active
* key id and that key's fingerprint, so it is an instance fact (it survives
* the admin who gave it) and a rotation or a re-keyed install makes the step
* due again. Nothing here ever reads, returns or stores the key itself.
*/
import { prisma } from "@/lib/db";
import { envValue } from "@/lib/env";
import { getActiveKeyId, getKeyFingerprint } from "@/lib/crypto";
/** Where the operator sets the key; picks the tab the step opens on. */
export const KEY_BACKUP_PLATFORMS = [
"compose",
"truenas",
"coolify",
"unraid",
"portainer",
] as const;
export type KeyBackupPlatform = (typeof KEY_BACKUP_PLATFORMS)[number];
/** Wire code for a confirmation given for a key that is no longer active. */
export const KEY_BACKUP_STALE_CODE = "encryption.keyBackup.stale";
/**
* `HEALTHLOG_PLATFORM`, when it names a known platform; `compose` otherwise.
* Catalog templates set it so the step opens on the right instructions.
*/
export function resolvePlatformHint(
raw: string | undefined = envValue("HEALTHLOG_PLATFORM"),
): KeyBackupPlatform {
const value = raw?.trim().toLowerCase();
return (KEY_BACKUP_PLATFORMS as readonly string[]).includes(value ?? "")
? (value as KeyBackupPlatform)
: "compose";
}
export interface KeyBackupStatus {
/** True until a confirmation exists for the active key id AND fingerprint. */
due: boolean;
activeKeyId: string;
/** Fingerprint of the active key (first 12 hex of SHA-256 over its bytes). */
fingerprint: string;
confirmedAt: string | null;
confirmedKeyId: string | null;
confirmedFingerprint: string | null;
/** The confirming admin, while that account exists. */
confirmedBy: { id: string; email: string | null } | null;
platformHint: KeyBackupPlatform;
}
interface StoredConfirmation {
encryptionKeyBackupConfirmedAt: Date | null;
encryptionKeyBackupConfirmedKeyId: string | null;
encryptionKeyBackupConfirmedFingerprint: string | null;
}
/** Pure rule: is the stored confirmation for this exact key? */
export function isKeyBackupDue(
stored: StoredConfirmation | null,
activeKeyId: string,
fingerprint: string,
): boolean {
return !(
stored?.encryptionKeyBackupConfirmedAt &&
stored.encryptionKeyBackupConfirmedKeyId === activeKeyId &&
stored.encryptionKeyBackupConfirmedFingerprint === fingerprint
);
}
function activeKey(): { activeKeyId: string; fingerprint: string } {
const activeKeyId = getActiveKeyId();
// The active id is always configured, so the fingerprint is never null.
return { activeKeyId, fingerprint: getKeyFingerprint(activeKeyId) ?? "" };
}
export async function readKeyBackupStatus(): Promise<KeyBackupStatus> {
const { activeKeyId, fingerprint } = activeKey();
const row = await prisma.appSettings.findUnique({
where: { id: "singleton" },
select: {
encryptionKeyBackupConfirmedAt: true,
encryptionKeyBackupConfirmedKeyId: true,
encryptionKeyBackupConfirmedFingerprint: true,
encryptionKeyBackupConfirmedByUserId: true,
},
});
const confirmedBy = row?.encryptionKeyBackupConfirmedByUserId
? await prisma.user.findUnique({
where: { id: row.encryptionKeyBackupConfirmedByUserId },
select: { id: true, email: true },
})
: null;
return {
due: isKeyBackupDue(row, activeKeyId, fingerprint),
activeKeyId,
fingerprint,
confirmedAt: row?.encryptionKeyBackupConfirmedAt?.toISOString() ?? null,
confirmedKeyId: row?.encryptionKeyBackupConfirmedKeyId ?? null,
confirmedFingerprint: row?.encryptionKeyBackupConfirmedFingerprint ?? null,
confirmedBy: confirmedBy ?? null,
platformHint: resolvePlatformHint(),
};
}
export type ConfirmKeyBackupResult =
| { ok: true; status: KeyBackupStatus }
| { ok: false; activeKeyId: string; fingerprint: string };
/**
* Record the confirmation, but only for the key the admin was shown: a page
* opened before a rotation must not confirm the new key.
*/
export async function confirmKeyBackup(input: {
keyId: string;
fingerprint: string;
userId: string;
}): Promise<ConfirmKeyBackupResult> {
const { activeKeyId, fingerprint } = activeKey();
if (input.keyId !== activeKeyId || input.fingerprint !== fingerprint) {
return { ok: false, activeKeyId, fingerprint };
}
const now = new Date();
await prisma.appSettings.upsert({
where: { id: "singleton" },
create: {
id: "singleton",
encryptionKeyBackupConfirmedAt: now,
encryptionKeyBackupConfirmedKeyId: activeKeyId,
encryptionKeyBackupConfirmedFingerprint: fingerprint,
encryptionKeyBackupConfirmedByUserId: input.userId,
},
update: {
encryptionKeyBackupConfirmedAt: now,
encryptionKeyBackupConfirmedKeyId: activeKeyId,
encryptionKeyBackupConfirmedFingerprint: fingerprint,
encryptionKeyBackupConfirmedByUserId: input.userId,
},
});
return { ok: true, status: await readKeyBackupStatus() };
}
+6
View File
@@ -353,6 +353,12 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
HostMetric: "instance-wide host telemetry, not attributable to an account",
OffhostBackupKeyUse:
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
RestoreDrillOutcome:
"the monthly restore drill's last verdict for the instance's off-host bucket; says nothing about any account",
RecoveryReadinessNotice:
"per backup signal, what the admins were last told; a fact about the instance's backups, not about any account",
RecoveryKitConfirmation:
"the instance's recovery kit confirmation (key ids and fingerprints); the confirming admin is a plain id, not a relation",
OffhostPurgeRequest:
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
CycleSymptomCategory:
+2 -22
View File
@@ -23,12 +23,7 @@
*/
import { envOr } from "@/lib/env";
import { Buffer } from "node:buffer";
import {
createCipheriv,
createDecipheriv,
createHash,
randomBytes,
} from "node:crypto";
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
import { Readable, Transform } from "node:stream";
import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
import type { PrismaClient } from "@/generated/prisma/client";
@@ -44,6 +39,7 @@ import {
import { annotate, getEvent } from "@/lib/logging/context";
import { envValue } from "@/lib/env";
import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue";
import { decodeBackupKey, offhostKeyId } from "./offhost-key";
const ALGORITHM = "aes-256-gcm";
const IV_LENGTH = 12;
@@ -74,17 +70,6 @@ export class OffhostBackupNotConfiguredError extends Error {
}
}
function decodeBackupKey(raw: string): Buffer {
if (/^[0-9a-fA-F]{64}$/.test(raw)) return Buffer.from(raw, "hex");
if (/^[A-Za-z0-9+/=]+$/.test(raw)) {
const buf = Buffer.from(raw, "base64");
if (buf.length === 32) return buf;
}
throw new Error(
"BACKUP_ENCRYPTION_KEY must be 64 hex chars or 32-byte base64",
);
}
/**
* Whether this host has the five variables the nightly job needs.
*
@@ -200,11 +185,6 @@ function asKeyRing(key: Buffer | OffhostKeyRing): OffhostKeyRing {
return Buffer.isBuffer(key) ? { active: key, previous: [] } : key;
}
/** The id a version-4 object records for the key it was written under. */
export function offhostKeyId(key: Buffer): string {
return createHash("sha256").update(key).digest("hex").slice(0, 12);
}
function offhostAad(objectKey: string): Buffer {
return Buffer.from(`${OFFHOST_AAD_PREFIX}${objectKey}`, "utf8");
}
+28
View File
@@ -0,0 +1,28 @@
/**
* The off-host backup key as a value: how the operator writes it and the id
* an object records for it. Kept apart from the uploader so the recovery kit
* can name the keys without loading the backup machinery.
*/
import { Buffer } from "node:buffer";
import { createHash } from "node:crypto";
/** One off-host key as the operator writes it: 64 hex or 32-byte base64. */
export function decodeBackupKey(raw: string): Buffer {
if (/^[0-9a-fA-F]{64}$/.test(raw)) return Buffer.from(raw, "hex");
if (/^[A-Za-z0-9+/=]+$/.test(raw)) {
const buf = Buffer.from(raw, "base64");
if (buf.length === 32) return buf;
}
throw new Error(
"BACKUP_ENCRYPTION_KEY must be 64 hex chars or 32-byte base64",
);
}
/**
* The id a version-4 object records for the key it was written under: the
* first twelve hex characters of the key's SHA-256. The recovery kit uses the
* same value as the key's fingerprint, so a kit entry matches an object header.
*/
export function offhostKeyId(key: Buffer): string {
return createHash("sha256").update(key).digest("hex").slice(0, 12);
}
+6 -4
View File
@@ -96,11 +96,13 @@ export const SENSITIVE_KEY_PATTERNS: readonly RegExp[] = [
// `client_secret` are already covered by `/token/i` + `/secret/i`; the
// verifier is the one secret-shaped key those patterns miss.
/verifier/i,
// v1.40.0 — the encryption key an admin pastes into "Check my copy"
// (`POST /api/admin/encryption/key-backup/verify`). The route never logs its
// body; this keeps it out of any excerpt a future change might take.
// Anchored so `encryptionKeyId` and friends stay readable.
// v1.40.0 — the encryption key an admin pastes into "Check my copy", and the
// copy of any recovery kit key, sent as `keyCopy`
// (`POST /api/admin/recovery-kit/verify`). The route never logs its body;
// this keeps it out of any excerpt a future change might take. Anchored so
// `encryptionKeyId` and friends stay readable.
/^encryption[_-]?key$/i,
/^key[_-]?copy$/i,
];
/**
+2 -1
View File
@@ -199,7 +199,7 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
"encounter.reminder-conflict",
"encounter.reminder-not-found",
],
encryption: ["encryption.keyBackup.stale", "encryption.key_mismatch"],
encryption: ["encryption.key_mismatch"],
environment: [
"environment.backfill_pending",
"environment.backfill_rate_limited",
@@ -398,6 +398,7 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
"profile_claim.taken",
],
record_write: ["record_write.rate_limited"],
recovery: ["recovery.kit.stale"],
"report-selection": [
"report-selection.body.invalid_json",
"report-selection.body.invalid_shape",
+4 -3
View File
@@ -55,9 +55,10 @@ export const adminKeys = {
["admin", "maintenance", "measurements"] as const,
/** v1.43 — the pulse day-mean method, its dry run and apply step. */
adminPulseDayMean: () => ["admin", "maintenance", "pulse-day-mean"] as const,
/** v1.40.0 — the "Back up your encryption key" step (admin + dashboard banner). */
adminEncryptionKeyBackup: () =>
["admin", "encryption", "key-backup"] as const,
/** The recovery kit (admin Encryption section + dashboard banner). */
adminRecoveryKit: () => ["admin", "recovery-kit"] as const,
/** Every backup signal's verdict (admin overview + Backups). */
adminRecoveryReadiness: () => ["admin", "recovery-readiness"] as const,
adminCoachFeedback: () => ["admin", "coach-feedback"] as const,
adminFeedback: (status: string) => ["admin", "feedback", status] as const,
adminFeedbackRoot: () => ["admin", "feedback"] as const,
@@ -0,0 +1,231 @@
/**
* The recovery kit rules: which keys the kit lists, what a confirmation
* covers, and when the kit is due again.
*
* Mutation check: compare fingerprints out of the covered test (id only) and
* the re-keyed-install case goes red; drop the backup needs from the
* application list and the retired-key case goes red.
*/
import { createHash } from "node:crypto";
import { describe, expect, it } from "vitest";
import { offhostKeyId } from "@/lib/jobs/offhost-key";
import type { BackupKeyNeeds } from "@/lib/recovery/backup-key-needs";
import {
applicationKitKeys,
assessRecoveryKit,
backupKitKeys,
fingerprintText,
neededKeyIds,
parseKitEntries,
resolvePlatformHint,
sameKit,
tokenKitKeys,
type KitEntry,
} from "@/lib/recovery/kit-rules";
const NOW = new Date("2026-10-25T12:00:00Z");
const DAY = 86_400_000;
const NO_NEEDS: BackupKeyNeeds = {
stored: [],
unrecorded: { copies: 0, oldestAt: null },
offhost: [],
offhostExpirationDays: null,
retiredKeysStillNeeded: [],
};
const FINGERPRINTS: Record<string, string> = {
v1: "111111111111",
v2: "222222222222",
};
function appKeys(configured: string[], active: string, needs = NO_NEEDS) {
return applicationKitKeys({
activeKeyId: active,
configuredKeyIds: configured,
fingerprintOf: (id) => (configured.includes(id) ? FINGERPRINTS[id] : null),
needs,
now: NOW,
});
}
const BACKUP_HEX = "ab".repeat(32);
const BACKUP_OLD_HEX = "cd".repeat(32);
describe("which keys the kit lists", () => {
it("lists every configured application key, the active one marked", () => {
const keys = appKeys(["v1", "v2"], "v2");
expect(keys.map((k) => [k.id, k.need.active, k.fingerprint])).toEqual([
["v1", false, "111111111111"],
["v2", true, "222222222222"],
]);
});
it("keeps a retired key a stored copy lists, even once the server dropped it", () => {
const needs: BackupKeyNeeds = {
...NO_NEEDS,
stored: [{ keyId: "v1", copies: 3, oldestAt: NOW.toISOString() }],
};
const [v1] = appKeys(["v2"], "v2", needs);
expect(v1).toMatchObject({
id: "v1",
fingerprint: null,
problem: "not_configured",
need: { configured: false, storedCopies: 3 },
});
});
it("keeps an off-host key while objects inside the retention need it", () => {
const row = (neededUntil: string | null) => ({
...NO_NEEDS,
offhost: [
{
keyId: "v1",
firstWrittenAt: NOW.toISOString(),
lastWrittenAt: NOW.toISOString(),
neededUntil,
},
],
});
const until = new Date(NOW.getTime() + 10 * DAY).toISOString();
expect(appKeys(["v2"], "v2", row(until))[0].need.offhostUntil).toBe(until);
expect(appKeys(["v2"], "v2", row(null))[0].need.offhostUntil).toBe(
"unknown",
);
// Past the retention the objects are gone, so the key drops out.
const past = new Date(NOW.getTime() - DAY).toISOString();
expect(appKeys(["v2"], "v2", row(past)).map((k) => k.id)).toEqual(["v2"]);
expect([...neededKeyIds(row(past), NOW)]).toEqual([]);
});
it("names the off-host keys by the id their objects record", () => {
const keys = backupKitKeys(BACKUP_HEX, ` ${BACKUP_OLD_HEX} ,`);
expect(keys.map((k) => [k.id, k.need.active, k.source])).toEqual([
[
offhostKeyId(Buffer.from(BACKUP_HEX, "hex")),
true,
"BACKUP_ENCRYPTION_KEY",
],
[
offhostKeyId(Buffer.from(BACKUP_OLD_HEX, "hex")),
false,
"BACKUP_ENCRYPTION_PREVIOUS_KEYS",
],
]);
expect(keys.every((k) => k.fingerprint === k.id)).toBe(true);
expect(backupKitKeys(undefined, undefined)).toEqual([]);
});
it("flags an off-host key that does not decode instead of hiding it", () => {
const [bad] = backupKitKeys("not-a-key", undefined);
expect(bad).toMatchObject({ fingerprint: null, problem: "unreadable" });
});
it("fingerprints the token key as the text the operator set", () => {
const value = "x".repeat(40);
const [token] = tokenKitKeys(value);
expect(token.fingerprint).toBe(
createHash("sha256").update(value).digest("hex").slice(0, 12),
);
expect(token.fingerprint).toBe(fingerprintText(value));
expect(JSON.stringify(token)).not.toContain(value);
});
});
describe("what a confirmation covers", () => {
const listed = [...appKeys(["v1"], "v1"), ...tokenKitKeys("t".repeat(40))];
const entries = (keys: typeof listed): KitEntry[] =>
keys.map(({ kind, id, fingerprint }) => ({ kind, id, fingerprint }));
it("is due without any confirmation, and done for the exact set", () => {
expect(assessRecoveryKit(listed, null).due).toBe(true);
expect(assessRecoveryKit(listed, entries(listed)).due).toBe(false);
});
it("keeps a confirmation of the active key only, from the old step, for that key", () => {
const legacy: KitEntry[] = [
{ kind: "application", id: "v1", fingerprint: "111111111111" },
];
const verdict = assessRecoveryKit(listed, legacy);
expect(verdict.due).toBe(true);
expect(verdict.keys.find((k) => k.id === "v1")?.confirmed).toBe(true);
expect(verdict.unconfirmed.map((k) => k.id)).toEqual([
"API_TOKEN_HMAC_KEY",
]);
});
it("is due again after a rotation, a re-keyed install or a new backup key", () => {
const confirmed = entries(listed);
// Rotation: a new active id.
const rotated = [
...appKeys(["v1", "v2"], "v2"),
...tokenKitKeys("t".repeat(40)),
];
expect(
assessRecoveryKit(rotated, confirmed).unconfirmed.map((k) => k.id),
).toEqual(["v2"]);
// Same id, other key.
const rekeyed = listed.map((k) =>
k.id === "v1" ? { ...k, fingerprint: "999999999999" } : k,
);
expect(assessRecoveryKit(rekeyed, confirmed).due).toBe(true);
// A backup key appears.
const withBackup = [...listed, ...backupKitKeys(BACKUP_HEX, undefined)];
expect(assessRecoveryKit(withBackup, confirmed).unconfirmed).toHaveLength(
1,
);
});
it("is not due when a key merely drops out of the kit", () => {
const wider = [
...appKeys(["v1", "v2"], "v2"),
...tokenKitKeys("t".repeat(40)),
];
const narrower = [
...appKeys(["v2"], "v2"),
...tokenKitKeys("t".repeat(40)),
];
expect(assessRecoveryKit(narrower, entries(wider)).due).toBe(false);
});
it("covers a key this server does not hold by its id", () => {
const needs: BackupKeyNeeds = {
...NO_NEEDS,
stored: [{ keyId: "v0", copies: 1, oldestAt: NOW.toISOString() }],
};
const kit = appKeys(["v1"], "v1", needs);
const confirmed = entries(kit);
expect(confirmed.find((e) => e.id === "v0")?.fingerprint).toBeNull();
expect(assessRecoveryKit(kit, confirmed).due).toBe(false);
});
it("matches the shown kit against the current one as a set", () => {
const a = entries(listed);
expect(sameKit([...a].reverse(), a)).toBe(true);
expect(sameKit(a.slice(1), a)).toBe(false);
expect(
sameKit(
a.map((e) =>
e.id === "v1" ? { ...e, fingerprint: "000000000000" } : e,
),
a,
),
).toBe(false);
});
it("reads a stored confirmation it cannot parse as none", () => {
expect(parseKitEntries({ not: "a list" })).toEqual([]);
expect(
parseKitEntries([{ kind: "nope", id: "v1", fingerprint: null }]),
).toEqual([]);
});
});
describe("platform hint", () => {
it("opens on a known platform and falls back to compose", () => {
expect(resolvePlatformHint("TrueNAS")).toBe("truenas");
expect(resolvePlatformHint("kubernetes")).toBe("compose");
expect(resolvePlatformHint(undefined)).toBe("compose");
});
});
+99
View File
@@ -0,0 +1,99 @@
/**
* Which application keys the backups still need.
*
* Rotation re-encrypts every row in the database and re-seals every stored
* backup's envelope, but the content INSIDE a backup keeps the key it was
* written under: a restore writes it back verbatim. So "every row is on the
* active key" is not the same as "the old key can go". This answers the second
* question per key id, from what each copy recorded as it was written: the
* stored copies that need it and the oldest of them, and for the off-host
* bucket the last night an object needing it went in, plus how long the
* bucket's lifecycle rule keeps objects after that.
*
* Two readers ask it: the admin encryption view, which shows the table, and
* the recovery kit, which lists every key a restore needs. One definition, so
* the kit cannot drop a key the encryption view still calls needed.
*/
import { prisma } from "@/lib/db";
import {
offhostBackupConfigured,
probeOffhostLifecycle,
} from "@/lib/jobs/offhost-backup";
import { neededKeyIds } from "@/lib/recovery/kit-rules";
export interface BackupKeyNeeds {
stored: Array<{ keyId: string; copies: number; oldestAt: string }>;
/** Copies written before v1.39.3, which recorded no key ids. */
unrecorded: { copies: number; oldestAt: string | null };
offhost: Array<{
keyId: string;
firstWrittenAt: string;
lastWrittenAt: string;
/** When the last object needing this key expires; null when unknown. */
neededUntil: string | null;
}>;
/** The bucket's expiry, when it has one the credential may read. */
offhostExpirationDays: number | null;
/** Retired key ids some backup still needs (never the active one). */
retiredKeysStillNeeded: string[];
}
export async function readBackupKeyNeeds(
activeKeyId: string,
now: Date,
): Promise<BackupKeyNeeds> {
const [stored, unrecorded, offhostRows] = await Promise.all([
prisma.$queryRaw<
Array<{ key_id: string; copies: number; oldest_at: Date }>
>`
SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at
FROM data_backups, unnest(inner_key_ids) AS k
WHERE inner_key_ids_recorded
GROUP BY k
ORDER BY k
`,
prisma.dataBackup.aggregate({
where: { innerKeyIdsRecorded: false },
_count: { _all: true },
_min: { createdAt: true },
}),
prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
]);
const lifecycle =
offhostRows.length > 0 && offhostBackupConfigured()
? await probeOffhostLifecycle()
: null;
const expirationDays =
lifecycle?.state === "configured" ? lifecycle.expirationDays : null;
const offhost = offhostRows.map((row) => ({
keyId: row.keyId,
firstWrittenAt: row.firstWrittenAt.toISOString(),
lastWrittenAt: row.lastWrittenAt.toISOString(),
neededUntil:
expirationDays === null
? null
: new Date(
row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000,
).toISOString(),
}));
const needs: BackupKeyNeeds = {
stored: stored.map((row) => ({
keyId: row.key_id,
copies: row.copies,
oldestAt: row.oldest_at.toISOString(),
})),
unrecorded: {
copies: unrecorded._count._all,
oldestAt: unrecorded._min.createdAt?.toISOString() ?? null,
},
offhost,
offhostExpirationDays: expirationDays,
retiredKeysStillNeeded: [],
};
const retired = neededKeyIds(needs, now);
retired.delete(activeKeyId);
needs.retiredKeysStillNeeded = [...retired].sort();
return needs;
}
+302
View File
@@ -0,0 +1,302 @@
/**
* The recovery kit: every key a restore of this instance needs, by id and
* fingerprint, and the admin's confirmation that a copy of each is kept
* outside the server.
*
* It replaced a step that confirmed one key, the active application key. An
* operator could do everything that step asked and still be unable to read a
* single backup: the off-host copies are sealed under `BACKUP_ENCRYPTION_KEY`,
* a copy written before a rotation needs the retired application key it was
* written under, and every session, token and share link hangs off
* `API_TOKEN_HMAC_KEY`. The kit lists all of them:
*
* - `application`: every id in `ENCRYPTION_KEYS` this server holds, plus
* every retired id a stored copy or an off-host object inside the
* bucket's retention still lists (`backup-key-needs.ts`);
* - `backup`: `BACKUP_ENCRYPTION_KEY` and each key in
* `BACKUP_ENCRYPTION_PREVIOUS_KEYS`, when set;
* - `token`: `API_TOKEN_HMAC_KEY`.
*
* The confirmation records the set the admin was shown. The kit is due again
* as soon as a key it needs is not in that set: a rotation, a new backup key,
* a re-keyed install. A key that drops out of the kit (a retired key whose
* last copy expired) does not make it due, because nothing is missing.
*
* Fingerprints only, never key material: the first twelve hex characters of
* SHA-256 over the key, the same rule each key's own surface already uses (the
* application key's boot check, the off-host object header). The token key is
* hashed as the text the operator set, so `printf %s "$API_TOKEN_HMAC_KEY" |
* sha256sum` reproduces it.
*/
import { createHash } from "node:crypto";
import { z } from "zod/v4";
import { envValue } from "@/lib/env";
import { decodeBackupKey, offhostKeyId } from "@/lib/jobs/offhost-key";
import type { BackupKeyNeeds } from "@/lib/recovery/backup-key-needs";
/** Where the operator sets the keys; picks the tab the kit opens on. */
export const KIT_PLATFORMS = [
"compose",
"truenas",
"coolify",
"unraid",
"portainer",
] as const;
export type KitPlatform = (typeof KIT_PLATFORMS)[number];
/** Wire code for a confirmation of a kit that has changed since it was shown. */
export const RECOVERY_KIT_STALE_CODE = "recovery.kit.stale";
export const KIT_KEY_KINDS = ["application", "backup", "token"] as const;
export type KitKeyKind = (typeof KIT_KEY_KINDS)[number];
/** The variable each kind of key is set in. */
export const KIT_KEY_SOURCES = {
application: "ENCRYPTION_KEYS",
backup: "BACKUP_ENCRYPTION_KEY",
backupPrevious: "BACKUP_ENCRYPTION_PREVIOUS_KEYS",
token: "API_TOKEN_HMAC_KEY",
} as const;
export type KitKeySource =
(typeof KIT_KEY_SOURCES)[keyof typeof KIT_KEY_SOURCES];
/** One key as a confirmation stores it. */
export interface KitEntry {
kind: KitKeyKind;
id: string;
/** Null for a key this server does not hold but a copy still needs. */
fingerprint: string | null;
}
/** Why the kit lists a key. */
export interface KitKeyNeed {
/** The key new data is written under. */
active: boolean;
/** This server holds the key. */
configured: boolean;
/** Stored copies in this database whose content needs it. */
storedCopies: number;
/**
* Off-host objects need it until this instant; `"unknown"` when the bucket's
* retention cannot be read, so they may need it indefinitely.
*/
offhostUntil: string | null;
}
/** One key the kit lists, with its verdict. */
export interface KitKey extends KitEntry {
source: KitKeySource;
need: KitKeyNeed;
/** The confirmation covers this key. */
confirmed: boolean;
/**
* `not_configured`: a copy needs it and this server does not hold it, so
* only a copy kept elsewhere can read those backups. `unreadable`: the
* variable is set but does not decode as a key.
*/
problem: "not_configured" | "unreadable" | null;
}
export interface RecoveryKit {
keys: KitKey[];
/** A key the kit lists is not covered by the confirmation. */
due: boolean;
/** Keys the confirmation does not cover, by kind and id. */
unconfirmed: Array<Pick<KitEntry, "kind" | "id">>;
confirmedAt: string | null;
/** The confirming admin, while that account exists. */
confirmedBy: { id: string; email: string | null } | null;
platformHint: KitPlatform;
}
/**
* `HEALTHLOG_PLATFORM`, when it names a known platform; `compose` otherwise.
* Catalog templates set it so the kit opens on the right instructions.
*/
export function resolvePlatformHint(
raw: string | undefined = envValue("HEALTHLOG_PLATFORM"),
): KitPlatform {
const value = raw?.trim().toLowerCase();
return (KIT_PLATFORMS as readonly string[]).includes(value ?? "")
? (value as KitPlatform)
: "compose";
}
/** The fingerprint rule for a key that is text rather than decoded bytes. */
export function fingerprintText(value: string): string {
return createHash("sha256").update(value, "utf8").digest("hex").slice(0, 12);
}
const kitEntrySchema = z.object({
kind: z.enum(KIT_KEY_KINDS),
id: z.string().min(1).max(64),
fingerprint: z
.string()
.regex(/^[0-9a-f]{12}$/)
.nullable(),
});
/** The request body of a confirmation: the kit as the admin saw it. */
export const kitConfirmationSchema = z
.object({ entries: z.array(kitEntrySchema).min(1).max(64) })
.strict();
/**
* The stored entries, validated. A row that does not parse confirms nothing,
* which leaves the kit due rather than trusting a shape it cannot read.
*/
export function parseKitEntries(raw: unknown): KitEntry[] {
const parsed = z.array(kitEntrySchema).safeParse(raw);
return parsed.success ? parsed.data : [];
}
const entryKey = (e: Pick<KitEntry, "kind" | "id">) => `${e.kind}:${e.id}`;
/**
* Pure rule: which listed keys a confirmation covers.
*
* A key this server holds is covered by an entry with the same kind, id and
* fingerprint, so a re-keyed install under the same id is due again. A key it
* does not hold has no fingerprint to compare, so an entry with its kind and id
* covers it: the admin said a copy of that id is kept.
*/
export function assessRecoveryKit(
listed: ReadonlyArray<Omit<KitKey, "confirmed">>,
confirmation: readonly KitEntry[] | null,
): { keys: KitKey[]; due: boolean; unconfirmed: KitEntry[] } {
const exact = new Set(
(confirmation ?? []).map((e) => `${entryKey(e)}:${e.fingerprint ?? ""}`),
);
const byId = new Set((confirmation ?? []).map(entryKey));
const keys = listed.map((key) => ({
...key,
confirmed:
key.fingerprint === null
? byId.has(entryKey(key))
: exact.has(`${entryKey(key)}:${key.fingerprint}`),
}));
const unconfirmed = keys
.filter((key) => !key.confirmed)
.map(({ kind, id, fingerprint }) => ({ kind, id, fingerprint }));
return { keys, due: unconfirmed.length > 0, unconfirmed };
}
/** Pure rule: does what the admin was shown match what the kit lists now? */
export function sameKit(
shown: readonly KitEntry[],
listed: readonly KitEntry[],
): boolean {
const canon = (entries: readonly KitEntry[]) =>
[
...new Set(entries.map((e) => `${entryKey(e)}:${e.fingerprint ?? ""}`)),
].sort();
const a = canon(shown);
const b = canon(listed);
return a.length === b.length && a.every((value, i) => value === b[i]);
}
/**
* Every key id a stored copy lists, and every key id an off-host object still
* inside the bucket's retention needs. An off-host key whose expiry cannot be
* read stays needed: without the lifecycle rule nothing proves the objects are
* gone.
*/
export function neededKeyIds(needs: BackupKeyNeeds, now: Date): Set<string> {
const needed = new Set<string>();
for (const row of needs.stored) needed.add(row.keyId);
for (const row of needs.offhost) {
if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) {
needed.add(row.keyId);
}
}
return needed;
}
/**
* Pure: the application keys the kit lists, from the ids this server holds
* and what the backups still need.
*/
export function applicationKitKeys(args: {
activeKeyId: string;
configuredKeyIds: readonly string[];
fingerprintOf: (id: string) => string | null;
needs: BackupKeyNeeds;
now: Date;
}): Array<Omit<KitKey, "confirmed">> {
const needed = neededKeyIds(args.needs, args.now);
const ids = new Set([...args.configuredKeyIds, ...needed]);
return [...ids].sort().map((id) => {
const fingerprint = args.fingerprintOf(id);
const offhost = args.needs.offhost.find((row) => row.keyId === id);
return {
kind: "application" as const,
id,
fingerprint,
source: KIT_KEY_SOURCES.application,
need: {
active: id === args.activeKeyId,
configured: fingerprint !== null,
storedCopies:
args.needs.stored.find((row) => row.keyId === id)?.copies ?? 0,
offhostUntil:
offhost && needed.has(id) ? (offhost.neededUntil ?? "unknown") : null,
},
problem: fingerprint === null ? ("not_configured" as const) : null,
};
});
}
function noNeed(active: boolean): KitKeyNeed {
return { active, configured: true, storedCopies: 0, offhostUntil: null };
}
/** Pure: the off-host keys, from the two variables as the operator set them. */
export function backupKitKeys(
activeRaw: string | undefined,
previousRaw: string | undefined,
): Array<Omit<KitKey, "confirmed">> {
const out: Array<Omit<KitKey, "confirmed">> = [];
const add = (raw: string, source: KitKeySource, active: boolean) => {
let fingerprint: string | null = null;
try {
fingerprint = offhostKeyId(decodeBackupKey(raw));
} catch {
fingerprint = null;
}
out.push({
kind: "backup",
// The fingerprint is the id off-host objects record for their key, so
// the operator matches a kit entry to an object header directly. A key
// that does not decode has neither; it is named by where it is set.
id: fingerprint ?? `${source}#${out.length + 1}`,
fingerprint,
source,
need: noNeed(active),
problem: fingerprint === null ? "unreadable" : null,
});
};
const active = activeRaw?.trim();
if (active) add(active, KIT_KEY_SOURCES.backup, true);
for (const raw of (previousRaw ?? "").split(",")) {
if (raw.trim()) add(raw.trim(), KIT_KEY_SOURCES.backupPrevious, false);
}
return out;
}
/** Pure: the token key, which the app cannot run without. */
export function tokenKitKeys(
raw: string | undefined,
): Array<Omit<KitKey, "confirmed">> {
const value = raw ?? "";
return [
{
kind: "token",
id: KIT_KEY_SOURCES.token,
fingerprint: value ? fingerprintText(value) : null,
source: KIT_KEY_SOURCES.token,
need: { ...noNeed(true), configured: value.length > 0 },
problem: value ? null : "not_configured",
},
];
}
+173
View File
@@ -0,0 +1,173 @@
/**
* The recovery kit on this server: which keys it lists right now, what the
* stored confirmation covers, recording a confirmation, and "Check my copy".
* The rules themselves are pure and live in `kit-rules.ts`.
*/
import { createHash, timingSafeEqual } from "node:crypto";
import { prisma } from "@/lib/db";
import { envValue } from "@/lib/env";
import {
candidateMatchesKey,
getActiveKeyId,
getConfiguredKeyIds,
getKeyFingerprint,
} from "@/lib/crypto";
import { decodeBackupKey, offhostKeyId } from "@/lib/jobs/offhost-key";
import { readBackupKeyNeeds } from "@/lib/recovery/backup-key-needs";
import {
applicationKitKeys,
assessRecoveryKit,
backupKitKeys,
KIT_KEY_SOURCES,
parseKitEntries,
resolvePlatformHint,
sameKit,
tokenKitKeys,
type KitEntry,
type KitKey,
type KitKeyKind,
type RecoveryKit,
} from "@/lib/recovery/kit-rules";
/** Every key the kit lists on this server right now, without a verdict. */
export async function listKitKeys(
now: Date = new Date(),
): Promise<Array<Omit<KitKey, "confirmed">>> {
const activeKeyId = getActiveKeyId();
const needs = await readBackupKeyNeeds(activeKeyId, now);
return [
...applicationKitKeys({
activeKeyId,
configuredKeyIds: getConfiguredKeyIds(),
fingerprintOf: getKeyFingerprint,
needs,
now,
}),
...backupKitKeys(
envValue("BACKUP_ENCRYPTION_KEY"),
envValue("BACKUP_ENCRYPTION_PREVIOUS_KEYS"),
),
...tokenKitKeys(envValue("API_TOKEN_HMAC_KEY")),
];
}
export async function readRecoveryKit(
now: Date = new Date(),
): Promise<RecoveryKit> {
const [listed, row] = await Promise.all([
listKitKeys(now),
prisma.recoveryKitConfirmation.findUnique({
where: { id: "singleton" },
}),
]);
const verdict = assessRecoveryKit(
listed,
row ? parseKitEntries(row.entries) : null,
);
const confirmedBy = row?.confirmedByUserId
? await prisma.user.findUnique({
where: { id: row.confirmedByUserId },
select: { id: true, email: true },
})
: null;
return {
keys: verdict.keys,
due: verdict.due,
unconfirmed: verdict.unconfirmed.map(({ kind, id }) => ({ kind, id })),
confirmedAt: row?.confirmedAt.toISOString() ?? null,
confirmedBy: confirmedBy ?? null,
platformHint: resolvePlatformHint(),
};
}
export type ConfirmRecoveryKitResult =
{ ok: true; kit: RecoveryKit } | { ok: false; kit: RecoveryKit };
/**
* Record the confirmation, but only for the kit the admin was shown: a page
* opened before a rotation or a new backup key must not confirm a key the
* admin never saw.
*/
export async function confirmRecoveryKit(input: {
shown: readonly KitEntry[];
userId: string;
now?: Date;
}): Promise<ConfirmRecoveryKitResult> {
const now = input.now ?? new Date();
const listed = await listKitKeys(now);
const entries: KitEntry[] = listed.map(({ kind, id, fingerprint }) => ({
kind,
id,
fingerprint,
}));
if (!sameKit(input.shown, entries)) {
return { ok: false, kit: await readRecoveryKit(now) };
}
const data = {
confirmedAt: now,
confirmedByUserId: input.userId,
entries: entries.map(({ kind, id, fingerprint }) => ({
kind,
id,
fingerprint,
})),
};
await prisma.recoveryKitConfirmation.upsert({
where: { id: "singleton" },
create: { id: "singleton", ...data },
update: data,
});
return { ok: true, kit: await readRecoveryKit(now) };
}
/**
* "Check my copy": does `value` decode to the key the kit lists as
* `kind`/`id` on this server? Constant time on the compared bytes; a value
* that does not decode is simply not a match.
*/
export function verifyKitKey(input: {
kind: KitKeyKind;
id: string;
value: string;
}): boolean {
const value = input.value.trim();
switch (input.kind) {
case "application":
return candidateMatchesKey(value, input.id);
case "backup": {
let candidate: Buffer;
try {
candidate = decodeBackupKey(value);
} catch {
return false;
}
const configured = [
envValue("BACKUP_ENCRYPTION_KEY"),
...(envValue("BACKUP_ENCRYPTION_PREVIOUS_KEYS") ?? "").split(","),
]
.map((raw) => raw?.trim() ?? "")
.filter((raw) => raw.length > 0)
.flatMap((raw) => {
try {
return [decodeBackupKey(raw)];
} catch {
return [];
}
})
.find((key) => offhostKeyId(key) === input.id);
return (
configured !== undefined &&
configured.length === candidate.length &&
timingSafeEqual(configured, candidate)
);
}
case "token": {
const configured = envValue("API_TOKEN_HMAC_KEY");
if (!configured || input.id !== KIT_KEY_SOURCES.token) return false;
// Digests, so the comparison is constant time whatever the lengths.
const digest = (s: string) => createHash("sha256").update(s).digest();
return timingSafeEqual(digest(value), digest(configured));
}
}
}