mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
fix(backup): take off-host copies out with the account, and say which keys backups still need
Deleting an account, deleting a managed profile, "Delete all data" and the admin wipe now write an off-host deletion request in the same transaction as the deletion. A new job (offhost-backup-purge, kicked straight away and run nightly as a backstop) lists the bucket once, deletes every <date>/user-<id>.json.enc object of each pending account and removes the request only when nothing of the account is left. A refused delete keeps the request with the bucket's answer and is retried; the off-host card shows how many deletions are outstanding and why. A copy the nightly run was writing while the account was deleted is taken back after the upload. The off-host card also reads the bucket's lifecycle rule and says plainly when there is none (every nightly copy kept forever) or when the credential may not read it. Every stored backup now records the key ids its content was written under, and every off-host upload records the same per key. The admin encryption view lists, per key, the stored copies that need it with the oldest date, and when the last off-host copy needing it expires. The "safe to drop" badge now requires that no backup still needs a retired key, instead of only that every row was rotated. Migration 0358 adds data_backups.inner_key_ids, the key-use table and the deletion requests.
This commit is contained in:
1 parent
405c996b6a
commit
1c7785c23b
32 files changed
+1590
-42
No files matched your search
@@ -29,12 +29,23 @@ vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
$queryRaw: (...a: unknown[]) => queryRawMock(...a),
|
||||
user: { findMany: (...a: unknown[]) => findManyMock(...a) },
|
||||
offhostPurgeRequest: {
|
||||
count: async () => 2,
|
||||
findFirst: async () => ({
|
||||
requestedAt: new Date("2026-09-20T10:00:00.000Z"),
|
||||
lastFailure: "AccessDenied",
|
||||
}),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const configuredMock = vi.fn(() => true);
|
||||
vi.mock("@/lib/jobs/offhost-backup", () => ({
|
||||
offhostBackupConfigured: () => configuredMock(),
|
||||
probeOffhostLifecycle: async () => ({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
}),
|
||||
}));
|
||||
|
||||
import { GET } from "../route";
|
||||
@@ -139,10 +150,12 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
|
||||
const body = await read();
|
||||
|
||||
expect(body.offhost).toEqual({
|
||||
expect(body.offhost).toMatchObject({
|
||||
configured: false,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
// No bucket to ask on a host that is not configured.
|
||||
lifecycle: { state: "unknown", expirationDays: null },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -196,4 +209,20 @@ describe("GET /api/admin/backups — off-host freshness", () => {
|
||||
]);
|
||||
expect(JSON.stringify(body.offhost)).not.toMatch(/bucket|endpoint|secret/i);
|
||||
});
|
||||
|
||||
it("carries the bucket's lifecycle verdict and the deletions it still owes", async () => {
|
||||
configuredMock.mockReturnValue(true);
|
||||
queryRawMock.mockResolvedValue([]);
|
||||
findManyMock.mockResolvedValue([]);
|
||||
const body = await read();
|
||||
expect(body.offhost.lifecycle).toEqual({
|
||||
state: "missing",
|
||||
expirationDays: null,
|
||||
});
|
||||
expect(body.offhost.pendingDeletions).toEqual({
|
||||
count: 2,
|
||||
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFailure: "AccessDenied",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -27,7 +27,10 @@ import { annotate } from "@/lib/logging/context";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
import { readLastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
@@ -164,6 +167,17 @@ export const GET = apiHandler(async () => {
|
||||
a.username.localeCompare(b.username),
|
||||
);
|
||||
|
||||
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
|
||||
prisma.offhostPurgeRequest.count(),
|
||||
prisma.offhostPurgeRequest.findFirst({
|
||||
orderBy: { requestedAt: "asc" },
|
||||
select: { requestedAt: true, lastFailure: true },
|
||||
}),
|
||||
configured
|
||||
? probeOffhostLifecycle()
|
||||
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
|
||||
]);
|
||||
|
||||
const payload: BackupsList = {
|
||||
rows: list,
|
||||
// Matches the retention window the backup-prune job enforces so the
|
||||
@@ -182,6 +196,12 @@ export const GET = apiHandler(async () => {
|
||||
// An unconfigured host has nothing to list, and the card says so rather
|
||||
// than painting an empty table that reads as "no problems".
|
||||
rows: configured ? offhostRows : [],
|
||||
pendingDeletions: {
|
||||
count: purgeCount,
|
||||
oldestRequestedAt: oldestPurge?.requestedAt.toISOString() ?? null,
|
||||
lastFailure: oldestPurge?.lastFailure ?? null,
|
||||
},
|
||||
lifecycle,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
@@ -108,6 +113,9 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// columns does not. Same classification contract as the model list.
|
||||
await tx.user.updateMany({ data: USER_RESET });
|
||||
|
||||
// Every account's nightly off-host copies go too.
|
||||
await requestOffhostPurge(tx, PURGE_ALL_SUBJECTS, "admin_wipe");
|
||||
|
||||
return perModel;
|
||||
},
|
||||
{
|
||||
@@ -138,6 +146,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// v1.16.9 — the wipe touched every user's rows; clear every cache
|
||||
// bucket so no per-user payload survives the reset.
|
||||
invalidateAllCaches();
|
||||
await kickOffhostPurge();
|
||||
|
||||
return apiSuccess({ cleared: true, deletedRows, models: counts });
|
||||
});
|
||||
@@ -19,6 +19,10 @@ import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getConfiguredKeyIds } from "@/lib/crypto";
|
||||
import {
|
||||
offhostBackupConfigured,
|
||||
probeOffhostLifecycle,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import {
|
||||
scanCorpus,
|
||||
type CorpusClient,
|
||||
@@ -100,6 +104,100 @@ async function readRotationState(): Promise<RotationState> {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Which keys the backups still need.
|
||||
*
|
||||
* Rotation re-encrypts every row in the database and re-seals every stored
|
||||
* backup's envelope, but the content INSIDE a backup keeps the key it was
|
||||
* written under: a restore writes it back verbatim. So "every row is on the
|
||||
* active key" is not the same as "the old key can go". This answers the second
|
||||
* question per key id, from what each copy recorded as it was written: the
|
||||
* stored copies that need it and the oldest of them, and for the off-host
|
||||
* bucket the last night an object needing it went in, plus how long the
|
||||
* bucket's lifecycle rule keeps objects after that.
|
||||
*/
|
||||
export interface BackupKeyNeeds {
|
||||
stored: Array<{ keyId: string; copies: number; oldestAt: string }>;
|
||||
/** Copies written before v1.39.3, which recorded no key ids. */
|
||||
unrecorded: { copies: number; oldestAt: string | null };
|
||||
offhost: Array<{
|
||||
keyId: string;
|
||||
firstWrittenAt: string;
|
||||
lastWrittenAt: string;
|
||||
/** When the last object needing this key expires; null when unknown. */
|
||||
neededUntil: string | null;
|
||||
}>;
|
||||
/** The bucket's expiry, when it has one the credential may read. */
|
||||
offhostExpirationDays: number | null;
|
||||
/** Retired key ids some backup still needs (never the active one). */
|
||||
retiredKeysStillNeeded: string[];
|
||||
}
|
||||
|
||||
async function readBackupKeyNeeds(
|
||||
activeKeyId: string,
|
||||
now: Date,
|
||||
): Promise<BackupKeyNeeds> {
|
||||
const [stored, unrecorded, offhostRows] = await Promise.all([
|
||||
prisma.$queryRaw<
|
||||
Array<{ key_id: string; copies: number; oldest_at: Date }>
|
||||
>`
|
||||
SELECT k AS key_id, count(*)::int AS copies, min(created_at) AS oldest_at
|
||||
FROM data_backups, unnest(inner_key_ids) AS k
|
||||
WHERE inner_key_ids_recorded
|
||||
GROUP BY k
|
||||
ORDER BY k
|
||||
`,
|
||||
prisma.dataBackup.aggregate({
|
||||
where: { innerKeyIdsRecorded: false },
|
||||
_count: { _all: true },
|
||||
_min: { createdAt: true },
|
||||
}),
|
||||
prisma.offhostBackupKeyUse.findMany({ orderBy: { keyId: "asc" } }),
|
||||
]);
|
||||
const lifecycle =
|
||||
offhostRows.length > 0 && offhostBackupConfigured()
|
||||
? await probeOffhostLifecycle()
|
||||
: null;
|
||||
const expirationDays =
|
||||
lifecycle?.state === "configured" ? lifecycle.expirationDays : null;
|
||||
|
||||
const offhost = offhostRows.map((row) => ({
|
||||
keyId: row.keyId,
|
||||
firstWrittenAt: row.firstWrittenAt.toISOString(),
|
||||
lastWrittenAt: row.lastWrittenAt.toISOString(),
|
||||
neededUntil:
|
||||
expirationDays === null
|
||||
? null
|
||||
: new Date(
|
||||
row.lastWrittenAt.getTime() + (expirationDays + 1) * 86_400_000,
|
||||
).toISOString(),
|
||||
}));
|
||||
|
||||
const needed = new Set<string>();
|
||||
for (const row of stored) needed.add(row.key_id);
|
||||
for (const row of offhost) {
|
||||
if (row.neededUntil === null || Date.parse(row.neededUntil) > +now) {
|
||||
needed.add(row.keyId);
|
||||
}
|
||||
}
|
||||
needed.delete(activeKeyId);
|
||||
|
||||
return {
|
||||
stored: stored.map((row) => ({
|
||||
keyId: row.key_id,
|
||||
copies: row.copies,
|
||||
oldestAt: row.oldest_at.toISOString(),
|
||||
})),
|
||||
unrecorded: {
|
||||
copies: unrecorded._count._all,
|
||||
oldestAt: unrecorded._min.createdAt?.toISOString() ?? null,
|
||||
},
|
||||
offhost,
|
||||
offhostExpirationDays: expirationDays,
|
||||
retiredKeysStillNeeded: [...needed].sort(),
|
||||
};
|
||||
}
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
await requireAdmin();
|
||||
annotate({ action: { name: "admin.encryption.status" } });
|
||||
@@ -114,6 +212,7 @@ export const GET = apiHandler(async () => {
|
||||
}
|
||||
|
||||
const rotation = await readRotationState();
|
||||
const backups = await readBackupKeyNeeds(scan.activeKeyId, new Date());
|
||||
|
||||
annotate({
|
||||
meta: {
|
||||
@@ -133,5 +232,14 @@ export const GET = apiHandler(async () => {
|
||||
staleRows: scan.staleRows,
|
||||
columns: scan.columns,
|
||||
rotation,
|
||||
backups,
|
||||
// Every row rotated AND no backup still needs a retired key: the one
|
||||
// state in which dropping an old key loses nothing. Copies that recorded
|
||||
// no key ids count against it, because they may hold any key that
|
||||
// existed when they were written.
|
||||
safeToDropRetiredKeys:
|
||||
scan.rotationComplete &&
|
||||
backups.retiredKeysStillNeeded.length === 0 &&
|
||||
backups.unrecorded.copies === 0,
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,10 @@ import {
|
||||
MFA_STEP_UP_MAX_AGE_SECONDS,
|
||||
} from "@/lib/api-handler";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import {
|
||||
deleteGuardianAccountWithLifecycle,
|
||||
LastManagedGuardianError,
|
||||
@@ -101,6 +105,10 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// genuinely complete erasure.
|
||||
await tx.auditLog.deleteMany({ where: { userId } });
|
||||
|
||||
// The nightly off-host copies go with the account. The request commits
|
||||
// with the deletion; the purge job deletes the objects.
|
||||
await requestOffhostPurge(tx, userId, "account_deleted");
|
||||
|
||||
// Delete user — all other related data is removed via onDelete: Cascade.
|
||||
await tx.user.delete({ where: { id: userId } });
|
||||
});
|
||||
@@ -115,6 +123,8 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await kickOffhostPurge();
|
||||
|
||||
annotate({
|
||||
action: { name: "settings.account.delete" },
|
||||
meta: { username },
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { apiSuccess, apiError, getClientIp } from "@/lib/api-response";
|
||||
import { NextRequest } from "next/server";
|
||||
import {
|
||||
@@ -101,6 +105,11 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// does not. Same classification contract as the model list.
|
||||
await tx.user.update({ where: { id: userId }, data: USER_RESET });
|
||||
|
||||
// The nightly off-host copies of the record are part of "all your
|
||||
// data". The request commits with the wipe; the purge job deletes
|
||||
// the objects and removes it.
|
||||
await requestOffhostPurge(tx, userId, "data_wiped");
|
||||
|
||||
return perModel;
|
||||
},
|
||||
{
|
||||
@@ -144,6 +153,7 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
|
||||
// v1.16.9 — every cached payload was built on the rows just deleted;
|
||||
// hard-evict the user's buckets so no surface serves pre-wipe data.
|
||||
invalidateUserData(userId);
|
||||
await kickOffhostPurge();
|
||||
|
||||
return apiSuccess({ cleared: true, deletedRows, models: counts });
|
||||
});
|
||||
@@ -23,6 +23,11 @@ vi.mock("@tanstack/react-query", () => ({
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { OffhostBackupsSection } from "../offhost-backups-section";
|
||||
|
||||
const QUIET = {
|
||||
pendingDeletions: { count: 0, oldestRequestedAt: null, lastFailure: null },
|
||||
lifecycle: { state: "configured" as const, expirationDays: 30 },
|
||||
};
|
||||
|
||||
function render(data: BackupsList["offhost"] | null): string {
|
||||
queryResult.current = {
|
||||
data:
|
||||
@@ -46,7 +51,12 @@ function render(data: BackupsList["offhost"] | null): string {
|
||||
|
||||
describe("<OffhostBackupsSection>", () => {
|
||||
it("says off-host backup is not configured rather than showing an empty list", () => {
|
||||
const html = render({ configured: false, periodHours: 24, rows: [] });
|
||||
const html = render({
|
||||
configured: false,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
...QUIET,
|
||||
});
|
||||
expect(html).toContain("Off-host backup is not configured");
|
||||
expect(html).not.toContain('data-slot="offhost-backup-rows"');
|
||||
});
|
||||
@@ -55,6 +65,7 @@ describe("<OffhostBackupsSection>", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
...QUIET,
|
||||
rows: [
|
||||
{
|
||||
userId: "u1",
|
||||
@@ -109,4 +120,33 @@ describe("<OffhostBackupsSection>", () => {
|
||||
// Never the bucket's own coordinates.
|
||||
expect(html).not.toMatch(/secret|access[- ]key/i);
|
||||
});
|
||||
|
||||
it("says when the bucket keeps copies forever, and when deletions are stuck", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
pendingDeletions: {
|
||||
count: 2,
|
||||
oldestRequestedAt: "2026-09-20T10:00:00.000Z",
|
||||
lastFailure: "AccessDenied",
|
||||
},
|
||||
lifecycle: { state: "missing", expirationDays: null },
|
||||
});
|
||||
expect(html).toContain('data-lifecycle-state="missing"');
|
||||
expect(html).toContain("no lifecycle rule");
|
||||
expect(html).toContain('data-slot="offhost-pending-deletions"');
|
||||
expect(html).toContain("AccessDenied");
|
||||
});
|
||||
|
||||
it("states the expiry the bucket enforces", () => {
|
||||
const html = render({
|
||||
configured: true,
|
||||
periodHours: 24,
|
||||
rows: [],
|
||||
...QUIET,
|
||||
});
|
||||
expect(html).toContain("after 30 days");
|
||||
expect(html).not.toContain('data-slot="offhost-pending-deletions"');
|
||||
});
|
||||
});
|
||||
@@ -13,7 +13,13 @@
|
||||
*/
|
||||
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { KeyRound, Loader2, RotateCw, ShieldCheck } from "lucide-react";
|
||||
import {
|
||||
Archive,
|
||||
KeyRound,
|
||||
Loader2,
|
||||
RotateCw,
|
||||
ShieldCheck,
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
AlertDialog,
|
||||
@@ -32,7 +38,7 @@ import { StatTile } from "@/components/admin/_stat-tile";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { SettingsCard } from "@/components/settings/settings-card";
|
||||
import { SettingsCardHeader } from "@/components/settings/_card-header";
|
||||
import { useTranslations } from "@/lib/i18n/context";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import { apiFetchRaw, apiGet } from "@/lib/api/api-fetch";
|
||||
import { getApiErrorMessage } from "./_shared";
|
||||
@@ -60,10 +66,25 @@ interface EncryptionStatus {
|
||||
lastCompletedAt: string | null;
|
||||
lastResult: { scanned: number; rotated: number; errors: number } | null;
|
||||
};
|
||||
/** Which keys the backups still need; see the status route. */
|
||||
backups: {
|
||||
stored: Array<{ keyId: string; copies: number; oldestAt: string }>;
|
||||
unrecorded: { copies: number; oldestAt: string | null };
|
||||
offhost: Array<{
|
||||
keyId: string;
|
||||
firstWrittenAt: string;
|
||||
lastWrittenAt: string;
|
||||
neededUntil: string | null;
|
||||
}>;
|
||||
offhostExpirationDays: number | null;
|
||||
retiredKeysStillNeeded: string[];
|
||||
};
|
||||
safeToDropRetiredKeys: boolean;
|
||||
}
|
||||
|
||||
export function EncryptionSection() {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const statusQuery = useQuery({
|
||||
@@ -166,10 +187,20 @@ export function EncryptionSection() {
|
||||
value={s.staleRows.toLocaleString()}
|
||||
/>
|
||||
</div>
|
||||
{s.rotationComplete ? (
|
||||
{s.safeToDropRetiredKeys ? (
|
||||
<Badge className="border-success/40 bg-success/15 text-success">
|
||||
{t("admin.section.encryption.safeToDropLegacy")}
|
||||
</Badge>
|
||||
) : s.rotationComplete ? (
|
||||
// Every row is on the active key, and that is not the whole
|
||||
// answer: a backup keeps the key its content was written under.
|
||||
<Badge variant="secondary" data-slot="encryption-backups-need-keys">
|
||||
{s.backups.retiredKeysStillNeeded.length > 0
|
||||
? t("admin.section.encryption.backupsStillNeed", {
|
||||
keys: s.backups.retiredKeysStillNeeded.join(", "),
|
||||
})
|
||||
: t("admin.section.encryption.backupsUnrecordedBadge")}
|
||||
</Badge>
|
||||
) : (
|
||||
<Badge variant="secondary">
|
||||
{t("admin.section.encryption.rotationIncomplete", {
|
||||
@@ -179,6 +210,56 @@ export function EncryptionSection() {
|
||||
)}
|
||||
</SettingsCard>
|
||||
|
||||
{/* ── Keys the backups still need ───────────────────────────── */}
|
||||
<SettingsCard>
|
||||
<SettingsCardHeader
|
||||
icon={Archive}
|
||||
title={t("admin.section.encryption.backupsTitle")}
|
||||
description={t("admin.section.encryption.backupsDescription")}
|
||||
/>
|
||||
<p className="text-sm">{t("admin.section.encryption.backupsDetail")}</p>
|
||||
<ul className="space-y-1 text-sm" data-slot="encryption-backup-keys">
|
||||
{s.backups.stored.map((row) => (
|
||||
<li key={`stored-${row.keyId}`}>
|
||||
{t("admin.section.encryption.backupsStoredLine", {
|
||||
key: row.keyId,
|
||||
count: row.copies,
|
||||
when: fmt.dateTime(row.oldestAt),
|
||||
})}
|
||||
</li>
|
||||
))}
|
||||
{s.backups.unrecorded.copies > 0 && s.backups.unrecorded.oldestAt ? (
|
||||
<li>
|
||||
{t("admin.section.encryption.backupsUnrecordedLine", {
|
||||
count: s.backups.unrecorded.copies,
|
||||
when: fmt.dateTime(s.backups.unrecorded.oldestAt),
|
||||
})}
|
||||
</li>
|
||||
) : null}
|
||||
{s.backups.offhost.map((row) => (
|
||||
<li key={`offhost-${row.keyId}`}>
|
||||
{row.neededUntil
|
||||
? t("admin.section.encryption.backupsOffhostLine", {
|
||||
key: row.keyId,
|
||||
when: fmt.dateTime(row.lastWrittenAt),
|
||||
until: fmt.dateTime(row.neededUntil),
|
||||
})
|
||||
: t("admin.section.encryption.backupsOffhostLineUnknown", {
|
||||
key: row.keyId,
|
||||
when: fmt.dateTime(row.lastWrittenAt),
|
||||
})}
|
||||
</li>
|
||||
))}
|
||||
{s.backups.stored.length === 0 &&
|
||||
s.backups.unrecorded.copies === 0 &&
|
||||
s.backups.offhost.length === 0 ? (
|
||||
<li className="text-muted-foreground">
|
||||
{t("admin.section.encryption.backupsNone")}
|
||||
</li>
|
||||
) : null}
|
||||
</ul>
|
||||
</SettingsCard>
|
||||
|
||||
{/* ── Rotation status + trigger ──────────────────────────────── */}
|
||||
<SettingsCard>
|
||||
<SettingsCardHeader
|
||||
|
||||
@@ -11,11 +11,12 @@
|
||||
* been refusing the worker's signature since March.
|
||||
*
|
||||
* Every verdict is computed server-side from the ledger the worker writes. The
|
||||
* page never lists the bucket: the worker's grant is deliberately PutObject,
|
||||
* GetObject and AbortMultipartUpload — the three the runbook documents, the
|
||||
* abort being what the multipart path needs — and putting a credentialed
|
||||
* listing call on the render path of an admin page would be a poor trade for
|
||||
* a number the database already holds.
|
||||
* page never lists the bucket: putting a credentialed listing call on the
|
||||
* render path of an admin page would be a poor trade for a number the database
|
||||
* already holds. It does read the bucket's lifecycle rule (one small call,
|
||||
* cached for ten minutes on the server), because whether old copies expire at
|
||||
* all is something only the bucket knows, and it states deletions of deleted
|
||||
* or wiped accounts that the bucket has not carried out yet.
|
||||
*/
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
@@ -191,6 +192,35 @@ export function OffhostBackupsSection() {
|
||||
hours: offhost.periodHours,
|
||||
})}
|
||||
</p>
|
||||
<p
|
||||
className="text-xs"
|
||||
data-slot="offhost-lifecycle"
|
||||
data-lifecycle-state={offhost.lifecycle.state}
|
||||
>
|
||||
{offhost.lifecycle.state === "configured"
|
||||
? t("admin.section.backups.offhost.lifecycleConfigured", {
|
||||
days: offhost.lifecycle.expirationDays ?? 0,
|
||||
})
|
||||
: offhost.lifecycle.state === "missing"
|
||||
? t("admin.section.backups.offhost.lifecycleMissing")
|
||||
: t("admin.section.backups.offhost.lifecycleUnknown")}
|
||||
</p>
|
||||
{offhost.pendingDeletions.count > 0 ? (
|
||||
<p
|
||||
role="status"
|
||||
className="text-xs"
|
||||
data-slot="offhost-pending-deletions"
|
||||
>
|
||||
{offhost.pendingDeletions.lastFailure
|
||||
? t("admin.section.backups.offhost.pendingDeletionsFailing", {
|
||||
count: offhost.pendingDeletions.count,
|
||||
error: offhost.pendingDeletions.lastFailure,
|
||||
})
|
||||
: t("admin.section.backups.offhost.pendingDeletions", {
|
||||
count: offhost.pendingDeletions.count,
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
</SettingsCard>
|
||||
|
||||
@@ -293,6 +293,10 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
|
||||
AppSettings: "instance-wide configuration singleton",
|
||||
RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring",
|
||||
HostMetric: "instance-wide host telemetry, not attributable to an account",
|
||||
OffhostBackupKeyUse:
|
||||
"per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
|
||||
OffhostPurgeRequest:
|
||||
"the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
|
||||
CycleSymptomCategory:
|
||||
"seeded catalogue shared by every account; a user's custom symptoms live on CycleSymptom",
|
||||
IllnessSymptom: "seeded symptom catalogue shared by every account",
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
type BackupJsonProducer,
|
||||
type PackBackupOptions,
|
||||
} from "@/lib/export/backup-blob";
|
||||
import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
|
||||
import {
|
||||
BackupIntegrityError,
|
||||
newChunkStreamId,
|
||||
@@ -112,6 +113,11 @@ export async function storeBackupBlob(
|
||||
await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
|
||||
|
||||
const streamId = newChunkStreamId();
|
||||
// The key ids the copy's content was written under, read from the JSON
|
||||
// on its way into the pieces. Rotation re-seals the pieces but never
|
||||
// what is inside them, so this is what tells an operator how long a
|
||||
// retired key is still needed (`GET /api/admin/encryption/status`).
|
||||
const keyScanner = createBackupKeyIdTextScanner();
|
||||
const { chunks, bytes } = await packBackupChunks(
|
||||
async (sealed, seq) => {
|
||||
await tx.dataBackupChunk.create({
|
||||
@@ -120,7 +126,11 @@ export async function storeBackupBlob(
|
||||
});
|
||||
},
|
||||
streamId,
|
||||
producer,
|
||||
(write) =>
|
||||
producer(async (chunk) => {
|
||||
keyScanner.feed(chunk);
|
||||
await write(chunk);
|
||||
}),
|
||||
options,
|
||||
);
|
||||
|
||||
@@ -135,6 +145,8 @@ export async function storeBackupBlob(
|
||||
...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}),
|
||||
chunkCount: chunks,
|
||||
chunkStreamId: streamId,
|
||||
innerKeyIds: keyScanner.keyIds(),
|
||||
innerKeyIdsRecorded: true,
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -32,6 +32,14 @@ import {
|
||||
const ENC_KEY =
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
/** The purge ledger and key-use rows the run touches after each upload. */
|
||||
function purgeLedgerMocks() {
|
||||
return {
|
||||
offhostPurgeRequest: { count: vi.fn().mockResolvedValue(0) },
|
||||
offhostBackupKeyUse: { upsert: vi.fn() },
|
||||
};
|
||||
}
|
||||
|
||||
describe("offhost-backup envelope", () => {
|
||||
it("encrypts and decrypts JSON round-trip", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
@@ -184,8 +192,10 @@ describe("runOffhostBackup", () => {
|
||||
s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0]));
|
||||
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
measurement: { findMany: vi.fn().mockResolvedValue([]) },
|
||||
@@ -248,11 +258,43 @@ describe("runOffhostBackup", () => {
|
||||
expect(first.create.lastSuccessAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("takes back the copy of an account deleted while it was written", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const ledger = purgeLedgerMocks();
|
||||
const prisma = {
|
||||
...ledger,
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
// u1 is gone by the time its upload lands; u2 is not.
|
||||
count: vi.fn(async ({ where }: { where: { id: string } }) =>
|
||||
where.id === "u1" ? 0 : 1,
|
||||
),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
prisma as any,
|
||||
s3,
|
||||
new Date("2026-05-08T03:00:00Z"),
|
||||
);
|
||||
expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(false);
|
||||
expect(s3.store.has("2026-05-08/user-u2.json.enc")).toBe(true);
|
||||
expect(s3.deleteObject).toHaveBeenCalledWith("2026-05-08/user-u1.json.enc");
|
||||
expect(report.uploaded).toBe(1);
|
||||
// No ledger row for an account that no longer exists.
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uploads the canonical builder output without reshaping it", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
const canonicalPayload = {
|
||||
schemaVersion: BACKUP_SCHEMA_VERSION,
|
||||
@@ -310,8 +352,10 @@ describe("runOffhostBackup", () => {
|
||||
const s3 = makeS3Mock();
|
||||
const upsert = vi.fn();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
};
|
||||
@@ -357,10 +401,14 @@ describe("runOffhostBackup", () => {
|
||||
it("still reports a landed object as uploaded when the ledger write fails", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: {
|
||||
upsert: vi.fn().mockRejectedValue(new Error("pool timeout")),
|
||||
},
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
|
||||
const report = await runOffhostBackup(
|
||||
@@ -382,8 +430,12 @@ describe("runOffhostBackup", () => {
|
||||
it("refuses an account whose object outgrows one multipart upload", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
mocks.buildFullBackupPayload.mockResolvedValue({
|
||||
payload: {
|
||||
@@ -419,8 +471,12 @@ describe("runOffhostBackup", () => {
|
||||
it("reports the largest object it wrote", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
...purgeLedgerMocks(),
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
user: {
|
||||
count: vi.fn().mockResolvedValue(1),
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
|
||||
},
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
prisma as never,
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Off-host copies leave the bucket with the account.
|
||||
*
|
||||
* The bucket is a map behind the same `S3Like` boundary the real client
|
||||
* implements; the request ledger is an in-memory table. Mutations that must
|
||||
* turn this red: drop the `deleteObject` call (the objects stay), remove the
|
||||
* request on failure too (the refusal case loses the request), or match on the
|
||||
* key prefix instead of the full account pattern (another account's objects,
|
||||
* or the health-check probe, go too).
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
|
||||
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
processOffhostPurges,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
requestOffhostPurge,
|
||||
} from "../offhost-purge";
|
||||
|
||||
interface Row {
|
||||
id: string;
|
||||
subjectId: string;
|
||||
reason: string;
|
||||
requestedAt: Date;
|
||||
attempts: number;
|
||||
lastAttemptAt: Date | null;
|
||||
lastFailure: string | null;
|
||||
}
|
||||
|
||||
function makeLedger(existingUsers: string[] = []) {
|
||||
const rows: Row[] = [];
|
||||
let seq = 0;
|
||||
const prisma = {
|
||||
offhostPurgeRequest: {
|
||||
create: vi.fn(
|
||||
async ({ data }: { data: { subjectId: string; reason: string } }) => {
|
||||
const row: Row = {
|
||||
id: `r${++seq}`,
|
||||
subjectId: data.subjectId,
|
||||
reason: data.reason,
|
||||
requestedAt: new Date(Date.now() + seq),
|
||||
attempts: 0,
|
||||
lastAttemptAt: null,
|
||||
lastFailure: null,
|
||||
};
|
||||
rows.push(row);
|
||||
return { id: row.id };
|
||||
},
|
||||
),
|
||||
findMany: vi.fn(async () =>
|
||||
[...rows].sort((a, b) => +a.requestedAt - +b.requestedAt),
|
||||
),
|
||||
delete: vi.fn(async ({ where }: { where: { id: string } }) => {
|
||||
rows.splice(
|
||||
rows.findIndex((r) => r.id === where.id),
|
||||
1,
|
||||
);
|
||||
}),
|
||||
updateMany: vi.fn(
|
||||
async ({
|
||||
where,
|
||||
data,
|
||||
}: {
|
||||
where: { id: { in: string[] } };
|
||||
data: { lastAttemptAt: Date; lastFailure: string };
|
||||
}) => {
|
||||
for (const row of rows) {
|
||||
if (!where.id.in.includes(row.id)) continue;
|
||||
row.attempts += 1;
|
||||
row.lastAttemptAt = data.lastAttemptAt;
|
||||
row.lastFailure = data.lastFailure;
|
||||
}
|
||||
},
|
||||
),
|
||||
},
|
||||
user: {
|
||||
count: vi.fn(async ({ where }: { where: { id: string } }) =>
|
||||
existingUsers.includes(where.id) ? 1 : 0,
|
||||
),
|
||||
},
|
||||
};
|
||||
return { rows, prisma };
|
||||
}
|
||||
|
||||
function makeBucket(keys: string[]) {
|
||||
const store = new Set(keys);
|
||||
return {
|
||||
store,
|
||||
putObject: vi.fn(),
|
||||
putStream: vi.fn(),
|
||||
getObject: vi.fn(),
|
||||
headObject: vi.fn(),
|
||||
listObjects: vi.fn(async (prefix: string) =>
|
||||
[...store].filter((k) => k.startsWith(prefix)).map((key) => ({ key })),
|
||||
),
|
||||
deleteObject: vi.fn(async (key: string) => {
|
||||
store.delete(key);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const BUCKET = [
|
||||
"2026-09-01/user-gone.json.enc",
|
||||
"2026-09-02/user-gone.json.enc",
|
||||
"2026-09-02/user-gone2.json.enc",
|
||||
"2026-09-02/user-stays.json.enc",
|
||||
"_healthcheck/1.bin",
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.clearAllMocks();
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "https://r2.example");
|
||||
vi.stubEnv("BACKUP_S3_BUCKET", "hl-backups");
|
||||
vi.stubEnv("BACKUP_S3_ACCESS_KEY", "AKIA");
|
||||
vi.stubEnv("BACKUP_S3_SECRET_KEY", "secret");
|
||||
vi.stubEnv("BACKUP_ENCRYPTION_KEY", "ab".repeat(32));
|
||||
});
|
||||
|
||||
describe("off-host purge", () => {
|
||||
it("deletes every copy of the account, and nothing else", async () => {
|
||||
const { rows, prisma } = makeLedger();
|
||||
await requestOffhostPurge(prisma as never, "gone", "account_deleted");
|
||||
const bucket = makeBucket(BUCKET);
|
||||
|
||||
const report = await processOffhostPurges(prisma as never, bucket);
|
||||
|
||||
expect([...bucket.store].sort()).toEqual([
|
||||
"2026-09-02/user-gone2.json.enc",
|
||||
"2026-09-02/user-stays.json.enc",
|
||||
"_healthcheck/1.bin",
|
||||
]);
|
||||
expect(report).toMatchObject({
|
||||
completed: 1,
|
||||
failed: 0,
|
||||
objectsDeleted: 2,
|
||||
});
|
||||
expect(rows).toHaveLength(0);
|
||||
// The account is gone, and so is its audit trail: the receipt carries the
|
||||
// reason and the count, not the account.
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"offhost.backup.purged",
|
||||
expect.objectContaining({
|
||||
userId: null,
|
||||
details: expect.objectContaining({
|
||||
reason: "account_deleted",
|
||||
objectsDeleted: 2,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the request, with the bucket's answer, when a delete is refused", async () => {
|
||||
const { rows, prisma } = makeLedger(["gone"]);
|
||||
await requestOffhostPurge(prisma as never, "gone", "data_wiped");
|
||||
const bucket = makeBucket(BUCKET);
|
||||
bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied"));
|
||||
|
||||
const report = await processOffhostPurges(prisma as never, bucket);
|
||||
|
||||
expect(report).toMatchObject({ completed: 0, failed: 1 });
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" });
|
||||
|
||||
// The next run finishes it, and the account that still exists keeps the
|
||||
// receipt.
|
||||
const again = await processOffhostPurges(prisma as never, bucket);
|
||||
expect(again.completed).toBe(1);
|
||||
expect(rows).toHaveLength(0);
|
||||
expect(auditLog).toHaveBeenCalledWith(
|
||||
"offhost.backup.purged",
|
||||
expect.objectContaining({ userId: "gone" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("clears every account object for the admin wipe", async () => {
|
||||
const { prisma } = makeLedger();
|
||||
await requestOffhostPurge(
|
||||
prisma as never,
|
||||
PURGE_ALL_SUBJECTS,
|
||||
"admin_wipe",
|
||||
);
|
||||
const bucket = makeBucket(BUCKET);
|
||||
await processOffhostPurges(prisma as never, bucket);
|
||||
expect([...bucket.store]).toEqual(["_healthcheck/1.bin"]);
|
||||
});
|
||||
|
||||
it("records nothing on a host without off-host backup", async () => {
|
||||
vi.stubEnv("BACKUP_S3_ENDPOINT", "");
|
||||
const { rows, prisma } = makeLedger();
|
||||
expect(
|
||||
await requestOffhostPurge(prisma as never, "gone", "account_deleted"),
|
||||
).toBe(false);
|
||||
expect(rows).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not list the bucket when nothing is pending", async () => {
|
||||
const { prisma } = makeLedger();
|
||||
const bucket = makeBucket(BUCKET);
|
||||
await processOffhostPurges(prisma as never, bucket);
|
||||
expect(bucket.listObjects).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -144,6 +144,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"offhost_backup_oversized",
|
||||
"offhost_backup_total_users",
|
||||
"offhost_backup_uploaded",
|
||||
"offhost_purge_completed",
|
||||
"offhost_purge_failed",
|
||||
"offhost_purge_objects_deleted",
|
||||
"offhost_purge_pending",
|
||||
"outcome",
|
||||
"passes_failed",
|
||||
"per_sample_rows_deleted",
|
||||
|
||||
@@ -27,6 +27,7 @@ import { Readable, Transform } from "node:stream";
|
||||
import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto";
|
||||
import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
|
||||
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
|
||||
@@ -489,6 +490,24 @@ export interface S3Like {
|
||||
prefix: string,
|
||||
): Promise<Array<{ key: string; lastModified?: Date }>>;
|
||||
deleteObject(key: string): Promise<void>;
|
||||
/**
|
||||
* The bucket's lifecycle rule, as far as the credential may read it.
|
||||
* Optional so a test double that has no opinion leaves it out.
|
||||
*/
|
||||
getLifecycle?(): Promise<OffhostLifecycle>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the bucket retires old copies by itself.
|
||||
*
|
||||
* `configured` names the shortest expiry of an enabled rule; `missing` means
|
||||
* the bucket answered that it has no rule; `unknown` means it would not say
|
||||
* (a credential without `s3:GetLifecycleConfiguration`, a target that does not
|
||||
* implement the call, a network failure) and says nothing either way.
|
||||
*/
|
||||
export interface OffhostLifecycle {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
}
|
||||
|
||||
export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
@@ -601,9 +620,78 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
|
||||
new mod.DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }),
|
||||
);
|
||||
},
|
||||
getLifecycle: async () => {
|
||||
try {
|
||||
const out = await client.send(
|
||||
new mod.GetBucketLifecycleConfigurationCommand({
|
||||
Bucket: cfg.bucket,
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(5_000) },
|
||||
);
|
||||
const days = (out.Rules ?? [])
|
||||
.filter((rule) => rule.Status === "Enabled")
|
||||
.map((rule) => rule.Expiration?.Days)
|
||||
.filter((d): d is number => typeof d === "number" && d > 0);
|
||||
return days.length > 0
|
||||
? { state: "configured", expirationDays: Math.min(...days) }
|
||||
: { state: "missing", expirationDays: null };
|
||||
} catch (err) {
|
||||
const name = (err as { name?: string; Code?: string }).name;
|
||||
const code = (err as { Code?: string }).Code;
|
||||
if (
|
||||
name === "NoSuchLifecycleConfiguration" ||
|
||||
code === "NoSuchLifecycleConfiguration"
|
||||
) {
|
||||
return { state: "missing", expirationDays: null };
|
||||
}
|
||||
return { state: "unknown", expirationDays: null };
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const LIFECYCLE_CACHE_MS = 10 * 60 * 1000;
|
||||
let lifecycleCache: { at: number; value: OffhostLifecycle } | null = null;
|
||||
|
||||
/**
|
||||
* The bucket's lifecycle rule for the admin card, cached for ten minutes so a
|
||||
* page render does not become a bucket call. Never throws.
|
||||
*/
|
||||
export async function probeOffhostLifecycle(
|
||||
s3Override?: S3Like,
|
||||
now: number = Date.now(),
|
||||
): Promise<OffhostLifecycle> {
|
||||
if (
|
||||
!s3Override &&
|
||||
lifecycleCache &&
|
||||
now - lifecycleCache.at < LIFECYCLE_CACHE_MS
|
||||
) {
|
||||
return lifecycleCache.value;
|
||||
}
|
||||
const cfg = loadOffhostConfigSafe();
|
||||
if (!cfg) return { state: "unknown", expirationDays: null };
|
||||
let value: OffhostLifecycle;
|
||||
try {
|
||||
const s3 = s3Override ?? (await getS3Client(cfg));
|
||||
value = s3.getLifecycle
|
||||
? await s3.getLifecycle()
|
||||
: { state: "unknown", expirationDays: null };
|
||||
} catch {
|
||||
value = { state: "unknown", expirationDays: null };
|
||||
}
|
||||
if (!s3Override) lifecycleCache = { at: now, value };
|
||||
return value;
|
||||
}
|
||||
|
||||
/** `loadOffhostConfig`, answering null for a malformed key instead of throwing. */
|
||||
function loadOffhostConfigSafe(): OffhostBackupConfig | null {
|
||||
try {
|
||||
return loadOffhostConfig();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
interface BackupRunReport {
|
||||
config: { endpoint: string; bucket: string; region: string };
|
||||
uploaded: number;
|
||||
@@ -642,6 +730,30 @@ export interface RunOffhostBackupOptions extends UploadBackupOptions {
|
||||
shouldStop?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Note, per key id, that an object needing it went into the bucket now. Never
|
||||
* fails the account: the object is in the bucket whatever this row says.
|
||||
*/
|
||||
async function recordOffhostKeyUse(
|
||||
prisma: PrismaClient,
|
||||
keyIds: readonly string[],
|
||||
at: Date,
|
||||
): Promise<void> {
|
||||
for (const keyId of keyIds) {
|
||||
try {
|
||||
await prisma.offhostBackupKeyUse.upsert({
|
||||
where: { keyId },
|
||||
create: { keyId, firstWrittenAt: at, lastWrittenAt: at },
|
||||
update: { lastWrittenAt: at },
|
||||
});
|
||||
} catch (err) {
|
||||
getEvent()?.addWarning(
|
||||
`offhost-backup key-use write failed: ${(err as Error).message?.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function runOffhostBackup(
|
||||
prisma: PrismaClient,
|
||||
s3Override?: S3Like,
|
||||
@@ -689,24 +801,54 @@ export async function runOffhostBackup(
|
||||
break;
|
||||
}
|
||||
let objectBytes: number | null = null;
|
||||
const objectKey = `${dateKey}/user-${user.id}.json.enc`;
|
||||
const accountStartedAt = new Date();
|
||||
// The key ids the object's content needs, read as it is written: the
|
||||
// admin encryption view uses them to say how long a retired key is still
|
||||
// needed for what is in the bucket.
|
||||
const keyScanner = createBackupKeyIdTextScanner();
|
||||
try {
|
||||
objectBytes = await uploadEncryptedBackup(
|
||||
s3,
|
||||
`${dateKey}/user-${user.id}.json.enc`,
|
||||
objectKey,
|
||||
cfg.encryptionKey,
|
||||
// The same writer the weekly in-database pass uses. The payload
|
||||
// builder was always shared; everything after it was not, which is why
|
||||
// this job kept dying on a record the weekly one had learned to
|
||||
// survive.
|
||||
(write) =>
|
||||
streamFullBackupJson(prisma, user.id, write, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
}),
|
||||
streamFullBackupJson(
|
||||
prisma,
|
||||
user.id,
|
||||
async (chunk) => {
|
||||
keyScanner.feed(chunk);
|
||||
await write(chunk);
|
||||
},
|
||||
{
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
},
|
||||
),
|
||||
options,
|
||||
);
|
||||
// The account was deleted or wiped while its copy was being written.
|
||||
// The purge that request started may already have run, so the copy
|
||||
// this run just put there is removed here rather than left behind.
|
||||
const purgedMeanwhile =
|
||||
(await prisma.offhostPurgeRequest.count({
|
||||
where: {
|
||||
subjectId: { in: [user.id, "*"] },
|
||||
requestedAt: { gte: accountStartedAt },
|
||||
},
|
||||
})) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0;
|
||||
if (purgedMeanwhile) {
|
||||
await s3.deleteObject(objectKey);
|
||||
objectBytes = null;
|
||||
continue;
|
||||
}
|
||||
largestObjectBytes = Math.max(largestObjectBytes, objectBytes);
|
||||
uploaded++;
|
||||
await recordOffhostKeyUse(prisma, keyScanner.keyIds(), new Date());
|
||||
} catch (err) {
|
||||
failed++;
|
||||
if (err instanceof OffhostBackupTooLargeError) oversized++;
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* Off-host copies leave the bucket with the account.
|
||||
*
|
||||
* The nightly job writes `<YYYY-MM-DD>/user-<id>.json.enc` for every account,
|
||||
* and until v1.39.3 nothing ever removed one: deleting an account, or wiping
|
||||
* its data, took the rows in this database and left every nightly copy of the
|
||||
* record in the operator's bucket for as long as its lifecycle rule kept them,
|
||||
* or forever where no rule was set. A confirmation that says "permanently
|
||||
* delete" has to mean the off-host copies too.
|
||||
*
|
||||
* How. The deletion writes an `OffhostPurgeRequest` in its own transaction, so
|
||||
* the request exists exactly when the deletion committed. This job then lists
|
||||
* the bucket once, deletes every object of every pending account, and removes
|
||||
* the request when nothing of the account is left. A request the bucket
|
||||
* refuses (a credential without DeleteObject, an outage) stays, counts its
|
||||
* attempts and keeps the bucket's answer, which the admin off-host card shows;
|
||||
* the job runs again every night and after every nightly upload, so a late
|
||||
* copy the upload wrote for an account deleted while it ran is caught too.
|
||||
*
|
||||
* `subjectId` "*" is the admin wipe of every account: every account object
|
||||
* goes. The health-check probes and anything else in the bucket that is not
|
||||
* an account object is never touched.
|
||||
*/
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import {
|
||||
getS3Client,
|
||||
loadOffhostConfig,
|
||||
offhostBackupConfigured,
|
||||
type S3Like,
|
||||
} from "@/lib/jobs/offhost-backup";
|
||||
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
|
||||
import { getWorkerPrisma } from "@/lib/jobs/reminder/shared";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
|
||||
export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge";
|
||||
/** 03:40, after the nightly upload (02:30) has had its hour. */
|
||||
export const OFFHOST_PURGE_CRON = "40 3 * * *";
|
||||
|
||||
/** Every subject, for the admin wipe of every account. */
|
||||
export const PURGE_ALL_SUBJECTS = "*";
|
||||
|
||||
export type OffhostPurgeReason =
|
||||
"account_deleted" | "data_wiped" | "managed_profile_deleted" | "admin_wipe";
|
||||
|
||||
/** An account object: `<date>/user-<id>.json.enc`. */
|
||||
const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/;
|
||||
|
||||
/** Requests one run works through; the next run takes the rest. */
|
||||
const REQUESTS_PER_RUN = 200;
|
||||
|
||||
/** Deletes one run may issue, so a huge bucket cannot run the job forever. */
|
||||
const MAX_DELETES_PER_RUN = 50_000;
|
||||
|
||||
type Db = Pick<Prisma.TransactionClient, "offhostPurgeRequest">;
|
||||
|
||||
/**
|
||||
* Record that `subjectId`'s off-host copies have to go. Call it with the
|
||||
* transaction that deletes the account or wipes its data, so the request and
|
||||
* the deletion commit together. A host without off-host backup configured has
|
||||
* nothing in any bucket this could reach, and records nothing.
|
||||
*/
|
||||
export async function requestOffhostPurge(
|
||||
db: Db,
|
||||
subjectId: string,
|
||||
reason: OffhostPurgeReason,
|
||||
): Promise<boolean> {
|
||||
if (!offhostBackupConfigured()) return false;
|
||||
await db.offhostPurgeRequest.create({
|
||||
data: { subjectId, reason },
|
||||
select: { id: true },
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wake the purge job now rather than at its nightly slot. Best effort: the
|
||||
* request row is what guarantees the purge, and the cron picks it up if the
|
||||
* queue is not reachable from here.
|
||||
*/
|
||||
export async function kickOffhostPurge(): Promise<void> {
|
||||
try {
|
||||
await getGlobalBoss()?.send(
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
{},
|
||||
{
|
||||
singletonKey: OFFHOST_PURGE_QUEUE,
|
||||
retryLimit: 3,
|
||||
retryDelay: 300,
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
// The cron is the backstop.
|
||||
}
|
||||
}
|
||||
|
||||
export interface OffhostPurgeReport {
|
||||
pending: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
objectsDeleted: number;
|
||||
}
|
||||
|
||||
function trimError(err: unknown): string {
|
||||
return (err instanceof Error ? err.message : String(err)).slice(0, 300);
|
||||
}
|
||||
|
||||
/**
|
||||
* Work through the pending requests: list the bucket once, delete every
|
||||
* object of every pending subject, remove each request whose objects are all
|
||||
* gone. Answers what it did.
|
||||
*/
|
||||
export async function processOffhostPurges(
|
||||
prisma: PrismaClient,
|
||||
s3Override?: S3Like,
|
||||
): Promise<OffhostPurgeReport> {
|
||||
const requests = await prisma.offhostPurgeRequest.findMany({
|
||||
orderBy: { requestedAt: "asc" },
|
||||
take: REQUESTS_PER_RUN,
|
||||
});
|
||||
const report: OffhostPurgeReport = {
|
||||
pending: requests.length,
|
||||
completed: 0,
|
||||
failed: 0,
|
||||
objectsDeleted: 0,
|
||||
};
|
||||
if (requests.length === 0) return report;
|
||||
|
||||
const cfg = loadOffhostConfig();
|
||||
if (!cfg) {
|
||||
// Configuration removed after the requests were written: leave them, an
|
||||
// operator who puts it back gets the purge.
|
||||
return report;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const markFailed = async (ids: string[], err: unknown) => {
|
||||
await prisma.offhostPurgeRequest.updateMany({
|
||||
where: { id: { in: ids } },
|
||||
data: {
|
||||
attempts: { increment: 1 },
|
||||
lastAttemptAt: now,
|
||||
lastFailure: trimError(err),
|
||||
},
|
||||
});
|
||||
report.failed += ids.length;
|
||||
};
|
||||
|
||||
let objects: Array<{ key: string }>;
|
||||
let s3: S3Like;
|
||||
try {
|
||||
s3 = s3Override ?? (await getS3Client(cfg));
|
||||
objects = await s3.listObjects("");
|
||||
} catch (err) {
|
||||
await markFailed(
|
||||
requests.map((r) => r.id),
|
||||
err,
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
const everything = requests.some((r) => r.subjectId === PURGE_ALL_SUBJECTS);
|
||||
const subjects = new Set(requests.map((r) => r.subjectId));
|
||||
const bySubject = new Map<string, string[]>();
|
||||
for (const { key } of objects) {
|
||||
const match = ACCOUNT_OBJECT.exec(key);
|
||||
if (!match) continue;
|
||||
const subject = match[1];
|
||||
if (!everything && !subjects.has(subject)) continue;
|
||||
const list = bySubject.get(subject) ?? [];
|
||||
list.push(key);
|
||||
bySubject.set(subject, list);
|
||||
}
|
||||
|
||||
// One subject at a time, so a refusal part-way fails only the subjects it
|
||||
// reached, and a request is removed only when its own objects are gone.
|
||||
const failedSubjects = new Map<string, unknown>();
|
||||
let deletes = 0;
|
||||
for (const [subject, keys] of bySubject) {
|
||||
for (const key of keys) {
|
||||
if (deletes >= MAX_DELETES_PER_RUN) {
|
||||
failedSubjects.set(
|
||||
subject,
|
||||
new Error("Deletion budget for one run reached; continues next run"),
|
||||
);
|
||||
break;
|
||||
}
|
||||
try {
|
||||
await s3.deleteObject(key);
|
||||
deletes++;
|
||||
report.objectsDeleted++;
|
||||
} catch (err) {
|
||||
failedSubjects.set(subject, err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const request of requests) {
|
||||
const failure =
|
||||
request.subjectId === PURGE_ALL_SUBJECTS
|
||||
? failedSubjects.size > 0
|
||||
? [...failedSubjects.values()][0]
|
||||
: null
|
||||
: failedSubjects.get(request.subjectId);
|
||||
if (failure) {
|
||||
await markFailed([request.id], failure);
|
||||
continue;
|
||||
}
|
||||
const deleted =
|
||||
request.subjectId === PURGE_ALL_SUBJECTS
|
||||
? [...bySubject.values()].reduce((sum, keys) => sum + keys.length, 0)
|
||||
: (bySubject.get(request.subjectId)?.length ?? 0);
|
||||
await prisma.offhostPurgeRequest.delete({ where: { id: request.id } });
|
||||
report.completed++;
|
||||
// The account a deleted subject named is gone, and so is its audit
|
||||
// history; the receipt names the reason and the count, not the account.
|
||||
// A wiped account still exists and keeps the receipt as its own.
|
||||
const accountRemains =
|
||||
request.reason === "data_wiped" &&
|
||||
(await prisma.user.count({ where: { id: request.subjectId } })) > 0;
|
||||
await auditLog("offhost.backup.purged", {
|
||||
userId: accountRemains ? request.subjectId : null,
|
||||
actorUserId: null,
|
||||
details: {
|
||||
reason: request.reason,
|
||||
objectsDeleted: deleted,
|
||||
attempts: request.attempts + 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function handleOffhostPurge(
|
||||
jobs: Job<object>[],
|
||||
): Promise<JobOutcome> {
|
||||
void jobs;
|
||||
return withBackgroundEvent("job.offhost_purge", async (evt) => {
|
||||
try {
|
||||
const report = await processOffhostPurges(getWorkerPrisma());
|
||||
evt.addMeta("offhost_purge_pending", report.pending);
|
||||
evt.addMeta("offhost_purge_completed", report.completed);
|
||||
evt.addMeta("offhost_purge_failed", report.failed);
|
||||
evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted);
|
||||
if (report.failed > 0) {
|
||||
await reportWorkerError(
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
new Error(
|
||||
`${report.failed} off-host deletion request(s) could not be completed; they stay pending and are retried nightly. Check that the bucket credential allows DeleteObject.`,
|
||||
),
|
||||
);
|
||||
}
|
||||
return jobDone({
|
||||
offhost_purge_pending: report.pending,
|
||||
offhost_purge_completed: report.completed,
|
||||
offhost_purge_failed: report.failed,
|
||||
offhost_purge_objects_deleted: report.objectsDeleted,
|
||||
});
|
||||
} catch (err) {
|
||||
evt.addWarning(`offhost-purge failed: ${err}`);
|
||||
await reportWorkerError(OFFHOST_PURGE_QUEUE, err);
|
||||
return jobFailed("offhost purge failed", err);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -67,6 +67,11 @@ import {
|
||||
enqueueBootTimeIntakeSlotDedup,
|
||||
type IntakeSlotDedupPayload,
|
||||
} from "@/lib/medications/intake-slot-dedup";
|
||||
import {
|
||||
handleOffhostPurge,
|
||||
OFFHOST_PURGE_CRON,
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import {
|
||||
handleRestoreDrill,
|
||||
RESTORE_DRILL_CRON,
|
||||
@@ -353,6 +358,10 @@ const allQueues = [
|
||||
STEP_UP_ELEVATION_CLEANUP_QUEUE,
|
||||
OFFHOST_BACKUP_QUEUE,
|
||||
RESTORE_DRILL_QUEUE,
|
||||
// v1.39.3 — deleting an account or wiping its data takes its off-host
|
||||
// copies out of the bucket. Without this entry the requests would be
|
||||
// written and never worked.
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
// v1.39.1 — an admin's restore of a stored backup, run off the request, and
|
||||
// the boot sweep that re-queues one a stopped worker left running. Without
|
||||
// this entry every restore request would queue a job nobody works.
|
||||
@@ -505,6 +514,9 @@ const schedules: ScheduleEntry[] = [
|
||||
],
|
||||
[OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON],
|
||||
[RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON],
|
||||
// Nightly backstop for the purge the deletion kicks straight away: a
|
||||
// request the bucket refused, or one written while the queue was down.
|
||||
[OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry],
|
||||
[HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry],
|
||||
[FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry],
|
||||
// v1.4.37 — hourly geo backfill. The helper is idempotent + capped
|
||||
@@ -747,6 +759,12 @@ export async function registerMaintenanceQueues(
|
||||
{ localConcurrency: 1 },
|
||||
handleRestoreDrill,
|
||||
);
|
||||
await createAndWork(
|
||||
boss,
|
||||
OFFHOST_PURGE_QUEUE,
|
||||
{ localConcurrency: 1 },
|
||||
handleOffhostPurge,
|
||||
);
|
||||
// One restore at a time per worker: each holds a transaction over a whole
|
||||
// account and a batch of readings in memory, and two at once on a small host
|
||||
// is how a restore that fits alone would run out of heap.
|
||||
|
||||
@@ -101,6 +101,7 @@ const subsystemSurface = {
|
||||
"step-up-elevation-cleanup": { audience: "system" },
|
||||
"data-backup-offhost": { audience: "system" },
|
||||
"data-restore-drill": { audience: "system" },
|
||||
"offhost-backup-purge": { audience: "system" },
|
||||
"backup-restore": { audience: "system" },
|
||||
"host-metric-sample": { audience: "system" },
|
||||
"feedback-aggregator": { audience: "system" },
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
kickOffhostPurge,
|
||||
requestOffhostPurge,
|
||||
} from "@/lib/jobs/offhost-purge";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
@@ -209,9 +213,11 @@ export async function deleteManagedProfile(input: {
|
||||
details: { profileId: profile.id },
|
||||
client: tx,
|
||||
});
|
||||
await requestOffhostPurge(tx, profile.id, "managed_profile_deleted");
|
||||
await tx.user.delete({ where: { id: profile.id } });
|
||||
});
|
||||
});
|
||||
await kickOffhostPurge();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -66,6 +66,8 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
|
||||
"auth.token.invalid",
|
||||
],
|
||||
backup: [
|
||||
"backup.foreign_reference",
|
||||
"backup.key.missing",
|
||||
"backup.payload.undecryptable",
|
||||
"backup.restore.active",
|
||||
"backup.section.missing",
|
||||
|
||||
@@ -58,6 +58,25 @@ export interface OffhostBackupOverview {
|
||||
periodHours: number;
|
||||
/** One row per account. Empty when off-host backup is not configured. */
|
||||
rows: OffhostAccountRow[];
|
||||
/**
|
||||
* Deleted or wiped accounts whose copies are still in the bucket. A count
|
||||
* above zero with an error means the bucket refused the deletion (most
|
||||
* often a credential without DeleteObject); the job retries nightly.
|
||||
*/
|
||||
pendingDeletions: {
|
||||
count: number;
|
||||
oldestRequestedAt: string | null;
|
||||
lastFailure: string | null;
|
||||
};
|
||||
/**
|
||||
* Whether the bucket retires old copies by itself. Read from the bucket
|
||||
* (`GetBucketLifecycleConfiguration`), cached for ten minutes; `unknown`
|
||||
* when the credential may not read it.
|
||||
*/
|
||||
lifecycle: {
|
||||
state: "configured" | "missing" | "unknown";
|
||||
expirationDays: number | null;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BackupsList {
|
||||
|
||||
Reference in new issue
Block a user