Files
HealthLog/src/app/api/admin/backups/route.ts
T
Marc-André Bombeck f18e6ed34b fix(backup): notice a backup run the process died under, and take its account last
A weekly or nightly backup job whose process died (a restart, a
container out of memory) stayed active until its two- or four-hour
expiry, and its retry walked the accounts in the same order: the account
that killed it has the oldest copy, so it led again, and every account
after it went without a copy for as long as that record stayed too
large.

Both passes now send with a five-minute pg-boss heartbeat, so a dead run
fails within minutes and its retry picks it up. Each pass records when it
started and finished every account (backup_pass_attempts, migration
0373); an account whose last attempt never finished goes behind all the
others. The backups page shows when the run in progress started and names
an account a run died under, for the weekly and the off-host card, and a
heartbeat timeout reads as what it means.
2026-10-04 15:56:56 +02:00

236 lines
8.7 KiB
TypeScript

/**
* GET /api/admin/backups — admin-only listing of `DataBackup` rows.
*
* Returns one row per (userId, type) pair. The `data` payload is NOT
* shipped by THIS route — only metadata (id, userId, username, type, size in
* bytes, createdAt), so the listing costs a count and not a decrypt. The
* payload has its own route: `/api/admin/backups/<id>/download` decrypts one
* snapshot for the admin who asks for it, and the ciphertext stays in the
* database either way.
*
* Alongside the rows, the response says whether the SCHEDULE is still alive:
* how old the newest scheduled copy is and how the last scheduled run ended.
* Without those two the page cannot tell a working backup from one that
* stopped six weeks ago — every row it lists has a perfectly ordinary
* timestamp either way.
*
* And alongside both, the off-host leg: one row per account saying when this
* host last put that account's encrypted copy in the operator's bucket and how
* stale that is against the nightly schedule. It comes from the ledger the
* worker writes, never from a bucket listing — the answer has to hold on a host
* whose worker holds no listing grant on the bucket.
*/
import { prisma } from "@/lib/db";
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
import {
readLastQueueRun,
readQueueRunningSince,
} from "@/lib/jobs/job-failures";
import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
import {
OFFHOST_BACKUP_QUEUE,
offhostBackupConfigured,
probeOffhostLifecycle,
} from "@/lib/jobs/offhost-backup";
import {
classifyOffhostBackup,
OFFHOST_BACKUP_PERIOD_HOURS,
type OffhostBackupFreshness,
} from "@/lib/jobs/offhost-backup-freshness";
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
// so callers (in particular `components/admin/backups-section.tsx`) don't
// have to reach across the component → route-handler layer boundary.
import type {
BackupRow,
BackupsList,
OffhostAccountRow,
} from "@/types/backups";
export const dynamic = "force-dynamic";
/** The type the weekly worker writes; anything else was uploaded by hand. */
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
interface BackupMetaRow {
id: string;
user_id: string;
username: string;
type: string;
size_bytes: number;
created_at: Date;
}
export const GET = apiHandler(async () => {
await requireAdmin();
annotate({ action: { name: "admin.backups.list" } });
// Metadata only, and the size read as `octet_length` in the database rather
// than by pulling every blob into this process to measure it. The listing
// needs a number, not the ciphertext, and on an instance with large records
// fetching them all was its own way of running out of memory. A copy is
// either one value in `data` (written before v1.39.2) or a run of pieces in
// `data_backup_chunks`, so the size is whichever of the two it has: what
// the copy occupies in the database.
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
SELECT b.id, b.user_id, u.username, b.type,
COALESCE(octet_length(b.data), 0)
+ COALESCE(c.bytes, 0) AS size_bytes,
b.created_at
FROM data_backups b
JOIN users u ON u.id = b.user_id
LEFT JOIN (
SELECT backup_id, SUM(octet_length(data)) AS bytes
FROM data_backup_chunks
GROUP BY backup_id
) c ON c.backup_id = b.id
ORDER BY b.created_at DESC
`;
const list: BackupRow[] = rows.map((row) => ({
id: row.id,
userId: row.user_id,
username: row.username,
type: row.type,
sizeBytes: Number(row.size_bytes),
createdAt: row.created_at.toISOString(),
}));
// The off-host leg, per account, read from the ledger the nightly worker
// writes rather than from the bucket. The page must be able to answer "which
// account has no recent copy off-host" on a host whose worker holds the
// three grants the runbook documents and no listing grant at all, and a
// listing call from a page render would put the operator's credentials on
// the request path of every page view.
const now = new Date();
const offhostAccounts = await prisma.user.findMany({
select: {
id: true,
username: true,
offhostBackupState: {
select: {
lastAttemptAt: true,
lastSuccessAt: true,
sizeBytes: true,
},
},
},
orderBy: { username: "asc" },
});
// Evidence beats this process's environment. `offhostBackupConfigured()`
// reads the variables of whichever process serves this request, and a
// hand-rolled split that gives `BACKUP_S3_*` only to the worker would make
// the web process say "nothing leaves this host" over a ledger full of
// fresh rows. A row in that table is proof an upload happened; no
// environment read can outrank it.
const configured =
offhostBackupConfigured() ||
offhostAccounts.some((account) => account.offhostBackupState !== null);
const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => {
const state = account.offhostBackupState;
const verdict = classifyOffhostBackup({
lastAttemptAt: state?.lastAttemptAt ?? null,
lastSuccessAt: state?.lastSuccessAt ?? null,
now,
});
return {
userId: account.id,
username: account.username,
lastAttemptAt: state?.lastAttemptAt.toISOString() ?? null,
lastSuccessAt: state?.lastSuccessAt?.toISOString() ?? null,
// BigInt column -> number on the wire. The value is bytes of one
// object, capped by the uploader at 80 GB, so it is nowhere near
// Number.MAX_SAFE_INTEGER and JSON has no BigInt.
sizeBytes:
state?.sizeBytes === null || state?.sizeBytes === undefined
? null
: Number(state.sizeBytes),
ageHours: verdict.ageHours,
freshness: verdict.freshness,
};
});
// Worst first. Sorting by username puts the one account that needs
// attention wherever the alphabet happens to leave it, which on a
// hundred-account cohort is the same "one bad row hides in the crowd"
// problem this card was built to end, moved out of a count and into a list.
const FRESHNESS_ORDER: Record<OffhostBackupFreshness, number> = {
stale: 0,
due: 1,
never: 2,
unknown: 3,
fresh: 4,
};
offhostRows.sort(
(a, b) =>
FRESHNESS_ORDER[a.freshness] - FRESHNESS_ORDER[b.freshness] ||
a.username.localeCompare(b.username),
);
// A wipe whose copies are already gone waits for the next nightly run
// before its request is removed (see `offhost-purge.ts`); nothing of it is
// left in the bucket, so it is not counted as a pending deletion.
const stillInBucket = {
NOT: {
reason: "data_wiped",
lastFailure: null,
lastAttemptAt: { not: null },
},
};
const [weeklyRunningSince, offhostRunningSince] = await Promise.all([
readQueueRunningSince(DATA_BACKUP_QUEUE),
readQueueRunningSince(OFFHOST_BACKUP_QUEUE),
]);
const [scheduleActivity, offhostActivity] = await Promise.all([
readBackupPassActivity(prisma, DATA_BACKUP_QUEUE, weeklyRunningSince),
readBackupPassActivity(prisma, OFFHOST_BACKUP_QUEUE, offhostRunningSince),
]);
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
prisma.offhostPurgeRequest.count({ where: stillInBucket }),
prisma.offhostPurgeRequest.findFirst({
where: stillInBucket,
orderBy: { requestedAt: "asc" },
select: { requestedAt: true, lastFailure: true },
}),
configured
? probeOffhostLifecycle()
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
]);
const payload: BackupsList = {
rows: list,
// Matches the retention window the backup-prune job enforces so the
// backups page states the same number the worker acts on.
retentionDays: 90,
schedule: summariseBackupSchedule({
scheduledCreatedAt: rows
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
.map((row) => row.created_at),
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
now,
}),
scheduleActivity,
offhost: {
configured,
activity: offhostActivity,
periodHours: OFFHOST_BACKUP_PERIOD_HOURS,
// An unconfigured host has nothing to list, and the card says so rather
// than painting an empty table that reads as "no problems".
rows: configured ? offhostRows : [],
pendingDeletions: {
count: purgeCount,
oldestRequestedAt: oldestPurge?.requestedAt.toISOString() ?? null,
lastFailure: oldestPurge?.lastFailure ?? null,
},
lifecycle,
},
};
return apiSuccess(payload);
});