mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 05:42:19 +00:00
A weekly or nightly backup job whose process died (a restart, a container out of memory) stayed active until its two- or four-hour expiry, and its retry walked the accounts in the same order: the account that killed it has the oldest copy, so it led again, and every account after it went without a copy for as long as that record stayed too large. Both passes now send with a five-minute pg-boss heartbeat, so a dead run fails within minutes and its retry picks it up. Each pass records when it started and finished every account (backup_pass_attempts, migration 0373); an account whose last attempt never finished goes behind all the others. The backups page shows when the run in progress started and names an account a run died under, for the weekly and the off-host card, and a heartbeat timeout reads as what it means.
236 lines
8.7 KiB
TypeScript
236 lines
8.7 KiB
TypeScript
/**
|
|
* GET /api/admin/backups — admin-only listing of `DataBackup` rows.
|
|
*
|
|
* Returns one row per (userId, type) pair. The `data` payload is NOT
|
|
* shipped by THIS route — only metadata (id, userId, username, type, size in
|
|
* bytes, createdAt), so the listing costs a count and not a decrypt. The
|
|
* payload has its own route: `/api/admin/backups/<id>/download` decrypts one
|
|
* snapshot for the admin who asks for it, and the ciphertext stays in the
|
|
* database either way.
|
|
*
|
|
* Alongside the rows, the response says whether the SCHEDULE is still alive:
|
|
* how old the newest scheduled copy is and how the last scheduled run ended.
|
|
* Without those two the page cannot tell a working backup from one that
|
|
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
|
* timestamp either way.
|
|
*
|
|
* And alongside both, the off-host leg: one row per account saying when this
|
|
* host last put that account's encrypted copy in the operator's bucket and how
|
|
* stale that is against the nightly schedule. It comes from the ledger the
|
|
* worker writes, never from a bucket listing — the answer has to hold on a host
|
|
* whose worker holds no listing grant on the bucket.
|
|
*/
|
|
import { prisma } from "@/lib/db";
|
|
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
|
import { apiSuccess } from "@/lib/api-response";
|
|
import { annotate } from "@/lib/logging/context";
|
|
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
|
import {
|
|
readLastQueueRun,
|
|
readQueueRunningSince,
|
|
} from "@/lib/jobs/job-failures";
|
|
import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts";
|
|
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
|
import {
|
|
OFFHOST_BACKUP_QUEUE,
|
|
offhostBackupConfigured,
|
|
probeOffhostLifecycle,
|
|
} from "@/lib/jobs/offhost-backup";
|
|
import {
|
|
classifyOffhostBackup,
|
|
OFFHOST_BACKUP_PERIOD_HOURS,
|
|
type OffhostBackupFreshness,
|
|
} from "@/lib/jobs/offhost-backup-freshness";
|
|
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
|
|
// so callers (in particular `components/admin/backups-section.tsx`) don't
|
|
// have to reach across the component → route-handler layer boundary.
|
|
import type {
|
|
BackupRow,
|
|
BackupsList,
|
|
OffhostAccountRow,
|
|
} from "@/types/backups";
|
|
|
|
export const dynamic = "force-dynamic";
|
|
|
|
/** The type the weekly worker writes; anything else was uploaded by hand. */
|
|
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
|
|
|
|
interface BackupMetaRow {
|
|
id: string;
|
|
user_id: string;
|
|
username: string;
|
|
type: string;
|
|
size_bytes: number;
|
|
created_at: Date;
|
|
}
|
|
|
|
export const GET = apiHandler(async () => {
|
|
await requireAdmin();
|
|
annotate({ action: { name: "admin.backups.list" } });
|
|
|
|
// Metadata only, and the size read as `octet_length` in the database rather
|
|
// than by pulling every blob into this process to measure it. The listing
|
|
// needs a number, not the ciphertext, and on an instance with large records
|
|
// fetching them all was its own way of running out of memory. A copy is
|
|
// either one value in `data` (written before v1.39.2) or a run of pieces in
|
|
// `data_backup_chunks`, so the size is whichever of the two it has: what
|
|
// the copy occupies in the database.
|
|
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
|
|
SELECT b.id, b.user_id, u.username, b.type,
|
|
COALESCE(octet_length(b.data), 0)
|
|
+ COALESCE(c.bytes, 0) AS size_bytes,
|
|
b.created_at
|
|
FROM data_backups b
|
|
JOIN users u ON u.id = b.user_id
|
|
LEFT JOIN (
|
|
SELECT backup_id, SUM(octet_length(data)) AS bytes
|
|
FROM data_backup_chunks
|
|
GROUP BY backup_id
|
|
) c ON c.backup_id = b.id
|
|
ORDER BY b.created_at DESC
|
|
`;
|
|
|
|
const list: BackupRow[] = rows.map((row) => ({
|
|
id: row.id,
|
|
userId: row.user_id,
|
|
username: row.username,
|
|
type: row.type,
|
|
sizeBytes: Number(row.size_bytes),
|
|
createdAt: row.created_at.toISOString(),
|
|
}));
|
|
|
|
// The off-host leg, per account, read from the ledger the nightly worker
|
|
// writes rather than from the bucket. The page must be able to answer "which
|
|
// account has no recent copy off-host" on a host whose worker holds the
|
|
// three grants the runbook documents and no listing grant at all, and a
|
|
// listing call from a page render would put the operator's credentials on
|
|
// the request path of every page view.
|
|
const now = new Date();
|
|
const offhostAccounts = await prisma.user.findMany({
|
|
select: {
|
|
id: true,
|
|
username: true,
|
|
offhostBackupState: {
|
|
select: {
|
|
lastAttemptAt: true,
|
|
lastSuccessAt: true,
|
|
sizeBytes: true,
|
|
},
|
|
},
|
|
},
|
|
orderBy: { username: "asc" },
|
|
});
|
|
|
|
// Evidence beats this process's environment. `offhostBackupConfigured()`
|
|
// reads the variables of whichever process serves this request, and a
|
|
// hand-rolled split that gives `BACKUP_S3_*` only to the worker would make
|
|
// the web process say "nothing leaves this host" over a ledger full of
|
|
// fresh rows. A row in that table is proof an upload happened; no
|
|
// environment read can outrank it.
|
|
const configured =
|
|
offhostBackupConfigured() ||
|
|
offhostAccounts.some((account) => account.offhostBackupState !== null);
|
|
|
|
const offhostRows: OffhostAccountRow[] = offhostAccounts.map((account) => {
|
|
const state = account.offhostBackupState;
|
|
const verdict = classifyOffhostBackup({
|
|
lastAttemptAt: state?.lastAttemptAt ?? null,
|
|
lastSuccessAt: state?.lastSuccessAt ?? null,
|
|
now,
|
|
});
|
|
return {
|
|
userId: account.id,
|
|
username: account.username,
|
|
lastAttemptAt: state?.lastAttemptAt.toISOString() ?? null,
|
|
lastSuccessAt: state?.lastSuccessAt?.toISOString() ?? null,
|
|
// BigInt column -> number on the wire. The value is bytes of one
|
|
// object, capped by the uploader at 80 GB, so it is nowhere near
|
|
// Number.MAX_SAFE_INTEGER and JSON has no BigInt.
|
|
sizeBytes:
|
|
state?.sizeBytes === null || state?.sizeBytes === undefined
|
|
? null
|
|
: Number(state.sizeBytes),
|
|
ageHours: verdict.ageHours,
|
|
freshness: verdict.freshness,
|
|
};
|
|
});
|
|
|
|
// Worst first. Sorting by username puts the one account that needs
|
|
// attention wherever the alphabet happens to leave it, which on a
|
|
// hundred-account cohort is the same "one bad row hides in the crowd"
|
|
// problem this card was built to end, moved out of a count and into a list.
|
|
const FRESHNESS_ORDER: Record<OffhostBackupFreshness, number> = {
|
|
stale: 0,
|
|
due: 1,
|
|
never: 2,
|
|
unknown: 3,
|
|
fresh: 4,
|
|
};
|
|
offhostRows.sort(
|
|
(a, b) =>
|
|
FRESHNESS_ORDER[a.freshness] - FRESHNESS_ORDER[b.freshness] ||
|
|
a.username.localeCompare(b.username),
|
|
);
|
|
|
|
// A wipe whose copies are already gone waits for the next nightly run
|
|
// before its request is removed (see `offhost-purge.ts`); nothing of it is
|
|
// left in the bucket, so it is not counted as a pending deletion.
|
|
const stillInBucket = {
|
|
NOT: {
|
|
reason: "data_wiped",
|
|
lastFailure: null,
|
|
lastAttemptAt: { not: null },
|
|
},
|
|
};
|
|
const [weeklyRunningSince, offhostRunningSince] = await Promise.all([
|
|
readQueueRunningSince(DATA_BACKUP_QUEUE),
|
|
readQueueRunningSince(OFFHOST_BACKUP_QUEUE),
|
|
]);
|
|
const [scheduleActivity, offhostActivity] = await Promise.all([
|
|
readBackupPassActivity(prisma, DATA_BACKUP_QUEUE, weeklyRunningSince),
|
|
readBackupPassActivity(prisma, OFFHOST_BACKUP_QUEUE, offhostRunningSince),
|
|
]);
|
|
const [purgeCount, oldestPurge, lifecycle] = await Promise.all([
|
|
prisma.offhostPurgeRequest.count({ where: stillInBucket }),
|
|
prisma.offhostPurgeRequest.findFirst({
|
|
where: stillInBucket,
|
|
orderBy: { requestedAt: "asc" },
|
|
select: { requestedAt: true, lastFailure: true },
|
|
}),
|
|
configured
|
|
? probeOffhostLifecycle()
|
|
: Promise.resolve({ state: "unknown" as const, expirationDays: null }),
|
|
]);
|
|
|
|
const payload: BackupsList = {
|
|
rows: list,
|
|
// Matches the retention window the backup-prune job enforces so the
|
|
// backups page states the same number the worker acts on.
|
|
retentionDays: 90,
|
|
schedule: summariseBackupSchedule({
|
|
scheduledCreatedAt: rows
|
|
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
|
|
.map((row) => row.created_at),
|
|
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
|
|
now,
|
|
}),
|
|
scheduleActivity,
|
|
offhost: {
|
|
configured,
|
|
activity: offhostActivity,
|
|
periodHours: OFFHOST_BACKUP_PERIOD_HOURS,
|
|
// An unconfigured host has nothing to list, and the card says so rather
|
|
// than painting an empty table that reads as "no problems".
|
|
rows: configured ? offhostRows : [],
|
|
pendingDeletions: {
|
|
count: purgeCount,
|
|
oldestRequestedAt: oldestPurge?.requestedAt.toISOString() ?? null,
|
|
lastFailure: oldestPurge?.lastFailure ?? null,
|
|
},
|
|
lifecycle,
|
|
},
|
|
};
|
|
|
|
return apiSuccess(payload);
|
|
});
|