fix(db-backup): read the off-host leg of the dump on its own

A run that kept its dump on the volume and failed the upload is the newest
good dump and the newest failure at the same instant. Compared with ">", the
card showed no failure and readiness said fresh, night after night, while no
dump left the host.

The health read now carries the off-host leg separately: when the newest dump
reached the bucket, and the newest one that did not. One pure reading of it
(databaseDumpProblems) feeds both the backups card and the readiness model.
A configured bucket that missed the newest dump is attention (offhost_failed)
with the upload's error, and failing once the newest dump there is older than
36 hours (offhost_stale) or none ever arrived (offhost_never). The card shows
the refused upload and when the bucket last got a dump. A failure at the
instant of the newest dump for another reason, retention throwing, is now
shown too.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 16:53:24 +01:00
1 parent 4887ed8071
commit 664b46d2c6
18 files changed
+427 -37

No files matched your search

+44
View File
@@ -0,0 +1,44 @@
/**
* What is wrong with the nightly dump, read from its health once, for the
* backups card and the readiness model alike, so the two cannot disagree.
*
* Pure and free of server imports: the card is a client component.
*/
import type { DatabaseBackupHealth } from "./health";
export interface DatabaseDumpProblems {
/**
* The newest run failed at or after the newest kept dump, for a reason other
* than the upload: nothing was kept, or a later step (retention) failed.
*/
failure: { at: string; error: string } | null;
/**
* The bucket is configured and the newest dump taken did not reach it.
* `lastUploadAt` is the newest dump that did, or null when none ever did.
*/
offhost: { at: string; error: string; lastUploadAt: string | null } | null;
}
export function databaseDumpProblems(
health: Pick<
DatabaseBackupHealth,
"lastGood" | "lastFailure" | "offhostConfigured" | "offhost"
>,
): DatabaseDumpProblems {
const offhost =
health.offhostConfigured && health.offhost.failure
? {
...health.offhost.failure,
lastUploadAt: health.offhost.lastUploadAt,
}
: null;
const failure = health.lastFailure;
const good = health.lastGood;
// At or after: a run that kept its dump and then failed is the newest good
// dump and the newest failure, at the same instant.
const sinceGood =
failure !== null && (good === null || failure.at >= good.at);
// That same run failing only its upload is the off-host problem, said once.
const isTheUpload = offhost !== null && failure?.at === offhost.at;
return { failure: sinceGood && !isTheUpload ? failure : null, offhost };
}
+47 -6
View File
@@ -57,6 +57,20 @@ export interface DatabaseBackupHealth {
} | null;
/** The newest run that failed, or that a dying process left unfinished. */
lastFailure: { at: string; error: string } | null;
/**
* The off-host leg on its own. A run that kept its dump on the server and
* failed the upload is a good dump and a failed run at once; read only as
* those two, it looked healthy while nothing reached the bucket.
*/
offhost: {
/** When the newest dump reached the bucket; null when none ever did. */
lastUploadAt: string | null;
/**
* The newest dump that was taken but did not reach the bucket, when no
* later one did. Null while the newest dump taken is in the bucket.
*/
failure: { at: string; error: string } | null;
};
}
/**
@@ -80,8 +94,8 @@ export async function getDatabaseBackupHealth(
const keptWhere = local.persistent
? [{ localResult: "written" }, { offhostResult: "uploaded" }]
: [{ offhostResult: "uploaded" }];
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
[
const [lastGood, lastFailed, lastAbandoned, lastUploaded, lastNotUploaded] =
await Promise.all([
prisma.databaseBackupRun.findFirst({
where: { fileName: { not: null }, OR: keptWhere },
orderBy: { startedAt: "desc" },
@@ -107,9 +121,19 @@ export async function getDatabaseBackupHealth(
orderBy: { startedAt: "desc" },
select: { startedAt: true },
}),
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
],
);
prisma.databaseBackupRun.findFirst({
where: { offhostResult: "uploaded" },
orderBy: { startedAt: "desc" },
select: { startedAt: true, finishedAt: true },
}),
// A dump was taken (the file name is set only when the run finishes)
// and its upload failed.
prisma.databaseBackupRun.findFirst({
where: { fileName: { not: null }, offhostResult: "failed" },
orderBy: { startedAt: "desc" },
select: { startedAt: true, finishedAt: true, error: true },
}),
]);
const failures = [
lastFailed && {
@@ -131,7 +155,7 @@ export async function getDatabaseBackupHealth(
localDirWritable: local.writable,
localPersistent: local.persistent,
localReason: local.reason,
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
offhostConfigured: offhostBackupConfigured() || lastUploaded !== null,
lastGood: lastGood
? {
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
@@ -143,5 +167,22 @@ export async function getDatabaseBackupHealth(
}
: null,
lastFailure: failures[0] ?? null,
offhost: {
lastUploadAt: lastUploaded ? runAt(lastUploaded) : null,
failure:
lastNotUploaded &&
(lastUploaded === null ||
lastNotUploaded.startedAt > lastUploaded.startedAt)
? {
at: runAt(lastNotUploaded),
error: lastNotUploaded.error ?? "the upload failed",
}
: null,
},
};
}
/** When a run ended, or began when it never ended. */
function runAt(run: { startedAt: Date; finishedAt: Date | null }): string {
return (run.finishedAt ?? run.startedAt).toISOString();
}