mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 15:42:24 +00:00
fix(db-backup): read the off-host leg of the dump on its own
A run that kept its dump on the volume and failed the upload is the newest good dump and the newest failure at the same instant. Compared with ">", the card showed no failure and readiness said fresh, night after night, while no dump left the host. The health read now carries the off-host leg separately: when the newest dump reached the bucket, and the newest one that did not. One pure reading of it (databaseDumpProblems) feeds both the backups card and the readiness model. A configured bucket that missed the newest dump is attention (offhost_failed) with the upload's error, and failing once the newest dump there is older than 36 hours (offhost_stale) or none ever arrived (offhost_never). The card shows the refused upload and when the bucket last got a dump. A failure at the instant of the newest dump for another reason, retention throwing, is now shown too.
This commit is contained in:
1 parent
4887ed8071
commit
664b46d2c6
18 files changed
+427
-37
No files matched your search
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* What is wrong with the nightly dump, read from its health once, for the
|
||||
* backups card and the readiness model alike, so the two cannot disagree.
|
||||
*
|
||||
* Pure and free of server imports: the card is a client component.
|
||||
*/
|
||||
import type { DatabaseBackupHealth } from "./health";
|
||||
|
||||
export interface DatabaseDumpProblems {
|
||||
/**
|
||||
* The newest run failed at or after the newest kept dump, for a reason other
|
||||
* than the upload: nothing was kept, or a later step (retention) failed.
|
||||
*/
|
||||
failure: { at: string; error: string } | null;
|
||||
/**
|
||||
* The bucket is configured and the newest dump taken did not reach it.
|
||||
* `lastUploadAt` is the newest dump that did, or null when none ever did.
|
||||
*/
|
||||
offhost: { at: string; error: string; lastUploadAt: string | null } | null;
|
||||
}
|
||||
|
||||
export function databaseDumpProblems(
|
||||
health: Pick<
|
||||
DatabaseBackupHealth,
|
||||
"lastGood" | "lastFailure" | "offhostConfigured" | "offhost"
|
||||
>,
|
||||
): DatabaseDumpProblems {
|
||||
const offhost =
|
||||
health.offhostConfigured && health.offhost.failure
|
||||
? {
|
||||
...health.offhost.failure,
|
||||
lastUploadAt: health.offhost.lastUploadAt,
|
||||
}
|
||||
: null;
|
||||
const failure = health.lastFailure;
|
||||
const good = health.lastGood;
|
||||
// At or after: a run that kept its dump and then failed is the newest good
|
||||
// dump and the newest failure, at the same instant.
|
||||
const sinceGood =
|
||||
failure !== null && (good === null || failure.at >= good.at);
|
||||
// That same run failing only its upload is the off-host problem, said once.
|
||||
const isTheUpload = offhost !== null && failure?.at === offhost.at;
|
||||
return { failure: sinceGood && !isTheUpload ? failure : null, offhost };
|
||||
}
|
||||
@@ -57,6 +57,20 @@ export interface DatabaseBackupHealth {
|
||||
} | null;
|
||||
/** The newest run that failed, or that a dying process left unfinished. */
|
||||
lastFailure: { at: string; error: string } | null;
|
||||
/**
|
||||
* The off-host leg on its own. A run that kept its dump on the server and
|
||||
* failed the upload is a good dump and a failed run at once; read only as
|
||||
* those two, it looked healthy while nothing reached the bucket.
|
||||
*/
|
||||
offhost: {
|
||||
/** When the newest dump reached the bucket; null when none ever did. */
|
||||
lastUploadAt: string | null;
|
||||
/**
|
||||
* The newest dump that was taken but did not reach the bucket, when no
|
||||
* later one did. Null while the newest dump taken is in the bucket.
|
||||
*/
|
||||
failure: { at: string; error: string } | null;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -80,8 +94,8 @@ export async function getDatabaseBackupHealth(
|
||||
const keptWhere = local.persistent
|
||||
? [{ localResult: "written" }, { offhostResult: "uploaded" }]
|
||||
: [{ offhostResult: "uploaded" }];
|
||||
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
|
||||
[
|
||||
const [lastGood, lastFailed, lastAbandoned, lastUploaded, lastNotUploaded] =
|
||||
await Promise.all([
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { fileName: { not: null }, OR: keptWhere },
|
||||
orderBy: { startedAt: "desc" },
|
||||
@@ -107,9 +121,19 @@ export async function getDatabaseBackupHealth(
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
|
||||
],
|
||||
);
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { offhostResult: "uploaded" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true },
|
||||
}),
|
||||
// A dump was taken (the file name is set only when the run finishes)
|
||||
// and its upload failed.
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { fileName: { not: null }, offhostResult: "failed" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true, error: true },
|
||||
}),
|
||||
]);
|
||||
|
||||
const failures = [
|
||||
lastFailed && {
|
||||
@@ -131,7 +155,7 @@ export async function getDatabaseBackupHealth(
|
||||
localDirWritable: local.writable,
|
||||
localPersistent: local.persistent,
|
||||
localReason: local.reason,
|
||||
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
|
||||
offhostConfigured: offhostBackupConfigured() || lastUploaded !== null,
|
||||
lastGood: lastGood
|
||||
? {
|
||||
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
|
||||
@@ -143,5 +167,22 @@ export async function getDatabaseBackupHealth(
|
||||
}
|
||||
: null,
|
||||
lastFailure: failures[0] ?? null,
|
||||
offhost: {
|
||||
lastUploadAt: lastUploaded ? runAt(lastUploaded) : null,
|
||||
failure:
|
||||
lastNotUploaded &&
|
||||
(lastUploaded === null ||
|
||||
lastNotUploaded.startedAt > lastUploaded.startedAt)
|
||||
? {
|
||||
at: runAt(lastNotUploaded),
|
||||
error: lastNotUploaded.error ?? "the upload failed",
|
||||
}
|
||||
: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** When a run ended, or began when it never ended. */
|
||||
function runAt(run: { startedAt: Date; finishedAt: Date | null }): string {
|
||||
return (run.finishedAt ?? run.startedAt).toISOString();
|
||||
}
|
||||
Reference in new issue
Block a user