mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 23:52:24 +00:00
fix(db-backup): read the off-host leg of the dump on its own
A run that kept its dump on the volume and failed the upload is the newest good dump and the newest failure at the same instant. Compared with ">", the card showed no failure and readiness said fresh, night after night, while no dump left the host. The health read now carries the off-host leg separately: when the newest dump reached the bucket, and the newest one that did not. One pure reading of it (databaseDumpProblems) feeds both the backups card and the readiness model. A configured bucket that missed the newest dump is attention (offhost_failed) with the upload's error, and failing once the newest dump there is older than 36 hours (offhost_stale) or none ever arrived (offhost_never). The card shows the refused upload and when the bucket last got a dump. A failure at the instant of the newest dump for another reason, retention throwing, is now shown too.
This commit is contained in:
1 parent
4887ed8071
commit
664b46d2c6
18 files changed
+427
-37
No files matched your search
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* What is wrong with the nightly dump, read from its health once, for the
|
||||
* backups card and the readiness model alike, so the two cannot disagree.
|
||||
*
|
||||
* Pure and free of server imports: the card is a client component.
|
||||
*/
|
||||
import type { DatabaseBackupHealth } from "./health";
|
||||
|
||||
export interface DatabaseDumpProblems {
|
||||
/**
|
||||
* The newest run failed at or after the newest kept dump, for a reason other
|
||||
* than the upload: nothing was kept, or a later step (retention) failed.
|
||||
*/
|
||||
failure: { at: string; error: string } | null;
|
||||
/**
|
||||
* The bucket is configured and the newest dump taken did not reach it.
|
||||
* `lastUploadAt` is the newest dump that did, or null when none ever did.
|
||||
*/
|
||||
offhost: { at: string; error: string; lastUploadAt: string | null } | null;
|
||||
}
|
||||
|
||||
export function databaseDumpProblems(
|
||||
health: Pick<
|
||||
DatabaseBackupHealth,
|
||||
"lastGood" | "lastFailure" | "offhostConfigured" | "offhost"
|
||||
>,
|
||||
): DatabaseDumpProblems {
|
||||
const offhost =
|
||||
health.offhostConfigured && health.offhost.failure
|
||||
? {
|
||||
...health.offhost.failure,
|
||||
lastUploadAt: health.offhost.lastUploadAt,
|
||||
}
|
||||
: null;
|
||||
const failure = health.lastFailure;
|
||||
const good = health.lastGood;
|
||||
// At or after: a run that kept its dump and then failed is the newest good
|
||||
// dump and the newest failure, at the same instant.
|
||||
const sinceGood =
|
||||
failure !== null && (good === null || failure.at >= good.at);
|
||||
// That same run failing only its upload is the off-host problem, said once.
|
||||
const isTheUpload = offhost !== null && failure?.at === offhost.at;
|
||||
return { failure: sinceGood && !isTheUpload ? failure : null, offhost };
|
||||
}
|
||||
@@ -57,6 +57,20 @@ export interface DatabaseBackupHealth {
|
||||
} | null;
|
||||
/** The newest run that failed, or that a dying process left unfinished. */
|
||||
lastFailure: { at: string; error: string } | null;
|
||||
/**
|
||||
* The off-host leg on its own. A run that kept its dump on the server and
|
||||
* failed the upload is a good dump and a failed run at once; read only as
|
||||
* those two, it looked healthy while nothing reached the bucket.
|
||||
*/
|
||||
offhost: {
|
||||
/** When the newest dump reached the bucket; null when none ever did. */
|
||||
lastUploadAt: string | null;
|
||||
/**
|
||||
* The newest dump that was taken but did not reach the bucket, when no
|
||||
* later one did. Null while the newest dump taken is in the bucket.
|
||||
*/
|
||||
failure: { at: string; error: string } | null;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -80,8 +94,8 @@ export async function getDatabaseBackupHealth(
|
||||
const keptWhere = local.persistent
|
||||
? [{ localResult: "written" }, { offhostResult: "uploaded" }]
|
||||
: [{ offhostResult: "uploaded" }];
|
||||
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
|
||||
[
|
||||
const [lastGood, lastFailed, lastAbandoned, lastUploaded, lastNotUploaded] =
|
||||
await Promise.all([
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { fileName: { not: null }, OR: keptWhere },
|
||||
orderBy: { startedAt: "desc" },
|
||||
@@ -107,9 +121,19 @@ export async function getDatabaseBackupHealth(
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true },
|
||||
}),
|
||||
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
|
||||
],
|
||||
);
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { offhostResult: "uploaded" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true },
|
||||
}),
|
||||
// A dump was taken (the file name is set only when the run finishes)
|
||||
// and its upload failed.
|
||||
prisma.databaseBackupRun.findFirst({
|
||||
where: { fileName: { not: null }, offhostResult: "failed" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
select: { startedAt: true, finishedAt: true, error: true },
|
||||
}),
|
||||
]);
|
||||
|
||||
const failures = [
|
||||
lastFailed && {
|
||||
@@ -131,7 +155,7 @@ export async function getDatabaseBackupHealth(
|
||||
localDirWritable: local.writable,
|
||||
localPersistent: local.persistent,
|
||||
localReason: local.reason,
|
||||
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
|
||||
offhostConfigured: offhostBackupConfigured() || lastUploaded !== null,
|
||||
lastGood: lastGood
|
||||
? {
|
||||
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
|
||||
@@ -143,5 +167,22 @@ export async function getDatabaseBackupHealth(
|
||||
}
|
||||
: null,
|
||||
lastFailure: failures[0] ?? null,
|
||||
offhost: {
|
||||
lastUploadAt: lastUploaded ? runAt(lastUploaded) : null,
|
||||
failure:
|
||||
lastNotUploaded &&
|
||||
(lastUploaded === null ||
|
||||
lastNotUploaded.startedAt > lastUploaded.startedAt)
|
||||
? {
|
||||
at: runAt(lastNotUploaded),
|
||||
error: lastNotUploaded.error ?? "the upload failed",
|
||||
}
|
||||
: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** When a run ended, or began when it never ended. */
|
||||
function runAt(run: { startedAt: Date; finishedAt: Date | null }): string {
|
||||
return (run.finishedAt ?? run.startedAt).toISOString();
|
||||
}
|
||||
@@ -71,6 +71,7 @@ function dump(
|
||||
error: lastFailureReason ?? "failed",
|
||||
}
|
||||
: null,
|
||||
offhost: { lastUploadAt: null, failure: null },
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
@@ -166,6 +167,89 @@ describe("database dump", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// The run that kept its dump here and failed its upload is the newest good
|
||||
// dump and the newest failure at the same instant; read as only those two,
|
||||
// it looked fresh while nothing reached the bucket.
|
||||
it("says when the bucket did not get the newest dump, and fails once its copy there is stale", () => {
|
||||
const keptAt = ago(2 * HOUR);
|
||||
const uploadFailed = (lastUploadAt: Date | null) =>
|
||||
dump({
|
||||
offhostConfigured: true,
|
||||
lastGoodAt: keptAt,
|
||||
lastFailureAt: keptAt,
|
||||
lastFailureReason: "off-host upload failed: AccessDenied",
|
||||
offhost: {
|
||||
lastUploadAt: lastUploadAt?.toISOString() ?? null,
|
||||
failure: {
|
||||
at: keptAt.toISOString(),
|
||||
error: "off-host upload failed: AccessDenied",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(evalDump(uploadFailed(ago(26 * HOUR)))).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "offhost_failed",
|
||||
detail: "off-host upload failed: AccessDenied",
|
||||
});
|
||||
expect(
|
||||
evalDump(uploadFailed(ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1))),
|
||||
).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "offhost_stale",
|
||||
params: { hours: 36 },
|
||||
});
|
||||
expect(evalDump(uploadFailed(null))).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "offhost_never",
|
||||
});
|
||||
// A bucket nobody configured is not missing a dump.
|
||||
expect(
|
||||
evalDump({ ...uploadFailed(null), offhostConfigured: false }),
|
||||
).toMatchObject({ state: "attention", reason: "last_failed" });
|
||||
});
|
||||
|
||||
it("names a failure after the newest dump at the same instant, such as retention throwing", () => {
|
||||
const keptAt = ago(2 * HOUR);
|
||||
expect(
|
||||
evalDump(
|
||||
dump({
|
||||
lastGoodAt: keptAt,
|
||||
lastFailureAt: keptAt,
|
||||
lastFailureReason: "EACCES: unlink",
|
||||
}),
|
||||
),
|
||||
).toMatchObject({
|
||||
state: "attention",
|
||||
reason: "last_failed",
|
||||
detail: "EACCES: unlink",
|
||||
});
|
||||
});
|
||||
|
||||
it("calls an upload-only setup whose upload never worked never succeeded", () => {
|
||||
const at = ago(HOUR);
|
||||
expect(
|
||||
evalDump(
|
||||
dump({
|
||||
localDirWritable: true,
|
||||
localPersistent: false,
|
||||
localReason: "container_layer",
|
||||
offhostConfigured: true,
|
||||
lastGoodAt: null,
|
||||
lastFailureAt: at,
|
||||
offhost: {
|
||||
lastUploadAt: null,
|
||||
failure: { at: at.toISOString(), error: "AccessDenied" },
|
||||
},
|
||||
}),
|
||||
ago(2 * HOUR),
|
||||
),
|
||||
).toMatchObject({
|
||||
state: "failing",
|
||||
reason: "never_succeeded",
|
||||
detail: "AccessDenied",
|
||||
});
|
||||
});
|
||||
|
||||
it("fails when a dump can be kept nowhere, and accepts a bucket-only setup", () => {
|
||||
expect(evalDump(dump({ localDirWritable: false }))).toMatchObject({
|
||||
state: "failing",
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
type OffhostBackupFreshness,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
|
||||
import { databaseDumpProblems } from "@/lib/database-backup/dump-problems";
|
||||
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
|
||||
import type { KitEntry } from "@/lib/recovery/kit-rules";
|
||||
|
||||
@@ -202,8 +203,9 @@ export function evaluateDatabaseDump(
|
||||
? signal(id, "inactive", "disabled", null)
|
||||
: signal(id, "attention", "disabled", lastGoodAt);
|
||||
}
|
||||
const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null;
|
||||
const failureReason = dump.lastFailure?.error ?? null;
|
||||
const problems = databaseDumpProblems(dump);
|
||||
const lastFailureAt = problems.failure ? new Date(problems.failure.at) : null;
|
||||
const failureReason = problems.failure?.error ?? null;
|
||||
// A directory that keeps nothing (not writable, or writable inside the
|
||||
// container and lost at the next update) is a valid setup when the dumps
|
||||
// go to the bucket; with no bucket either, no dump is kept anywhere.
|
||||
@@ -217,18 +219,19 @@ export function evaluateDatabaseDump(
|
||||
);
|
||||
}
|
||||
const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS;
|
||||
const failedSinceGood =
|
||||
lastFailureAt !== null &&
|
||||
(lastGoodAt === null || lastFailureAt > lastGoodAt);
|
||||
const failedSinceGood = lastFailureAt !== null;
|
||||
if (lastGoodAt === null) {
|
||||
if (failedSinceGood) {
|
||||
// Nothing kept anywhere yet; a dump that only failed its upload (no
|
||||
// directory that survives) has not succeeded either.
|
||||
const failed = problems.failure ?? problems.offhost;
|
||||
if (failed) {
|
||||
return signal(
|
||||
id,
|
||||
"failing",
|
||||
"never_succeeded",
|
||||
lastFailureAt,
|
||||
new Date(failed.at),
|
||||
{},
|
||||
failureReason,
|
||||
failed.error,
|
||||
);
|
||||
}
|
||||
const watched = input.dumpWatchedSince ?? now;
|
||||
@@ -249,6 +252,43 @@ export function evaluateDatabaseDump(
|
||||
failedSinceGood ? failureReason : null,
|
||||
);
|
||||
}
|
||||
// The dump is kept on this server, and the bucket configured for it did
|
||||
// not get it: the copy that outlives the host is missing. Past the same
|
||||
// limit as a stale dump, no copy off the host is recent enough to rebuild
|
||||
// from.
|
||||
if (problems.offhost) {
|
||||
const uploadedAt = problems.offhost.lastUploadAt
|
||||
? new Date(problems.offhost.lastUploadAt)
|
||||
: null;
|
||||
if (uploadedAt === null) {
|
||||
return signal(
|
||||
id,
|
||||
"failing",
|
||||
"offhost_never",
|
||||
lastGoodAt,
|
||||
{},
|
||||
problems.offhost.error,
|
||||
);
|
||||
}
|
||||
const uploadAgeMs = now.getTime() - uploadedAt.getTime();
|
||||
return uploadAgeMs > limitMs
|
||||
? signal(
|
||||
id,
|
||||
"failing",
|
||||
"offhost_stale",
|
||||
lastGoodAt,
|
||||
{ hours: wholeHours(uploadAgeMs) },
|
||||
problems.offhost.error,
|
||||
)
|
||||
: signal(
|
||||
id,
|
||||
"attention",
|
||||
"offhost_failed",
|
||||
lastGoodAt,
|
||||
{},
|
||||
problems.offhost.error,
|
||||
);
|
||||
}
|
||||
if (failedSinceGood) {
|
||||
return signal(
|
||||
id,
|
||||
|
||||
Reference in new issue
Block a user