fix(db-backup): read the off-host leg of the dump on its own

A run that kept its dump on the volume and failed the upload is the newest
good dump and the newest failure at the same instant. Compared with ">", the
card showed no failure and readiness said fresh, night after night, while no
dump left the host.

The health read now carries the off-host leg separately: when the newest dump
reached the bucket, and the newest one that did not. One pure reading of it
(databaseDumpProblems) feeds both the backups card and the readiness model.
A configured bucket that missed the newest dump is attention (offhost_failed)
with the upload's error, and failing once the newest dump there is older than
36 hours (offhost_stale) or none ever arrived (offhost_never). The card shows
the refused upload and when the bucket last got a dump. A failure at the
instant of the newest dump for another reason, retention throwing, is now
shown too.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 16:53:24 +01:00
1 parent 4887ed8071
commit 664b46d2c6
18 files changed
+427 -37

No files matched your search

+44
View File
@@ -0,0 +1,44 @@
/**
* What is wrong with the nightly dump, read from its health once, for the
* backups card and the readiness model alike, so the two cannot disagree.
*
* Pure and free of server imports: the card is a client component.
*/
import type { DatabaseBackupHealth } from "./health";
export interface DatabaseDumpProblems {
/**
* The newest run failed at or after the newest kept dump, for a reason other
* than the upload: nothing was kept, or a later step (retention) failed.
*/
failure: { at: string; error: string } | null;
/**
* The bucket is configured and the newest dump taken did not reach it.
* `lastUploadAt` is the newest dump that did, or null when none ever did.
*/
offhost: { at: string; error: string; lastUploadAt: string | null } | null;
}
export function databaseDumpProblems(
health: Pick<
DatabaseBackupHealth,
"lastGood" | "lastFailure" | "offhostConfigured" | "offhost"
>,
): DatabaseDumpProblems {
const offhost =
health.offhostConfigured && health.offhost.failure
? {
...health.offhost.failure,
lastUploadAt: health.offhost.lastUploadAt,
}
: null;
const failure = health.lastFailure;
const good = health.lastGood;
// At or after: a run that kept its dump and then failed is the newest good
// dump and the newest failure, at the same instant.
const sinceGood =
failure !== null && (good === null || failure.at >= good.at);
// That same run failing only its upload is the off-host problem, said once.
const isTheUpload = offhost !== null && failure?.at === offhost.at;
return { failure: sinceGood && !isTheUpload ? failure : null, offhost };
}
+47 -6
View File
@@ -57,6 +57,20 @@ export interface DatabaseBackupHealth {
} | null;
/** The newest run that failed, or that a dying process left unfinished. */
lastFailure: { at: string; error: string } | null;
/**
* The off-host leg on its own. A run that kept its dump on the server and
* failed the upload is a good dump and a failed run at once; read only as
* those two, it looked healthy while nothing reached the bucket.
*/
offhost: {
/** When the newest dump reached the bucket; null when none ever did. */
lastUploadAt: string | null;
/**
* The newest dump that was taken but did not reach the bucket, when no
* later one did. Null while the newest dump taken is in the bucket.
*/
failure: { at: string; error: string } | null;
};
}
/**
@@ -80,8 +94,8 @@ export async function getDatabaseBackupHealth(
const keptWhere = local.persistent
? [{ localResult: "written" }, { offhostResult: "uploaded" }]
: [{ offhostResult: "uploaded" }];
const [lastGood, lastFailed, lastAbandoned, everUploaded] = await Promise.all(
[
const [lastGood, lastFailed, lastAbandoned, lastUploaded, lastNotUploaded] =
await Promise.all([
prisma.databaseBackupRun.findFirst({
where: { fileName: { not: null }, OR: keptWhere },
orderBy: { startedAt: "desc" },
@@ -107,9 +121,19 @@ export async function getDatabaseBackupHealth(
orderBy: { startedAt: "desc" },
select: { startedAt: true },
}),
prisma.databaseBackupRun.count({ where: { offhostResult: "uploaded" } }),
],
);
prisma.databaseBackupRun.findFirst({
where: { offhostResult: "uploaded" },
orderBy: { startedAt: "desc" },
select: { startedAt: true, finishedAt: true },
}),
// A dump was taken (the file name is set only when the run finishes)
// and its upload failed.
prisma.databaseBackupRun.findFirst({
where: { fileName: { not: null }, offhostResult: "failed" },
orderBy: { startedAt: "desc" },
select: { startedAt: true, finishedAt: true, error: true },
}),
]);
const failures = [
lastFailed && {
@@ -131,7 +155,7 @@ export async function getDatabaseBackupHealth(
localDirWritable: local.writable,
localPersistent: local.persistent,
localReason: local.reason,
offhostConfigured: offhostBackupConfigured() || everUploaded > 0,
offhostConfigured: offhostBackupConfigured() || lastUploaded !== null,
lastGood: lastGood
? {
at: (lastGood.finishedAt ?? lastGood.startedAt).toISOString(),
@@ -143,5 +167,22 @@ export async function getDatabaseBackupHealth(
}
: null,
lastFailure: failures[0] ?? null,
offhost: {
lastUploadAt: lastUploaded ? runAt(lastUploaded) : null,
failure:
lastNotUploaded &&
(lastUploaded === null ||
lastNotUploaded.startedAt > lastUploaded.startedAt)
? {
at: runAt(lastNotUploaded),
error: lastNotUploaded.error ?? "the upload failed",
}
: null,
},
};
}
/** When a run ended, or began when it never ended. */
function runAt(run: { startedAt: Date; finishedAt: Date | null }): string {
return (run.finishedAt ?? run.startedAt).toISOString();
}
@@ -71,6 +71,7 @@ function dump(
error: lastFailureReason ?? "failed",
}
: null,
offhost: { lastUploadAt: null, failure: null },
...rest,
};
}
@@ -166,6 +167,89 @@ describe("database dump", () => {
});
});
// The run that kept its dump here and failed its upload is the newest good
// dump and the newest failure at the same instant; read as only those two,
// it looked fresh while nothing reached the bucket.
it("says when the bucket did not get the newest dump, and fails once its copy there is stale", () => {
const keptAt = ago(2 * HOUR);
const uploadFailed = (lastUploadAt: Date | null) =>
dump({
offhostConfigured: true,
lastGoodAt: keptAt,
lastFailureAt: keptAt,
lastFailureReason: "off-host upload failed: AccessDenied",
offhost: {
lastUploadAt: lastUploadAt?.toISOString() ?? null,
failure: {
at: keptAt.toISOString(),
error: "off-host upload failed: AccessDenied",
},
},
});
expect(evalDump(uploadFailed(ago(26 * HOUR)))).toMatchObject({
state: "attention",
reason: "offhost_failed",
detail: "off-host upload failed: AccessDenied",
});
expect(
evalDump(uploadFailed(ago(DUMP_FAILING_AFTER_HOURS * HOUR + 1))),
).toMatchObject({
state: "failing",
reason: "offhost_stale",
params: { hours: 36 },
});
expect(evalDump(uploadFailed(null))).toMatchObject({
state: "failing",
reason: "offhost_never",
});
// A bucket nobody configured is not missing a dump.
expect(
evalDump({ ...uploadFailed(null), offhostConfigured: false }),
).toMatchObject({ state: "attention", reason: "last_failed" });
});
it("names a failure after the newest dump at the same instant, such as retention throwing", () => {
const keptAt = ago(2 * HOUR);
expect(
evalDump(
dump({
lastGoodAt: keptAt,
lastFailureAt: keptAt,
lastFailureReason: "EACCES: unlink",
}),
),
).toMatchObject({
state: "attention",
reason: "last_failed",
detail: "EACCES: unlink",
});
});
it("calls an upload-only setup whose upload never worked never succeeded", () => {
const at = ago(HOUR);
expect(
evalDump(
dump({
localDirWritable: true,
localPersistent: false,
localReason: "container_layer",
offhostConfigured: true,
lastGoodAt: null,
lastFailureAt: at,
offhost: {
lastUploadAt: null,
failure: { at: at.toISOString(), error: "AccessDenied" },
},
}),
ago(2 * HOUR),
),
).toMatchObject({
state: "failing",
reason: "never_succeeded",
detail: "AccessDenied",
});
});
it("fails when a dump can be kept nowhere, and accepts a bucket-only setup", () => {
expect(evalDump(dump({ localDirWritable: false }))).toMatchObject({
state: "failing",
+48 -8
View File
@@ -30,6 +30,7 @@ import {
type OffhostBackupFreshness,
} from "@/lib/jobs/offhost-backup-freshness";
import type { LifecycleVerdict } from "@/lib/jobs/offhost-lifecycle";
import { databaseDumpProblems } from "@/lib/database-backup/dump-problems";
import type { DatabaseBackupHealth } from "@/lib/recovery/database-dump-health";
import type { KitEntry } from "@/lib/recovery/kit-rules";
@@ -202,8 +203,9 @@ export function evaluateDatabaseDump(
? signal(id, "inactive", "disabled", null)
: signal(id, "attention", "disabled", lastGoodAt);
}
const lastFailureAt = dump.lastFailure ? new Date(dump.lastFailure.at) : null;
const failureReason = dump.lastFailure?.error ?? null;
const problems = databaseDumpProblems(dump);
const lastFailureAt = problems.failure ? new Date(problems.failure.at) : null;
const failureReason = problems.failure?.error ?? null;
// A directory that keeps nothing (not writable, or writable inside the
// container and lost at the next update) is a valid setup when the dumps
// go to the bucket; with no bucket either, no dump is kept anywhere.
@@ -217,18 +219,19 @@ export function evaluateDatabaseDump(
);
}
const limitMs = DUMP_FAILING_AFTER_HOURS * HOUR_MS;
const failedSinceGood =
lastFailureAt !== null &&
(lastGoodAt === null || lastFailureAt > lastGoodAt);
const failedSinceGood = lastFailureAt !== null;
if (lastGoodAt === null) {
if (failedSinceGood) {
// Nothing kept anywhere yet; a dump that only failed its upload (no
// directory that survives) has not succeeded either.
const failed = problems.failure ?? problems.offhost;
if (failed) {
return signal(
id,
"failing",
"never_succeeded",
lastFailureAt,
new Date(failed.at),
{},
failureReason,
failed.error,
);
}
const watched = input.dumpWatchedSince ?? now;
@@ -249,6 +252,43 @@ export function evaluateDatabaseDump(
failedSinceGood ? failureReason : null,
);
}
// The dump is kept on this server, and the bucket configured for it did
// not get it: the copy that outlives the host is missing. Past the same
// limit as a stale dump, no copy off the host is recent enough to rebuild
// from.
if (problems.offhost) {
const uploadedAt = problems.offhost.lastUploadAt
? new Date(problems.offhost.lastUploadAt)
: null;
if (uploadedAt === null) {
return signal(
id,
"failing",
"offhost_never",
lastGoodAt,
{},
problems.offhost.error,
);
}
const uploadAgeMs = now.getTime() - uploadedAt.getTime();
return uploadAgeMs > limitMs
? signal(
id,
"failing",
"offhost_stale",
lastGoodAt,
{ hours: wholeHours(uploadAgeMs) },
problems.offhost.error,
)
: signal(
id,
"attention",
"offhost_failed",
lastGoodAt,
{},
problems.offhost.error,
);
}
if (failedSinceGood) {
return signal(
id,