mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 00:12:18 +00:00
feat(admin): say per account how fresh the off-host copy is
The nightly off-host job reports counts — uploaded, failed, total — and a count cannot name the account it is failing for. A cohort where one object has been refused every night for a month reads as "99 uploaded, 1 failed" each time, which looks like weather rather than like an account with no disaster-recovery copy at all. The worker now writes a ledger row when an object lands (account, instant, size), and Admin → Backups gains a second card that rates every account against the nightly schedule: fresh inside one run, due when one run produced nothing, stale when two did not, never when no object has ever landed. On a host without BACKUP_S3_* the card says off-host backup is not configured and stops there. The verdicts are computed from the ledger, never from a bucket listing: the worker's grant is deliberately PutObject + GetObject, and an admin page must not put the operator's credentials on its render path to state a number the database already holds.
This commit is contained in:
1 parent
348d9e0739
commit
1c10bbbde8
23 files changed
+797
-10
No files matched your search
@@ -210,6 +210,11 @@ export const WIPE_MODELS = [
|
||||
// just returned.
|
||||
"IdempotencyKey",
|
||||
"DataBackup",
|
||||
// What this host last put in the operator's bucket for this account. The
|
||||
// objects themselves stay — the worker holds no DeleteObject grant and the
|
||||
// bucket's lifecycle rule owns their retirement — but the row saying when
|
||||
// they were written is about this account and goes with it.
|
||||
"OffhostBackupState",
|
||||
"AuditLog",
|
||||
"ConsentReceipt",
|
||||
"UserKnownDevice",
|
||||
|
||||
@@ -642,6 +642,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
|
||||
"Replay protection for requests this host already answered. Restoring it would suppress a legitimate new request.",
|
||||
DataBackup:
|
||||
"The backup catalogue itself. A backup that contains the list of backups is a recursion with no reader.",
|
||||
OffhostBackupState:
|
||||
"When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.",
|
||||
ImportJob:
|
||||
"A job record pointing at an uploaded file that the backup does not carry, so restoring it would resurrect a task with nothing to work on.",
|
||||
InviteToken:
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyOffhostBackup,
|
||||
OFFHOST_BACKUP_PERIOD_HOURS,
|
||||
} from "@/lib/jobs/offhost-backup-freshness";
|
||||
|
||||
const NOW = new Date("2026-09-10T09:00:00.000Z");
|
||||
const HOUR = 3_600_000;
|
||||
const PERIOD = OFFHOST_BACKUP_PERIOD_HOURS * HOUR;
|
||||
|
||||
/** `age` hours before NOW. */
|
||||
function ago(ms: number): Date {
|
||||
return new Date(NOW.getTime() - ms);
|
||||
}
|
||||
|
||||
describe("classifyOffhostBackup", () => {
|
||||
it("says never when the worker has put nothing there for this account", () => {
|
||||
expect(classifyOffhostBackup({ lastSuccessAt: null, now: NOW })).toEqual({
|
||||
freshness: "never",
|
||||
ageHours: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("is fresh right up to the one-period boundary and due one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD), now: NOW }).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD + 1), now: NOW })
|
||||
.freshness,
|
||||
).toBe("due");
|
||||
});
|
||||
|
||||
it("is due right up to the two-period boundary and stale one millisecond past it", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2), now: NOW })
|
||||
.freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(PERIOD * 2 + 1), now: NOW })
|
||||
.freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("reports whole hours of age", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: ago(90 * 60_000), now: NOW }),
|
||||
).toEqual({ freshness: "fresh", ageHours: 1 });
|
||||
});
|
||||
|
||||
it("honours a caller's own period rather than the nightly default", () => {
|
||||
// A six-hour schedule: the same 12-hour-old object that is fresh under the
|
||||
// nightly cron is two periods behind under this one.
|
||||
const twelveHours = ago(12 * HOUR);
|
||||
expect(
|
||||
classifyOffhostBackup({ lastSuccessAt: twelveHours, now: NOW }).freshness,
|
||||
).toBe("fresh");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 6,
|
||||
}).freshness,
|
||||
).toBe("due");
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastSuccessAt: twelveHours,
|
||||
now: NOW,
|
||||
periodHours: 5,
|
||||
}).freshness,
|
||||
).toBe("stale");
|
||||
});
|
||||
|
||||
it("does not call a copy stale because the host clock ran backwards", () => {
|
||||
expect(
|
||||
classifyOffhostBackup({
|
||||
lastSuccessAt: new Date(NOW.getTime() + 2 * HOUR),
|
||||
now: NOW,
|
||||
}),
|
||||
).toEqual({ freshness: "fresh", ageHours: 0 });
|
||||
});
|
||||
});
|
||||
@@ -184,6 +184,7 @@ describe("runOffhostBackup", () => {
|
||||
s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0]));
|
||||
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
@@ -229,11 +230,26 @@ describe("runOffhostBackup", () => {
|
||||
// Nothing went through the whole-buffer arm.
|
||||
expect(s3.putObject).not.toHaveBeenCalled();
|
||||
expect(s3.putStream).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Every account that got an object gets a ledger row, because the run's
|
||||
// own counts cannot say WHICH account has a copy off-host and which one
|
||||
// has been failing quietly since March.
|
||||
expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(2);
|
||||
const ledgered = prisma.offhostBackupState.upsert.mock.calls.map(
|
||||
(call) => (call[0] as { where: { userId: string } }).where.userId,
|
||||
);
|
||||
expect(ledgered).toEqual(["u1", "u2"]);
|
||||
const first = prisma.offhostBackupState.upsert.mock.calls[0]?.[0] as {
|
||||
create: { sizeBytes: number; lastSuccessAt: Date };
|
||||
};
|
||||
expect(first.create.sizeBytes).toBe(ct.byteLength);
|
||||
expect(first.create.lastSuccessAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("uploads the canonical builder output without reshaping it", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
const canonicalPayload = {
|
||||
@@ -291,6 +307,7 @@ describe("runOffhostBackup", () => {
|
||||
it("counts per-user failures without aborting the whole run", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: {
|
||||
findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
|
||||
},
|
||||
@@ -319,6 +336,7 @@ describe("runOffhostBackup", () => {
|
||||
it("refuses an account whose object outgrows one multipart upload", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
mocks.buildFullBackupPayload.mockResolvedValue({
|
||||
@@ -355,6 +373,7 @@ describe("runOffhostBackup", () => {
|
||||
it("reports the largest object it wrote", async () => {
|
||||
const s3 = makeS3Mock();
|
||||
const prisma = {
|
||||
offhostBackupState: { upsert: vi.fn() },
|
||||
user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
|
||||
};
|
||||
const report = await runOffhostBackup(
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* How fresh one account's newest off-host copy is.
|
||||
*
|
||||
* The nightly off-host job reports counts — uploaded, failed, total — and a
|
||||
* count is exactly the shape that hides a single account. A cohort where one
|
||||
* account's object has failed every night for a month reports "99 uploaded, 1
|
||||
* failed" every night, which reads like weather rather than like an account
|
||||
* that has no disaster-recovery copy at all. The per-account ledger the worker
|
||||
* writes (`OffhostBackupState`) turns that into a fact the console can state
|
||||
* per row, and this function is the only place that fact is judged.
|
||||
*
|
||||
* Deliberately pure and deliberately not talking to the bucket: the admin page
|
||||
* computes staleness from the timestamp the worker wrote, so the answer is the
|
||||
* same whether or not the app process holds a read grant on the bucket, and a
|
||||
* listing of somebody's backups never crosses the network to show a page.
|
||||
*
|
||||
* Four states, because the run is nightly and one missed night is not the same
|
||||
* event as four:
|
||||
*
|
||||
* never — the worker has never put an object there for this account.
|
||||
* fresh — inside one schedule period. This is what every account looks like
|
||||
* on a host where the cron is doing its job.
|
||||
* due — past one period, inside two. One night produced nothing: a retry, a
|
||||
* long run, a restart during the window. Worth showing, not worth
|
||||
* alarming about.
|
||||
* stale — past two periods. Two consecutive nights produced nothing for this
|
||||
* account, which no ordinary hiccup explains.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Hours between two scheduled off-host runs. Tracks `OFFHOST_BACKUP_CRON`
|
||||
* (`30 2 * * *`) in `src/lib/jobs/reminder/register-maintenance.ts` — nightly.
|
||||
*/
|
||||
export const OFFHOST_BACKUP_PERIOD_HOURS = 24;
|
||||
|
||||
export type OffhostBackupFreshness = "never" | "fresh" | "due" | "stale";
|
||||
|
||||
export interface OffhostBackupVerdict {
|
||||
freshness: OffhostBackupFreshness;
|
||||
/** Whole hours since the newest object landed; null when there is none. */
|
||||
ageHours: number | null;
|
||||
}
|
||||
|
||||
export function classifyOffhostBackup(args: {
|
||||
lastSuccessAt: Date | null;
|
||||
now: Date;
|
||||
/** Defaults to the nightly schedule. Tests pass their own. */
|
||||
periodHours?: number;
|
||||
}): OffhostBackupVerdict {
|
||||
const { lastSuccessAt, now } = args;
|
||||
if (lastSuccessAt === null) return { freshness: "never", ageHours: null };
|
||||
|
||||
const periodMs =
|
||||
(args.periodHours ?? OFFHOST_BACKUP_PERIOD_HOURS) * 3_600_000;
|
||||
const ageMs = now.getTime() - lastSuccessAt.getTime();
|
||||
// A clock that ran backwards between the upload and this read reports a
|
||||
// negative age. That is a host-clock problem, not a backup problem, and
|
||||
// calling the copy stale over it would point the operator at the wrong
|
||||
// thing.
|
||||
const ageHours = Math.max(0, Math.floor(ageMs / 3_600_000));
|
||||
|
||||
if (ageMs <= periodMs) return { freshness: "fresh", ageHours };
|
||||
if (ageMs <= periodMs * 2) return { freshness: "due", ageHours };
|
||||
return { freshness: "stale", ageHours };
|
||||
}
|
||||
@@ -61,6 +61,27 @@ function decodeBackupKey(raw: string): Buffer {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this host has the five variables the nightly job needs.
|
||||
*
|
||||
* Separate from `loadOffhostConfig()` because a surface that only wants to say
|
||||
* "off-host backup is not set up here" must not decode the encryption key to
|
||||
* find out — `decodeBackupKey` throws on a malformed one, and an operator
|
||||
* whose key has a typo should see the admin page, not a 500.
|
||||
*/
|
||||
export function offhostBackupConfigured(): boolean {
|
||||
return OFFHOST_ENV_VARS.every((name) => Boolean(process.env[name]));
|
||||
}
|
||||
|
||||
/** The five variables the nightly job needs, in one place. */
|
||||
const OFFHOST_ENV_VARS = [
|
||||
"BACKUP_S3_ENDPOINT",
|
||||
"BACKUP_S3_BUCKET",
|
||||
"BACKUP_S3_ACCESS_KEY",
|
||||
"BACKUP_S3_SECRET_KEY",
|
||||
"BACKUP_ENCRYPTION_KEY",
|
||||
] as const;
|
||||
|
||||
export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
const endpoint = process.env.BACKUP_S3_ENDPOINT;
|
||||
const bucket = process.env.BACKUP_S3_BUCKET;
|
||||
@@ -573,6 +594,21 @@ export async function runOffhostBackup(
|
||||
}),
|
||||
options,
|
||||
);
|
||||
// The per-account ledger, written the moment the object exists. The run
|
||||
// itself only ever reported counts, and a count cannot say WHICH account
|
||||
// has no copy — which is the one question an operator asks about a
|
||||
// backup. `new Date()` rather than the run's `now`, because on a large
|
||||
// cohort the two are hours apart and the row is meant to say when this
|
||||
// account's object landed.
|
||||
await prisma.offhostBackupState.upsert({
|
||||
where: { userId: user.id },
|
||||
update: { lastSuccessAt: new Date(), sizeBytes: objectBytes },
|
||||
create: {
|
||||
userId: user.id,
|
||||
lastSuccessAt: new Date(),
|
||||
sizeBytes: objectBytes,
|
||||
},
|
||||
});
|
||||
largestObjectBytes = Math.max(largestObjectBytes, objectBytes);
|
||||
uploaded++;
|
||||
} catch (err) {
|
||||
|
||||
Reference in new issue
Block a user