Merge fix/backup-timeout

# Conflicts:
#	CHANGELOG.md
#	tests/integration/backup-round-trip.test.ts
This commit is contained in:
Marc-André Bombeck committed 2026-09-03 03:23:41 +02:00
commit b2a8edf65f
34 files changed
+1046 -62

No files matched your search

@@ -19,7 +19,7 @@ import { prisma } from "@/lib/db";
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
import { apiError, getClientIp } from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { decrypt } from "@/lib/crypto";
import { unpackBackupBlob } from "@/lib/export/backup-blob";
import { annotate } from "@/lib/logging/context";
import { parseBackupPayload } from "@/lib/validations/backup";
@@ -52,7 +52,7 @@ export const GET = apiHandler(
let plaintext: string;
try {
plaintext = decrypt(backup.data);
plaintext = unpackBackupBlob(backup.data);
} catch (err) {
// Decryption failure is rare but real — a rotated/missing key, or a
// corrupted blob from a partially-failed write. Surface it as a 500
@@ -19,7 +19,7 @@ import { prisma, toJson } from "@/lib/db";
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { decrypt } from "@/lib/crypto";
import { unpackBackupBlob } from "@/lib/export/backup-blob";
import { encryptNote } from "@/lib/crypto/note-cipher";
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
@@ -180,7 +180,7 @@ const handler = apiHandler(
let plaintext: string;
try {
plaintext = decrypt(backup.data);
plaintext = unpackBackupBlob(backup.data);
} catch (err) {
await auditLog("admin.backups.restore.failed", {
userId: admin.id,
@@ -16,7 +16,7 @@ import { NextRequest } from "next/server";
import { prisma } from "@/lib/db";
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
import { apiError, apiSuccess } from "@/lib/api-response";
import { decrypt } from "@/lib/crypto";
import { unpackBackupBlob } from "@/lib/export/backup-blob";
import { annotate } from "@/lib/logging/context";
import {
isCompatibleSchemaVersion,
@@ -44,7 +44,7 @@ export const GET = apiHandler(
let plaintext: string;
try {
plaintext = decrypt(backup.data);
plaintext = unpackBackupBlob(backup.data);
} catch {
return apiError("Failed to decrypt backup payload", 500);
}
+49 -14
View File
@@ -5,11 +5,20 @@
* shipped — only metadata (id, userId, username, type, size in bytes,
* createdAt). The encrypted blob remains server-side; admins can trigger
* a re-snapshot but not download another user's payload from the UI.
*
* Alongside the rows, the response says whether the SCHEDULE is still alive:
* how old the newest scheduled copy is and how the last scheduled run ended.
* Without those two the page cannot tell a working backup from one that
* stopped six weeks ago — every row it lists has a perfectly ordinary
* timestamp either way.
*/
import { prisma } from "@/lib/db";
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
import { readLastQueueRun } from "@/lib/jobs/job-failures";
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
// so callers (in particular `components/admin/backups-section.tsx`) don't
// have to reach across the component → route-handler layer boundary.
@@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups";
export const dynamic = "force-dynamic";
/** The type the weekly worker writes; anything else was uploaded by hand. */
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
interface BackupMetaRow {
id: string;
user_id: string;
username: string;
type: string;
size_bytes: number;
created_at: Date;
}
export const GET = apiHandler(async () => {
await requireAdmin();
annotate({ action: { name: "admin.backups.list" } });
// Pull metadata only — `data` would balloon the response and reveal
// ciphertext we don't need on the listing page.
const backups = await prisma.dataBackup.findMany({
include: { user: { select: { id: true, username: true } } },
orderBy: { createdAt: "desc" },
});
// Metadata only, and the size read as `octet_length` in the database rather
// than by pulling every blob into this process to measure it. The listing
// needs a number, not the ciphertext, and on an instance with large records
// fetching them all was its own way of running out of memory. Base64 is
// ASCII, so the column's byte length is exactly what the wire reports.
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
SELECT b.id, b.user_id, u.username, b.type,
octet_length(b.data) AS size_bytes,
b.created_at
FROM data_backups b
JOIN users u ON u.id = b.user_id
ORDER BY b.created_at DESC
`;
const rows: BackupRow[] = backups.map((b) => ({
id: b.id,
userId: b.userId,
username: b.user.username,
type: b.type,
sizeBytes: Buffer.byteLength(b.data, "utf8"),
createdAt: b.createdAt.toISOString(),
const list: BackupRow[] = rows.map((row) => ({
id: row.id,
userId: row.user_id,
username: row.username,
type: row.type,
sizeBytes: Number(row.size_bytes),
createdAt: row.created_at.toISOString(),
}));
const payload: BackupsList = {
rows,
rows: list,
// Matches the retention window the backup-prune job enforces so the
// backups page states the same number the worker acts on.
retentionDays: 90,
schedule: summariseBackupSchedule({
scheduledCreatedAt: rows
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
.map((row) => row.created_at),
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
now: new Date(),
}),
};
return apiSuccess(payload);
+12 -5
View File
@@ -15,11 +15,13 @@ import { auditLog } from "@/lib/auth/audit";
import { annotate } from "@/lib/logging/context";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { checkRateLimit } from "@/lib/rate-limit";
import {
DATA_BACKUP_QUEUE,
DATA_BACKUP_SEND_OPTIONS,
} from "@/lib/jobs/data-backup-policy";
export const dynamic = "force-dynamic";
const DATA_BACKUP_QUEUE = "data-backup";
export const POST = apiHandler(async (request: NextRequest) => {
const { user: admin } = await requireAdmin();
annotate({ action: { name: "admin.backups.run" } });
@@ -57,9 +59,14 @@ export const POST = apiHandler(async (request: NextRequest) => {
throw new HttpError(503, "Background worker is not running");
}
const jobId = await boss.send(DATA_BACKUP_QUEUE, {
triggeredAt: new Date().toISOString(),
});
// Same expiration window as the weekly cron. An ad-hoc snapshot runs the
// identical pass over the identical record, so it must not silently keep the
// 15-minute queue default the cron was rescued from.
const jobId = await boss.send(
DATA_BACKUP_QUEUE,
{ triggeredAt: new Date().toISOString() },
DATA_BACKUP_SEND_OPTIONS,
);
annotate({ meta: { job_id: jobId ?? null } });
+2 -2
View File
@@ -26,7 +26,7 @@ import { prisma } from "@/lib/db";
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
import { auditLog } from "@/lib/auth/audit";
import { encrypt } from "@/lib/crypto";
import { packBackupBlob } from "@/lib/export/backup-blob";
import { annotate } from "@/lib/logging/context";
import { checkRateLimit } from "@/lib/rate-limit";
import {
@@ -190,7 +190,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
// intact so multiple uploads can coexist for the same user without
// overwriting each other or the rolling WEEKLY_AUTO snapshot.
const uploadType = `MANUAL_UPLOAD_${Date.now()}`;
const encrypted = encrypt(JSON.stringify(payload));
const encrypted = packBackupBlob(JSON.stringify(payload));
const created = await prisma.dataBackup.create({
data: {
@@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server";
import de from "../../../../messages/de.json";
import en from "../../../../messages/en.json";
const healthySchedule = {
lastSuccessAt: "2026-09-01T03:00:00.000Z",
lastSuccessAgeDays: 2,
staleAfterDays: 10,
stale: false,
lastRun: {
at: "2026-09-01T03:00:12.000Z",
state: "completed" as const,
error: null,
},
lastRunFailed: false,
};
const mocks = vi.hoisted(() => ({
mutationOptions: [] as Array<{
onSuccess?: (data: { summary: Record<string, number> }) => void;
}>,
toastSuccess: vi.fn(),
// Mutable so one file can render both a healthy schedule and a broken one;
// the query mock reads it on every render.
queryData: { rows: [], retentionDays: 30 } as Record<string, unknown>,
}));
/**
@@ -34,7 +50,7 @@ vi.mock("sonner", () => ({
vi.mock("@tanstack/react-query", () => ({
useQuery: () => ({
data: { rows: [], retentionDays: 30 },
data: mocks.queryData,
isLoading: false,
isError: false,
}),
@@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context";
import { BackupsSection } from "../backups-section";
function render(locale: "en" | "de" = "en") {
mocks.queryData = {
rows: [],
retentionDays: 30,
schedule: healthySchedule,
};
return renderToStaticMarkup(
<I18nProvider initialLocale={locale}>
<BackupsSection />
@@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => {
});
});
/**
* The gap that let a weekly backup stop for a month and a half without anyone
* noticing: the page rendered whatever rows it had, and a copy from six weeks
* ago is indistinguishable from Sunday's until somebody subtracts the dates.
*/
describe("BackupsSection — schedule health", () => {
function withSchedule(schedule: Record<string, unknown>) {
mocks.queryData = { rows: [], retentionDays: 30, schedule };
return renderToStaticMarkup(
<I18nProvider initialLocale="en">
<BackupsSection />
</I18nProvider>,
);
}
it("stays quiet while the schedule is healthy", () => {
const html = render();
expect(html).not.toContain('data-slot="backup-schedule-notice"');
});
it("says so when the newest scheduled copy has gone stale", () => {
const html = withSchedule({
...healthySchedule,
lastSuccessAt: "2026-07-19T03:00:00.000Z",
lastSuccessAgeDays: 46,
stale: true,
lastRun: null,
lastRunFailed: false,
});
expect(html).toContain('data-slot="backup-schedule-notice"');
expect(html).toContain("The weekly backup has stopped");
expect(html).toContain("46");
});
it("names the reason when the last scheduled run failed", () => {
const html = withSchedule({
...healthySchedule,
lastRun: {
at: "2026-08-30T03:46:00.000Z",
state: "failed",
error: "job timed out",
},
lastRunFailed: true,
});
expect(html).toContain("The last scheduled backup run failed");
expect(html).toContain("job timed out");
});
it("survives a response that predates the schedule field", () => {
mocks.queryData = { rows: [], retentionDays: 30 };
expect(() =>
renderToStaticMarkup(
<I18nProvider initialLocale="en">
<BackupsSection />
</I18nProvider>,
),
).not.toThrow();
});
});
describe("BackupsSection — upload outcome", () => {
it("includes health profile fact revisions in the restored record total", () => {
mocks.toastSuccess.mockClear();
+70
View File
@@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import type { BackupRow, BackupsList } from "@/types/backups";
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
import type {
MissingBackupSection,
RestoreSkipSummary,
@@ -560,6 +561,71 @@ export function RestoreSkipReport({
);
}
/**
* Says out loud whether the SCHEDULE is still alive.
*
* The weekly pass on the maintainer's own instance stopped producing copies
* and nothing surfaced it for a month and a half: the table listed the rows it
* had, each with an ordinary timestamp, and a copy from six weeks ago reads
* exactly like one made on Sunday until somebody does the subtraction. Two
* independent facts are shown, because they fail at different times — the age
* of the newest scheduled copy never ages out, while the run's own outcome
* carries the reason ("job timed out") and survives only as long as pg-boss
* keeps the row.
*
* Not dismissible: it reflects live state, so the only way to clear it is to
* make a backup succeed.
*/
function ScheduleHealthNotice({
schedule,
}: {
schedule: BackupScheduleStatus;
}) {
const { t } = useTranslations();
const fmt = useFormatters();
if (!schedule.stale && !schedule.lastRunFailed) return null;
return (
<div
role="alert"
data-slot="backup-schedule-notice"
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" aria-hidden="true" />
<div>
<p className="font-medium">
{schedule.stale
? t("admin.section.backups.scheduleStaleTitle")
: t("admin.section.backups.scheduleFailedTitle")}
</p>
{schedule.stale ? (
<p className="text-xs">
{t("admin.section.backups.scheduleStaleDescription", {
days: schedule.lastSuccessAgeDays ?? 0,
threshold: schedule.staleAfterDays,
})}
</p>
) : null}
{schedule.lastRunFailed && schedule.lastRun ? (
<p className="text-xs">
{t("admin.section.backups.scheduleFailedDescription", {
when: fmt.dateTime(schedule.lastRun.at),
reason:
schedule.lastRun.error ||
t("admin.section.backups.scheduleFailedNoReason"),
})}
</p>
) : null}
<p className="text-muted-foreground mt-2 text-xs">
{t("admin.section.backups.scheduleRemedy")}
</p>
</div>
</div>
</div>
);
}
export function BackupsSection() {
const { t } = useTranslations();
const fmt = useFormatters();
@@ -833,6 +899,10 @@ export function BackupsSection() {
}
/>
{data?.schedule ? (
<ScheduleHealthNotice schedule={data.schedule} />
) : null}
{/* Upload card — separate from the table so admins can ingest a
backup file independently of any existing rows. The visible
button proxies a hidden file input so the layout stays clean
@@ -0,0 +1,65 @@
/**
* The stored-backup envelope: compress, then encrypt.
*
* The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to
* Postgres. For an account with a few hundred thousand measurements that means
* the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and
* the copy the database driver makes of it all alive at once — well over a
* gigabyte for a record whose JSON is a few hundred megabytes. The run died of
* memory, not of slow SQL.
*
* Compressing before the cipher is what makes the tail of that pipeline cheap:
* health JSON is extremely repetitive, so everything downstream of the gzip
* shrinks by an order of magnitude. The envelope has to stay readable both
* ways — rows written before this change are plain `encrypt(json)` and must
* keep restoring.
*/
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { describe, expect, it } from "vitest";
import { encrypt } from "@/lib/crypto";
import { packBackupBlob, unpackBackupBlob } from "../backup-blob";
/** A record-shaped string: many rows, few distinct keys. */
function sampleJson(rows: number): string {
return JSON.stringify({
schemaVersion: "2",
measurements: Array.from({ length: rows }, (_, i) => ({
id: `measurement-${i}`,
type: "PULSE",
value: 60 + (i % 30),
unit: "bpm",
measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(),
source: "APPLE_HEALTH",
deletedAt: null,
})),
});
}
describe("backup blob envelope", () => {
it("round-trips the exact JSON it was handed", () => {
const json = sampleJson(200);
expect(unpackBackupBlob(packBackupBlob(json))).toBe(json);
});
it("stores a record-shaped payload far smaller than the plaintext", () => {
const json = sampleJson(5_000);
const packed = packBackupBlob(json);
// The old envelope was 1.33× the JSON. Anything near that means the
// compression leg is not running.
expect(packed.length).toBeLessThan(json.length / 4);
});
it("still reads a row written before the envelope existed", () => {
const json = sampleJson(10);
expect(unpackBackupBlob(encrypt(json))).toBe(json);
});
it("fails closed on a corrupt blob rather than returning junk", () => {
const packed = packBackupBlob(sampleJson(10));
const mangled = `${packed.slice(0, -8)}AAAAAAAA`;
expect(() => unpackBackupBlob(mangled)).toThrow();
});
});
+59
View File
@@ -0,0 +1,59 @@
/**
* The envelope a `DataBackup.data` row is stored in: compress, then encrypt.
*
* Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))`
* straight to Postgres. For an account with a few hundred thousand
* measurements that is four full copies of the record alive at the same
* moment: the payload object graph, the JSON string, the base64 ciphertext at
* 1.33× the JSON, and the copy the database driver makes of that parameter on
* its way to the wire. Measured on a seeded 445 000-measurement account, the
* JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap
* it never reaches the insert, and on a bigger heap it only reaches it later.
* The weekly pass was not slow; it could not finish.
*
* Compressing first is what makes the whole tail of that pipeline cheap. A
* health record's JSON is extremely repetitive — the same twenty keys per row,
* timestamps sharing a prefix — so gzip takes that 242 MB to 14 MB, and every
* copy after it shrinks with it. The same run then completes in about eight
* seconds with a peak heap of 434 MB. It is also the reason the stored row
* stops being a liability of its own: the backup lives INSIDE the database it
* would be needed to restore, so its size is not a cosmetic concern.
*
* Both directions. Rows written before this envelope existed are plain
* `encrypt(json)` and have to keep restoring — an operator whose newest usable
* copy predates the fix is exactly the person who needs it to work. The marker
* sits inside the ciphertext rather than in front of it so the stored column
* keeps the one shape `decrypt()` already understands, key ids and all.
*/
import { Buffer } from "node:buffer";
import { gunzipSync, gzipSync } from "node:zlib";
import { decrypt, encrypt } from "@/lib/crypto";
/**
* Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd.
* Chosen so it can never be mistaken for the alternative: a plain payload is
* always a JSON object and therefore always starts with `{`.
*/
const GZIP_MARKER = "HLZ1:";
/** Serialised backup JSON → the string stored in `DataBackup.data`. */
export function packBackupBlob(json: string): string {
const compressed = gzipSync(json).toString("base64");
return encrypt(`${GZIP_MARKER}${compressed}`);
}
/**
* A stored `DataBackup.data` string → the backup JSON.
*
* Fails closed on every arm: a bad key, a mangled ciphertext or a truncated
* gzip member throws rather than returning a partial document, because every
* caller goes on to parse the result as a whole account.
*/
export function unpackBackupBlob(stored: string): string {
const plaintext = decrypt(stored);
if (!plaintext.startsWith(GZIP_MARKER)) return plaintext;
return gunzipSync(
Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"),
).toString("utf8");
}
+20
View File
@@ -316,6 +316,26 @@ function serialiseMoodContext(
};
}
/**
* The same payload, already serialised, with the object graph released.
*
* Its own frame on purpose, and both backup jobs go through it. The payload
* graph is the largest thing either job allocates — on an account with a few
* hundred thousand measurements it is several hundred megabytes of small
* objects — and neither job wants anything but the string. Returning from here
* makes the graph unreachable before the compress-and-encrypt step allocates
* anything of its own, which is part of why the weekly pass now fits in
* memory. Inlining these two statements back into a caller quietly undoes it.
*/
export async function buildFullBackupJson(
prisma: PrismaClient,
userId: string,
options: FullBackupOptions = {},
): Promise<string> {
const { payload } = await buildFullBackupPayload(prisma, userId, options);
return JSON.stringify(payload);
}
/**
* Build the canonical full-backup payload for `userId`. Portable exports omit
* tombstones and document ciphertext; disaster-recovery payloads preserve
@@ -0,0 +1,92 @@
/**
* What the backups page has to be able to say out loud.
*
* The weekly snapshot stopped producing a new copy and nothing surfaced it: the
* page listed the rows it had, each with a perfectly ordinary timestamp, and a
* copy that was a month and a half old looked exactly like one made on Sunday.
* The two facts that would have shown it are the age of the newest scheduled
* copy and the terminal state of the last scheduled run.
*/
import { describe, expect, it } from "vitest";
import {
BACKUP_STALE_AFTER_DAYS,
summariseBackupSchedule,
} from "../backup-schedule-status";
const NOW = new Date("2026-09-03T09:00:00.000Z");
const daysAgo = (days: number) =>
new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000);
describe("summariseBackupSchedule", () => {
it("reports a fresh weekly copy as neither stale nor failed", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(2), daysAgo(9)],
lastRun: {
at: daysAgo(2).toISOString(),
state: "completed",
error: null,
},
now: NOW,
});
expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString());
expect(status.lastSuccessAgeDays).toBe(2);
expect(status.stale).toBe(false);
expect(status.lastRunFailed).toBe(false);
expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS);
});
it("calls a copy older than the window stale", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(46)],
lastRun: null,
now: NOW,
});
expect(status.lastSuccessAgeDays).toBe(46);
expect(status.stale).toBe(true);
// No run row survives that long — the age is the durable signal.
expect(status.lastRunFailed).toBe(false);
});
it("surfaces a failed last run even while the copy is still in window", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(3)],
lastRun: {
at: daysAgo(1).toISOString(),
state: "failed",
error: "job timed out",
},
now: NOW,
});
expect(status.stale).toBe(false);
expect(status.lastRunFailed).toBe(true);
expect(status.lastRun?.error).toBe("job timed out");
});
it("does not call a brand-new instance stale", () => {
const status = summariseBackupSchedule({
scheduledCreatedAt: [],
lastRun: null,
now: NOW,
});
expect(status.lastSuccessAt).toBeNull();
expect(status.lastSuccessAgeDays).toBeNull();
expect(status.stale).toBe(false);
});
it("ignores manual uploads when judging the scheduled cadence", () => {
// The caller passes only WEEKLY_AUTO rows; a manual upload made today must
// not make a dead cron look alive. Proven here by the shape of the input:
// the summary knows nothing but scheduled copies.
const status = summariseBackupSchedule({
scheduledCreatedAt: [daysAgo(30)],
lastRun: null,
now: NOW,
});
expect(status.stale).toBe(true);
});
});
@@ -0,0 +1,55 @@
/**
* Send-policy pins for the weekly `data-backup` queue.
*
* The queue was registered with no send options at all, so every job it minted
* inherited pg-boss's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`.
* A full-record snapshot of a large account does not fit in fifteen minutes, so
* the pass was killed mid-run, redelivered twice against the same record, and
* gave up roughly 46 minutes after it started. Weekly, silently, for as long as
* the record stayed large.
*
* Three facts have to hold together or the window is not real:
*
* 1. the policy names a window well past the 15-minute default,
* 2. the CRON SCHEDULE carries it — send options ride the schedule because
* pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a
* queue-level default would be inert on every already-running instance,
* 3. the admin "run now" route sends with the same options, or an ad-hoc
* snapshot silently keeps the default that broke the cron one.
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
DATA_BACKUP_QUEUE,
DATA_BACKUP_SEND_OPTIONS,
} from "../data-backup-policy";
const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8");
describe("data-backup — pg-boss send policy", () => {
it("widens the expiration far past the 15-minute queue default", () => {
expect(DATA_BACKUP_QUEUE).toBe("data-backup");
// 900 s is the pg-boss 12.26 queue default (`QUEUE_DEFAULTS.expire_seconds`
// in `dist/plans.js`); anything at or below it re-opens the bug.
expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThan(900);
expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThanOrEqual(
60 * 60,
);
});
it("the weekly schedule carries the policy", () => {
const source = read("src/lib/jobs/reminder/register-maintenance.ts");
expect(source).toMatch(
/\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/,
);
});
it("the admin run-now route sends with the same policy", () => {
const source = read("src/app/api/admin/backups/run/route.ts");
expect(source).toMatch(
/boss\.send\(\s*DATA_BACKUP_QUEUE,[\s\S]{0,200}DATA_BACKUP_SEND_OPTIONS,?\s*\)/,
);
});
});
@@ -26,4 +26,16 @@ describe("pg-boss failed-row ledger retention", () => {
PG_BOSS_FAILED_ROW_AVAILABILITY_HOURS,
);
});
it("pins the queue defaults the data-backup window had to escape", () => {
const plans = readFileSync(join(PG_BOSS_ROOT, "dist/plans.js"), "utf8");
// These two lines are the whole story behind the weekly backup's "job
// timed out": a pass with no send options gets fifteen minutes and two
// redeliveries, so a snapshot that cannot finish in fifteen minutes gives
// up 45 minutes later having never once completed. If a pg-boss upgrade
// changes either default, `DATA_BACKUP_SEND_OPTIONS` deserves a re-read.
expect(plans).toContain("expire_seconds: FIFTEEN_MINUTES");
expect(plans).toContain("retry_limit: 2");
});
});
+38 -1
View File
@@ -1,3 +1,5 @@
import { createCipheriv, randomBytes } from "node:crypto";
import { describe, it, expect, beforeEach, vi } from "vitest";
import {
BACKUP_SCHEMA_VERSION,
@@ -8,8 +10,20 @@ const mocks = vi.hoisted(() => ({
buildFullBackupPayload: vi.fn(),
}));
// The uploader takes the already-serialised form, so the stand-in for
// `buildFullBackupJson` serialises whatever the payload mock was told to
// return. Every case below keeps stubbing the PAYLOAD, which is the thing
// these tests are actually about.
vi.mock("@/lib/export/full-backup-payload", () => ({
buildFullBackupPayload: mocks.buildFullBackupPayload,
buildFullBackupJson: async (...args: unknown[]) =>
JSON.stringify(
(
(await mocks.buildFullBackupPayload(...args)) as {
payload: unknown;
}
).payload,
),
}));
import {
encryptBackup,
@@ -27,10 +41,33 @@ describe("offhost-backup envelope", () => {
const key = Buffer.from(ENC_KEY, "hex");
const payload = JSON.stringify({ hello: "world", n: 42 });
const buf = encryptBackup(payload, key);
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x01");
// Version 2: the plaintext is gzipped before the cipher sees it. This
// assertion used to pin version 1, which is the version that could not
// encrypt a large account's dump without four full copies of it resident
// at once. The pin is flipped rather than dropped — the byte is the whole
// reason a reader can tell the two apart.
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x02");
expect(decryptBackup(buf, key)).toBe(payload);
});
it("still reads a version-1 object written before compression", () => {
const key = Buffer.from(ENC_KEY, "hex");
const payload = JSON.stringify({ hello: "world", n: 42 });
// Hand-built v1 envelope: magic || 0x01 || iv || tag || ciphertext, the
// exact bytes every object already sitting in an operator's bucket has.
const iv = randomBytes(12);
const cipher = createCipheriv("aes-256-gcm", key, iv);
const ct = Buffer.concat([cipher.update(payload, "utf8"), cipher.final()]);
const legacy = Buffer.concat([
Buffer.from("HLBK\x01", "binary"),
iv,
cipher.getAuthTag(),
ct,
]);
expect(decryptBackup(legacy, key)).toBe(payload);
});
it("rejects tampered ciphertext", () => {
const key = Buffer.from(ENC_KEY, "hex");
const buf = encryptBackup("data", key);
+78
View File
@@ -0,0 +1,78 @@
/**
* Whether the weekly backup is actually still happening, in two facts.
*
* The pass stopped producing a new copy and nothing said so for a month and a
* half. Everything that could have told the operator was either invisible or
* short-lived: the handler's own error never ran (the job was killed, not
* thrown), `recordError()` increments an unnamed counter, and the admin status
* page's failing-queue panel only looks back 72 hours — which for a weekly
* queue means a permanent failure is on screen three days in seven. The backups
* page itself showed a perfectly ordinary row with a perfectly ordinary
* timestamp, because a 46-day-old copy looks exactly like a fresh one until you
* read the date.
*
* So the surface carries both legs, and neither is derived from the other:
*
* - the AGE of the newest scheduled copy, which never ages out of any
* retention window and would have caught this on the second missed Sunday;
* - the terminal state of the last scheduled RUN, which is the only place the
* reason ("job timed out") is written down at all.
*
* Manual uploads are excluded by the shape of the input: the caller passes only
* the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look
* alive.
*/
import type { LastQueueRun } from "@/lib/jobs/job-failures";
/**
* How old the newest scheduled copy may get before the page says so. The cron
* is weekly, so seven days is normal and eight is a rounding error; ten means
* at least one Sunday produced nothing.
*/
export const BACKUP_STALE_AFTER_DAYS = 10;
const DAY_MS = 24 * 60 * 60 * 1000;
export interface BackupScheduleStatus {
/** ISO instant of the newest scheduled copy, or null when there is none. */
lastSuccessAt: string | null;
/** Whole days since that copy was written. */
lastSuccessAgeDays: number | null;
/** The threshold behind `stale`, so the UI can name it. */
staleAfterDays: number;
/** A scheduled copy exists and is older than the threshold. */
stale: boolean;
/** The last scheduled run pg-boss still remembers. */
lastRun: LastQueueRun | null;
/** That run ended `failed` or `cancelled`. */
lastRunFailed: boolean;
}
export function summariseBackupSchedule(args: {
/** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */
scheduledCreatedAt: readonly Date[];
lastRun: LastQueueRun | null;
now: Date;
}): BackupScheduleStatus {
const newest = args.scheduledCreatedAt.reduce<Date | null>(
(best, at) => (best === null || at > best ? at : best),
null,
);
const ageDays =
newest === null
? null
: Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS);
return {
lastSuccessAt: newest?.toISOString() ?? null,
lastSuccessAgeDays: ageDays,
staleAfterDays: BACKUP_STALE_AFTER_DAYS,
// An instance that has never produced a scheduled copy is not stale, it is
// new — the page's empty state already says there is nothing there, and
// calling a fresh install "overdue" would train the operator to ignore the
// warning that matters.
stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS,
lastRun: args.lastRun,
lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed",
};
}
+39
View File
@@ -0,0 +1,39 @@
/**
* Queue name and pg-boss send policy for the weekly on-host data backup.
*
* A leaf module on purpose: the cron registrar, the admin "run now" route and
* the policy test all need these two facts, and none of them should have to
* pull in the worker's handler graph to get them.
*
* The queue was registered with no send options, so every job it minted took
* pg-boss 12.26's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`
* (`QUEUE_DEFAULTS` in `pg-boss/dist/plans.js`). Fifteen minutes is a fine
* default for a cleanup tick and far too short for a pass that serialises every
* account's whole record: the job was killed mid-run, redelivered twice against
* the same record, and reached `failed` with `job timed out` about 46 minutes
* after it started. Nothing about that says "too slow" — the pass never got to
* finish once.
*
* Two hours is deliberately generous rather than tuned. The handler now costs a
* fraction of that on a large record (see the CHANGELOG entry), but the pass
* loops over EVERY account on the instance and a self-host's disk is whatever
* it is; the window's job is to be impossible to hit for the right reasons, and
* a failure that arrives at the two-hour mark is a real one worth reading.
* pg-boss refuses anything at or past 24 hours.
*
* The default `retryLimit: 2` stays. The old redeliveries were pointless
* because the first attempt could not succeed either; a transient failure — a
* pool hiccup, a locked row — genuinely benefits from a second attempt, and the
* alternative is losing the week.
*
* The options ride on the SEND, not on `createQueue`: pg-boss's `create_queue()`
* ends in `ON CONFLICT DO NOTHING`, so a queue-level default would apply to a
* fresh database and be inert on every instance that already has the queue —
* exactly the trap `reconcileQueuePolicies` exists to work around.
*/
export const DATA_BACKUP_QUEUE = "data-backup";
export const DATA_BACKUP_SEND_OPTIONS = {
expireInSeconds: 2 * 60 * 60,
} as const;
+74 -1
View File
@@ -98,7 +98,11 @@ export async function readFailingQueues(
max(completed_on) AS last_failed_at,
(
array_agg(
COALESCE(output->>'message', output::text)
COALESCE(
output->>'message',
output->'value'->>'message',
output::text
)
ORDER BY completed_on DESC NULLS LAST
)
)[1] AS last_error
@@ -124,6 +128,75 @@ export async function readFailingQueues(
}
}
/** The terminal state pg-boss recorded for the newest run of one queue. */
export interface LastQueueRun {
/** ISO instant the run reached its terminal state. */
at: string;
/** `failed` covers both a thrown handler and an expired (killed) job. */
state: "completed" | "failed" | "cancelled";
/** The failure text, redacted and truncated. Null when the run succeeded. */
error: string | null;
}
/**
* The newest terminal run of one queue, whatever its outcome.
*
* `readFailingQueues` answers "what broke recently" and is deliberately scoped
* to a 72-hour window, which is the right shape for a status page watching
* thirty queues that tick by the minute. It is the wrong shape for a WEEKLY
* pass: a Sunday-night failure has fallen out of the window by Wednesday, so a
* job that fails every single week is visible for three days in seven and
* invisible for four. This reader takes the newest run instead of a window, so
* a weekly queue's last outcome is readable right up until the next one.
*
* Bounded by pg-boss's own retention: terminal rows live `deletion_seconds`
* (seven days by default) past completion, so `null` here means "no run row
* survives", which is not the same as "the pass never ran". Callers pair it
* with a fact that does not age out — for backups, the age of the newest
* stored copy.
*/
export async function readLastQueueRun(
queue: string,
): Promise<LastQueueRun | null> {
try {
const rows = await prisma.$queryRaw<
Array<{ state: string; completed_on: Date | null; output: string | null }>
>`
SELECT
state,
completed_on,
-- pg-boss wraps a thrown value: an expired job's row reads
-- {"value":{"message":"job timed out"}}, so a plain ->>'message'
-- misses it and the operator gets raw JSON instead of the sentence.
COALESCE(
output->>'message',
output->'value'->>'message',
output::text
) AS output
FROM pgboss.job
WHERE name = ${queue}
AND state IN ('completed', 'failed', 'cancelled')
AND completed_on IS NOT NULL
ORDER BY completed_on DESC
LIMIT 1
`;
const row = rows[0];
if (!row || row.completed_on === null) return null;
const state =
row.state === "failed" || row.state === "cancelled"
? row.state
: "completed";
return {
at: row.completed_on.toISOString(),
state,
error: state === "completed" ? null : presentError(row.output),
};
} catch {
// No `pgboss` schema (web-only deployment) or no permission to read it.
return null;
}
}
export interface QueueFailureForUser {
/** ISO instant the user's newest job on this queue gave up. */
lastFailedAt: string;
+40 -12
View File
@@ -18,8 +18,9 @@
* the backup history. See docs/ops/backup-restore.md.
*/
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
import { gunzipSync, gzipSync } from "node:zlib";
import type { PrismaClient } from "@/generated/prisma/client";
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
import { getEvent } from "@/lib/logging/context";
const ALGORITHM = "aes-256-gcm";
@@ -84,20 +85,43 @@ export function loadOffhostConfig(): OffhostBackupConfig | null {
* Encrypt a JSON payload with the dedicated backup key.
*
* Wire format (binary):
* magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext
* magic(4)="HLBK" || version(1) || iv(12) || tag(16) || ciphertext
*
* Version 1 encrypts the JSON directly. Version 2 gzips it first, which is
* what the version byte was there for: a large account's dump is hundreds of
* megabytes of extremely repetitive JSON, and encrypting it whole means the
* string, the cipher's input copy, the ciphertext and the concatenation are
* all resident at once — the same arithmetic that made the in-database weekly
* backup unfinishable. Gzip takes an order of magnitude off every one of them
* and off the object in the bucket. Objects written under version 1 stay
* readable; both the restore drill and `scripts/restore-backup.ts` come
* through `decryptBackup`, so neither needs to know which it got.
*/
const BACKUP_ENVELOPE_PLAIN = 0x01;
const BACKUP_ENVELOPE_GZIP = 0x02;
export function encryptBackup(plaintext: string, key: Buffer): Buffer {
const iv = randomBytes(IV_LENGTH);
const cipher = createCipheriv(ALGORITHM, key, iv);
const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
const ct = Buffer.concat([
cipher.update(gzipSync(plaintext)),
cipher.final(),
]);
const tag = cipher.getAuthTag();
return Buffer.concat([Buffer.from("HLBK\x01", "binary"), iv, tag, ct]);
const header = Buffer.from([
...Buffer.from("HLBK", "binary"),
BACKUP_ENVELOPE_GZIP,
]);
return Buffer.concat([header, iv, tag, ct]);
}
export function decryptBackup(buf: Buffer, key: Buffer): string {
const magic = buf.subarray(0, 4).toString("binary");
const version = buf[4];
if (magic !== "HLBK" || version !== 0x01) {
if (
magic !== "HLBK" ||
(version !== BACKUP_ENVELOPE_PLAIN && version !== BACKUP_ENVELOPE_GZIP)
) {
throw new Error("Invalid backup envelope (bad magic or version)");
}
const iv = buf.subarray(5, 5 + IV_LENGTH);
@@ -105,7 +129,10 @@ export function decryptBackup(buf: Buffer, key: Buffer): string {
const ct = buf.subarray(5 + IV_LENGTH + TAG_LENGTH);
const dec = createDecipheriv(ALGORITHM, key, iv);
dec.setAuthTag(tag);
return Buffer.concat([dec.update(ct), dec.final()]).toString("utf8");
const plaintext = Buffer.concat([dec.update(ct), dec.final()]);
return version === BACKUP_ENVELOPE_GZIP
? gunzipSync(plaintext).toString("utf8")
: plaintext.toString("utf8");
}
export interface S3Like {
@@ -215,12 +242,13 @@ export async function runOffhostBackup(
const evt = getEvent();
for (const user of users) {
try {
const { payload } = await buildFullBackupPayload(prisma, user.id, {
purpose: "disaster-recovery",
exportedAt: now,
});
const plaintext = JSON.stringify(payload);
const ciphertext = encryptBackup(plaintext, cfg.encryptionKey);
const ciphertext = encryptBackup(
await buildFullBackupJson(prisma, user.id, {
purpose: "disaster-recovery",
exportedAt: now,
}),
cfg.encryptionKey,
);
const key = `${dateKey}/user-${user.id}.json.enc`;
await s3.putObject(key, ciphertext);
uploaded++;
@@ -4,16 +4,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
buildFullBackupPayload: vi.fn(),
encrypt: vi.fn((value: string) => value),
packBlob: vi.fn((value: string) => value),
getWorkerPrisma: vi.fn(),
upsert: vi.fn(),
}));
// The handler takes the already-serialised form; the stand-in serialises
// whatever the payload mock was told to return, so this file keeps stubbing
// and asserting the PAYLOAD, which is what it is about.
vi.mock("@/lib/export/full-backup-payload", () => ({
buildFullBackupPayload: mocks.buildFullBackupPayload,
buildFullBackupJson: async (...args: unknown[]) =>
JSON.stringify(
((await mocks.buildFullBackupPayload(...args)) as { payload: unknown })
.payload,
),
}));
vi.mock("@/lib/crypto", () => ({ encrypt: mocks.encrypt }));
// The envelope is exercised end-to-end in
// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so
// the stored bytes can be read back as JSON.
vi.mock("@/lib/export/backup-blob", () => ({
packBackupBlob: mocks.packBlob,
}));
vi.mock("@/lib/logging/background", () => ({
withBackgroundEvent: vi.fn(
+18 -9
View File
@@ -6,8 +6,8 @@
*/
import { type Job } from "pg-boss";
import { recordError } from "@/lib/jobs/worker-status";
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { encrypt } from "@/lib/crypto";
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
import { packBackupBlob } from "@/lib/export/backup-blob";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import {
@@ -79,15 +79,21 @@ export async function handleDataBackup(
let backed = 0;
let usersFailed = 0;
let largestBlobBytes = 0;
for (const user of users) {
try {
const { payload } = await buildFullBackupPayload(prisma, user.id, {
purpose: "disaster-recovery",
});
const backupJson = JSON.stringify(payload);
// Encrypt the backup data (contains sensitive health information)
const encryptedBackup = encrypt(backupJson);
// Compressed, then encrypted (the record contains sensitive health
// information). Both legs live in `packBackupBlob`, which is also
// what the restore path reads back through.
const encryptedBackup = packBackupBlob(
await buildFullBackupJson(prisma, user.id, {
purpose: "disaster-recovery",
}),
);
largestBlobBytes = Math.max(
largestBlobBytes,
Buffer.byteLength(encryptedBackup, "utf8"),
);
await prisma.dataBackup.upsert({
where: {
@@ -113,6 +119,9 @@ export async function handleDataBackup(
}
}
// The stored size is the one number that says whether this pass is
// heading back towards the wall it hit before: it tracks the record.
evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes);
evt.setBackground({
task_name: "job.data_backup",
result: { backed, total: users.length },
@@ -138,6 +138,10 @@ import {
} from "@/lib/jobs/environment-fetch";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
DATA_BACKUP_QUEUE,
DATA_BACKUP_SEND_OPTIONS,
} from "@/lib/jobs/data-backup-policy";
import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared";
import {
createAndSchedule,
@@ -221,8 +225,6 @@ import {
type AchievementUnlockSweepPayload,
} from "@/lib/jobs/achievement-unlock-sweep";
const DATA_BACKUP_QUEUE = "data-backup";
const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00
const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup";
@@ -478,7 +480,11 @@ const schedules: ScheduleEntry[] = [
// v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload)
// that fans out one per-user environment fetch per opted-in account.
[ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON],
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON],
// The send options are the point of this tuple: without them the weekly
// snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on
// any sizeable record. See `data-backup-policy.ts` for why the window rides
// the schedule rather than the queue.
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS],
[RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry],
[IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry],
[AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON],
+7
View File
@@ -9,6 +9,7 @@
* both sides a route-handler-independent shared type home; the route
* keeps owning the HTTP contract, the component keeps owning the UI.
*/
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
export interface BackupRow {
id: string;
@@ -25,6 +26,12 @@ export interface BackupRow {
export interface BackupsList {
rows: BackupRow[];
/**
* Whether the weekly schedule is still producing copies, and how the last
* scheduled run ended. A row's own timestamp cannot answer that — a copy
* made six weeks ago and one made on Sunday look identical in a table.
*/
schedule: BackupScheduleStatus;
/**
* Soft retention hint — the worker is configured for weekly backups
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and