mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
Merge fix/backup-timeout
# Conflicts: # CHANGELOG.md # tests/integration/backup-round-trip.test.ts
This commit is contained in:
commit
b2a8edf65f
34 files changed
+1046
-62
No files matched your search
@@ -19,7 +19,7 @@ import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { parseBackupPayload } from "@/lib/validations/backup";
|
||||
|
||||
@@ -52,7 +52,7 @@ export const GET = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch (err) {
|
||||
// Decryption failure is rare but real — a rotated/missing key, or a
|
||||
// corrupted blob from a partially-failed write. Surface it as a 500
|
||||
|
||||
@@ -19,7 +19,7 @@ import { prisma, toJson } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { encryptNote } from "@/lib/crypto/note-cipher";
|
||||
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
|
||||
import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
|
||||
@@ -180,7 +180,7 @@ const handler = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch (err) {
|
||||
await auditLog("admin.backups.restore.failed", {
|
||||
userId: admin.id,
|
||||
|
||||
@@ -16,7 +16,7 @@ import { NextRequest } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess } from "@/lib/api-response";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
isCompatibleSchemaVersion,
|
||||
@@ -44,7 +44,7 @@ export const GET = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch {
|
||||
return apiError("Failed to decrypt backup payload", 500);
|
||||
}
|
||||
|
||||
@@ -5,11 +5,20 @@
|
||||
* shipped — only metadata (id, userId, username, type, size in bytes,
|
||||
* createdAt). The encrypted blob remains server-side; admins can trigger
|
||||
* a re-snapshot but not download another user's payload from the UI.
|
||||
*
|
||||
* Alongside the rows, the response says whether the SCHEDULE is still alive:
|
||||
* how old the newest scheduled copy is and how the last scheduled run ended.
|
||||
* Without those two the page cannot tell a working backup from one that
|
||||
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
||||
* timestamp either way.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
import { readLastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
|
||||
// so callers (in particular `components/admin/backups-section.tsx`) don't
|
||||
// have to reach across the component → route-handler layer boundary.
|
||||
@@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/** The type the weekly worker writes; anything else was uploaded by hand. */
|
||||
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
|
||||
|
||||
interface BackupMetaRow {
|
||||
id: string;
|
||||
user_id: string;
|
||||
username: string;
|
||||
type: string;
|
||||
size_bytes: number;
|
||||
created_at: Date;
|
||||
}
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
await requireAdmin();
|
||||
annotate({ action: { name: "admin.backups.list" } });
|
||||
|
||||
// Pull metadata only — `data` would balloon the response and reveal
|
||||
// ciphertext we don't need on the listing page.
|
||||
const backups = await prisma.dataBackup.findMany({
|
||||
include: { user: { select: { id: true, username: true } } },
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
// Metadata only, and the size read as `octet_length` in the database rather
|
||||
// than by pulling every blob into this process to measure it. The listing
|
||||
// needs a number, not the ciphertext, and on an instance with large records
|
||||
// fetching them all was its own way of running out of memory. Base64 is
|
||||
// ASCII, so the column's byte length is exactly what the wire reports.
|
||||
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
|
||||
SELECT b.id, b.user_id, u.username, b.type,
|
||||
octet_length(b.data) AS size_bytes,
|
||||
b.created_at
|
||||
FROM data_backups b
|
||||
JOIN users u ON u.id = b.user_id
|
||||
ORDER BY b.created_at DESC
|
||||
`;
|
||||
|
||||
const rows: BackupRow[] = backups.map((b) => ({
|
||||
id: b.id,
|
||||
userId: b.userId,
|
||||
username: b.user.username,
|
||||
type: b.type,
|
||||
sizeBytes: Buffer.byteLength(b.data, "utf8"),
|
||||
createdAt: b.createdAt.toISOString(),
|
||||
const list: BackupRow[] = rows.map((row) => ({
|
||||
id: row.id,
|
||||
userId: row.user_id,
|
||||
username: row.username,
|
||||
type: row.type,
|
||||
sizeBytes: Number(row.size_bytes),
|
||||
createdAt: row.created_at.toISOString(),
|
||||
}));
|
||||
|
||||
const payload: BackupsList = {
|
||||
rows,
|
||||
rows: list,
|
||||
// Matches the retention window the backup-prune job enforces so the
|
||||
// backups page states the same number the worker acts on.
|
||||
retentionDays: 90,
|
||||
schedule: summariseBackupSchedule({
|
||||
scheduledCreatedAt: rows
|
||||
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
|
||||
.map((row) => row.created_at),
|
||||
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
|
||||
now: new Date(),
|
||||
}),
|
||||
};
|
||||
|
||||
return apiSuccess(payload);
|
||||
|
||||
@@ -15,11 +15,13 @@ import { auditLog } from "@/lib/auth/audit";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
export const POST = apiHandler(async (request: NextRequest) => {
|
||||
const { user: admin } = await requireAdmin();
|
||||
annotate({ action: { name: "admin.backups.run" } });
|
||||
@@ -57,9 +59,14 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
throw new HttpError(503, "Background worker is not running");
|
||||
}
|
||||
|
||||
const jobId = await boss.send(DATA_BACKUP_QUEUE, {
|
||||
triggeredAt: new Date().toISOString(),
|
||||
});
|
||||
// Same expiration window as the weekly cron. An ad-hoc snapshot runs the
|
||||
// identical pass over the identical record, so it must not silently keep the
|
||||
// 15-minute queue default the cron was rescued from.
|
||||
const jobId = await boss.send(
|
||||
DATA_BACKUP_QUEUE,
|
||||
{ triggeredAt: new Date().toISOString() },
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
);
|
||||
|
||||
annotate({ meta: { job_id: jobId ?? null } });
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import {
|
||||
@@ -190,7 +190,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
// intact so multiple uploads can coexist for the same user without
|
||||
// overwriting each other or the rolling WEEKLY_AUTO snapshot.
|
||||
const uploadType = `MANUAL_UPLOAD_${Date.now()}`;
|
||||
const encrypted = encrypt(JSON.stringify(payload));
|
||||
const encrypted = packBackupBlob(JSON.stringify(payload));
|
||||
|
||||
const created = await prisma.dataBackup.create({
|
||||
data: {
|
||||
|
||||
@@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server";
|
||||
import de from "../../../../messages/de.json";
|
||||
import en from "../../../../messages/en.json";
|
||||
|
||||
const healthySchedule = {
|
||||
lastSuccessAt: "2026-09-01T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 2,
|
||||
staleAfterDays: 10,
|
||||
stale: false,
|
||||
lastRun: {
|
||||
at: "2026-09-01T03:00:12.000Z",
|
||||
state: "completed" as const,
|
||||
error: null,
|
||||
},
|
||||
lastRunFailed: false,
|
||||
};
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
mutationOptions: [] as Array<{
|
||||
onSuccess?: (data: { summary: Record<string, number> }) => void;
|
||||
}>,
|
||||
toastSuccess: vi.fn(),
|
||||
// Mutable so one file can render both a healthy schedule and a broken one;
|
||||
// the query mock reads it on every render.
|
||||
queryData: { rows: [], retentionDays: 30 } as Record<string, unknown>,
|
||||
}));
|
||||
|
||||
/**
|
||||
@@ -34,7 +50,7 @@ vi.mock("sonner", () => ({
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: () => ({
|
||||
data: { rows: [], retentionDays: 30 },
|
||||
data: mocks.queryData,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
}),
|
||||
@@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { BackupsSection } from "../backups-section";
|
||||
|
||||
function render(locale: "en" | "de" = "en") {
|
||||
mocks.queryData = {
|
||||
rows: [],
|
||||
retentionDays: 30,
|
||||
schedule: healthySchedule,
|
||||
};
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale={locale}>
|
||||
<BackupsSection />
|
||||
@@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The gap that let a weekly backup stop for a month and a half without anyone
|
||||
* noticing: the page rendered whatever rows it had, and a copy from six weeks
|
||||
* ago is indistinguishable from Sunday's until somebody subtracts the dates.
|
||||
*/
|
||||
describe("BackupsSection — schedule health", () => {
|
||||
function withSchedule(schedule: Record<string, unknown>) {
|
||||
mocks.queryData = { rows: [], retentionDays: 30, schedule };
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<BackupsSection />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
it("stays quiet while the schedule is healthy", () => {
|
||||
const html = render();
|
||||
expect(html).not.toContain('data-slot="backup-schedule-notice"');
|
||||
});
|
||||
|
||||
it("says so when the newest scheduled copy has gone stale", () => {
|
||||
const html = withSchedule({
|
||||
...healthySchedule,
|
||||
lastSuccessAt: "2026-07-19T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 46,
|
||||
stale: true,
|
||||
lastRun: null,
|
||||
lastRunFailed: false,
|
||||
});
|
||||
|
||||
expect(html).toContain('data-slot="backup-schedule-notice"');
|
||||
expect(html).toContain("The weekly backup has stopped");
|
||||
expect(html).toContain("46");
|
||||
});
|
||||
|
||||
it("names the reason when the last scheduled run failed", () => {
|
||||
const html = withSchedule({
|
||||
...healthySchedule,
|
||||
lastRun: {
|
||||
at: "2026-08-30T03:46:00.000Z",
|
||||
state: "failed",
|
||||
error: "job timed out",
|
||||
},
|
||||
lastRunFailed: true,
|
||||
});
|
||||
|
||||
expect(html).toContain("The last scheduled backup run failed");
|
||||
expect(html).toContain("job timed out");
|
||||
});
|
||||
|
||||
it("survives a response that predates the schedule field", () => {
|
||||
mocks.queryData = { rows: [], retentionDays: 30 };
|
||||
expect(() =>
|
||||
renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<BackupsSection />
|
||||
</I18nProvider>,
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("BackupsSection — upload outcome", () => {
|
||||
it("includes health profile fact revisions in the restored record total", () => {
|
||||
mocks.toastSuccess.mockClear();
|
||||
|
||||
@@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type {
|
||||
MissingBackupSection,
|
||||
RestoreSkipSummary,
|
||||
@@ -560,6 +561,71 @@ export function RestoreSkipReport({
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Says out loud whether the SCHEDULE is still alive.
|
||||
*
|
||||
* The weekly pass on the maintainer's own instance stopped producing copies
|
||||
* and nothing surfaced it for a month and a half: the table listed the rows it
|
||||
* had, each with an ordinary timestamp, and a copy from six weeks ago reads
|
||||
* exactly like one made on Sunday until somebody does the subtraction. Two
|
||||
* independent facts are shown, because they fail at different times — the age
|
||||
* of the newest scheduled copy never ages out, while the run's own outcome
|
||||
* carries the reason ("job timed out") and survives only as long as pg-boss
|
||||
* keeps the row.
|
||||
*
|
||||
* Not dismissible: it reflects live state, so the only way to clear it is to
|
||||
* make a backup succeed.
|
||||
*/
|
||||
function ScheduleHealthNotice({
|
||||
schedule,
|
||||
}: {
|
||||
schedule: BackupScheduleStatus;
|
||||
}) {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
if (!schedule.stale && !schedule.lastRunFailed) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
role="alert"
|
||||
data-slot="backup-schedule-notice"
|
||||
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" aria-hidden="true" />
|
||||
<div>
|
||||
<p className="font-medium">
|
||||
{schedule.stale
|
||||
? t("admin.section.backups.scheduleStaleTitle")
|
||||
: t("admin.section.backups.scheduleFailedTitle")}
|
||||
</p>
|
||||
{schedule.stale ? (
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.scheduleStaleDescription", {
|
||||
days: schedule.lastSuccessAgeDays ?? 0,
|
||||
threshold: schedule.staleAfterDays,
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
{schedule.lastRunFailed && schedule.lastRun ? (
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.scheduleFailedDescription", {
|
||||
when: fmt.dateTime(schedule.lastRun.at),
|
||||
reason:
|
||||
schedule.lastRun.error ||
|
||||
t("admin.section.backups.scheduleFailedNoReason"),
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
<p className="text-muted-foreground mt-2 text-xs">
|
||||
{t("admin.section.backups.scheduleRemedy")}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function BackupsSection() {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
@@ -833,6 +899,10 @@ export function BackupsSection() {
|
||||
}
|
||||
/>
|
||||
|
||||
{data?.schedule ? (
|
||||
<ScheduleHealthNotice schedule={data.schedule} />
|
||||
) : null}
|
||||
|
||||
{/* Upload card — separate from the table so admins can ingest a
|
||||
backup file independently of any existing rows. The visible
|
||||
button proxies a hidden file input so the layout stays clean
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* The stored-backup envelope: compress, then encrypt.
|
||||
*
|
||||
* The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to
|
||||
* Postgres. For an account with a few hundred thousand measurements that means
|
||||
* the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and
|
||||
* the copy the database driver makes of it all alive at once — well over a
|
||||
* gigabyte for a record whose JSON is a few hundred megabytes. The run died of
|
||||
* memory, not of slow SQL.
|
||||
*
|
||||
* Compressing before the cipher is what makes the tail of that pipeline cheap:
|
||||
* health JSON is extremely repetitive, so everything downstream of the gzip
|
||||
* shrinks by an order of magnitude. The envelope has to stay readable both
|
||||
* ways — rows written before this change are plain `encrypt(json)` and must
|
||||
* keep restoring.
|
||||
*/
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob, unpackBackupBlob } from "../backup-blob";
|
||||
|
||||
/** A record-shaped string: many rows, few distinct keys. */
|
||||
function sampleJson(rows: number): string {
|
||||
return JSON.stringify({
|
||||
schemaVersion: "2",
|
||||
measurements: Array.from({ length: rows }, (_, i) => ({
|
||||
id: `measurement-${i}`,
|
||||
type: "PULSE",
|
||||
value: 60 + (i % 30),
|
||||
unit: "bpm",
|
||||
measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(),
|
||||
source: "APPLE_HEALTH",
|
||||
deletedAt: null,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
describe("backup blob envelope", () => {
|
||||
it("round-trips the exact JSON it was handed", () => {
|
||||
const json = sampleJson(200);
|
||||
expect(unpackBackupBlob(packBackupBlob(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("stores a record-shaped payload far smaller than the plaintext", () => {
|
||||
const json = sampleJson(5_000);
|
||||
const packed = packBackupBlob(json);
|
||||
// The old envelope was 1.33× the JSON. Anything near that means the
|
||||
// compression leg is not running.
|
||||
expect(packed.length).toBeLessThan(json.length / 4);
|
||||
});
|
||||
|
||||
it("still reads a row written before the envelope existed", () => {
|
||||
const json = sampleJson(10);
|
||||
expect(unpackBackupBlob(encrypt(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("fails closed on a corrupt blob rather than returning junk", () => {
|
||||
const packed = packBackupBlob(sampleJson(10));
|
||||
const mangled = `${packed.slice(0, -8)}AAAAAAAA`;
|
||||
expect(() => unpackBackupBlob(mangled)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* The envelope a `DataBackup.data` row is stored in: compress, then encrypt.
|
||||
*
|
||||
* Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))`
|
||||
* straight to Postgres. For an account with a few hundred thousand
|
||||
* measurements that is four full copies of the record alive at the same
|
||||
* moment: the payload object graph, the JSON string, the base64 ciphertext at
|
||||
* 1.33× the JSON, and the copy the database driver makes of that parameter on
|
||||
* its way to the wire. Measured on a seeded 445 000-measurement account, the
|
||||
* JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap
|
||||
* it never reaches the insert, and on a bigger heap it only reaches it later.
|
||||
* The weekly pass was not slow; it could not finish.
|
||||
*
|
||||
* Compressing first is what makes the whole tail of that pipeline cheap. A
|
||||
* health record's JSON is extremely repetitive — the same twenty keys per row,
|
||||
* timestamps sharing a prefix — so gzip takes that 242 MB to 14 MB, and every
|
||||
* copy after it shrinks with it. The same run then completes in about eight
|
||||
* seconds with a peak heap of 434 MB. It is also the reason the stored row
|
||||
* stops being a liability of its own: the backup lives INSIDE the database it
|
||||
* would be needed to restore, so its size is not a cosmetic concern.
|
||||
*
|
||||
* Both directions. Rows written before this envelope existed are plain
|
||||
* `encrypt(json)` and have to keep restoring — an operator whose newest usable
|
||||
* copy predates the fix is exactly the person who needs it to work. The marker
|
||||
* sits inside the ciphertext rather than in front of it so the stored column
|
||||
* keeps the one shape `decrypt()` already understands, key ids and all.
|
||||
*/
|
||||
import { Buffer } from "node:buffer";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
|
||||
import { decrypt, encrypt } from "@/lib/crypto";
|
||||
|
||||
/**
|
||||
* Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd.
|
||||
* Chosen so it can never be mistaken for the alternative: a plain payload is
|
||||
* always a JSON object and therefore always starts with `{`.
|
||||
*/
|
||||
const GZIP_MARKER = "HLZ1:";
|
||||
|
||||
/** Serialised backup JSON → the string stored in `DataBackup.data`. */
|
||||
export function packBackupBlob(json: string): string {
|
||||
const compressed = gzipSync(json).toString("base64");
|
||||
return encrypt(`${GZIP_MARKER}${compressed}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A stored `DataBackup.data` string → the backup JSON.
|
||||
*
|
||||
* Fails closed on every arm: a bad key, a mangled ciphertext or a truncated
|
||||
* gzip member throws rather than returning a partial document, because every
|
||||
* caller goes on to parse the result as a whole account.
|
||||
*/
|
||||
export function unpackBackupBlob(stored: string): string {
|
||||
const plaintext = decrypt(stored);
|
||||
if (!plaintext.startsWith(GZIP_MARKER)) return plaintext;
|
||||
return gunzipSync(
|
||||
Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"),
|
||||
).toString("utf8");
|
||||
}
|
||||
@@ -316,6 +316,26 @@ function serialiseMoodContext(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The same payload, already serialised, with the object graph released.
|
||||
*
|
||||
* Its own frame on purpose, and both backup jobs go through it. The payload
|
||||
* graph is the largest thing either job allocates — on an account with a few
|
||||
* hundred thousand measurements it is several hundred megabytes of small
|
||||
* objects — and neither job wants anything but the string. Returning from here
|
||||
* makes the graph unreachable before the compress-and-encrypt step allocates
|
||||
* anything of its own, which is part of why the weekly pass now fits in
|
||||
* memory. Inlining these two statements back into a caller quietly undoes it.
|
||||
*/
|
||||
export async function buildFullBackupJson(
|
||||
prisma: PrismaClient,
|
||||
userId: string,
|
||||
options: FullBackupOptions = {},
|
||||
): Promise<string> {
|
||||
const { payload } = await buildFullBackupPayload(prisma, userId, options);
|
||||
return JSON.stringify(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the canonical full-backup payload for `userId`. Portable exports omit
|
||||
* tombstones and document ciphertext; disaster-recovery payloads preserve
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* What the backups page has to be able to say out loud.
|
||||
*
|
||||
* The weekly snapshot stopped producing a new copy and nothing surfaced it: the
|
||||
* page listed the rows it had, each with a perfectly ordinary timestamp, and a
|
||||
* copy that was a month and a half old looked exactly like one made on Sunday.
|
||||
* The two facts that would have shown it are the age of the newest scheduled
|
||||
* copy and the terminal state of the last scheduled run.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
BACKUP_STALE_AFTER_DAYS,
|
||||
summariseBackupSchedule,
|
||||
} from "../backup-schedule-status";
|
||||
|
||||
const NOW = new Date("2026-09-03T09:00:00.000Z");
|
||||
const daysAgo = (days: number) =>
|
||||
new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000);
|
||||
|
||||
describe("summariseBackupSchedule", () => {
|
||||
it("reports a fresh weekly copy as neither stale nor failed", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(2), daysAgo(9)],
|
||||
lastRun: {
|
||||
at: daysAgo(2).toISOString(),
|
||||
state: "completed",
|
||||
error: null,
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString());
|
||||
expect(status.lastSuccessAgeDays).toBe(2);
|
||||
expect(status.stale).toBe(false);
|
||||
expect(status.lastRunFailed).toBe(false);
|
||||
expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS);
|
||||
});
|
||||
|
||||
it("calls a copy older than the window stale", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(46)],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.lastSuccessAgeDays).toBe(46);
|
||||
expect(status.stale).toBe(true);
|
||||
// No run row survives that long — the age is the durable signal.
|
||||
expect(status.lastRunFailed).toBe(false);
|
||||
});
|
||||
|
||||
it("surfaces a failed last run even while the copy is still in window", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(3)],
|
||||
lastRun: {
|
||||
at: daysAgo(1).toISOString(),
|
||||
state: "failed",
|
||||
error: "job timed out",
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.stale).toBe(false);
|
||||
expect(status.lastRunFailed).toBe(true);
|
||||
expect(status.lastRun?.error).toBe("job timed out");
|
||||
});
|
||||
|
||||
it("does not call a brand-new instance stale", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.lastSuccessAt).toBeNull();
|
||||
expect(status.lastSuccessAgeDays).toBeNull();
|
||||
expect(status.stale).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores manual uploads when judging the scheduled cadence", () => {
|
||||
// The caller passes only WEEKLY_AUTO rows; a manual upload made today must
|
||||
// not make a dead cron look alive. Proven here by the shape of the input:
|
||||
// the summary knows nothing but scheduled copies.
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(30)],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
});
|
||||
expect(status.stale).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Send-policy pins for the weekly `data-backup` queue.
|
||||
*
|
||||
* The queue was registered with no send options at all, so every job it minted
|
||||
* inherited pg-boss's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`.
|
||||
* A full-record snapshot of a large account does not fit in fifteen minutes, so
|
||||
* the pass was killed mid-run, redelivered twice against the same record, and
|
||||
* gave up roughly 46 minutes after it started. Weekly, silently, for as long as
|
||||
* the record stayed large.
|
||||
*
|
||||
* Three facts have to hold together or the window is not real:
|
||||
*
|
||||
* 1. the policy names a window well past the 15-minute default,
|
||||
* 2. the CRON SCHEDULE carries it — send options ride the schedule because
|
||||
* pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a
|
||||
* queue-level default would be inert on every already-running instance,
|
||||
* 3. the admin "run now" route sends with the same options, or an ad-hoc
|
||||
* snapshot silently keeps the default that broke the cron one.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "../data-backup-policy";
|
||||
|
||||
const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8");
|
||||
|
||||
describe("data-backup — pg-boss send policy", () => {
|
||||
it("widens the expiration far past the 15-minute queue default", () => {
|
||||
expect(DATA_BACKUP_QUEUE).toBe("data-backup");
|
||||
// 900 s is the pg-boss 12.26 queue default (`QUEUE_DEFAULTS.expire_seconds`
|
||||
// in `dist/plans.js`); anything at or below it re-opens the bug.
|
||||
expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThan(900);
|
||||
expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThanOrEqual(
|
||||
60 * 60,
|
||||
);
|
||||
});
|
||||
|
||||
it("the weekly schedule carries the policy", () => {
|
||||
const source = read("src/lib/jobs/reminder/register-maintenance.ts");
|
||||
expect(source).toMatch(
|
||||
/\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/,
|
||||
);
|
||||
});
|
||||
|
||||
it("the admin run-now route sends with the same policy", () => {
|
||||
const source = read("src/app/api/admin/backups/run/route.ts");
|
||||
expect(source).toMatch(
|
||||
/boss\.send\(\s*DATA_BACKUP_QUEUE,[\s\S]{0,200}DATA_BACKUP_SEND_OPTIONS,?\s*\)/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -26,4 +26,16 @@ describe("pg-boss failed-row ledger retention", () => {
|
||||
PG_BOSS_FAILED_ROW_AVAILABILITY_HOURS,
|
||||
);
|
||||
});
|
||||
|
||||
it("pins the queue defaults the data-backup window had to escape", () => {
|
||||
const plans = readFileSync(join(PG_BOSS_ROOT, "dist/plans.js"), "utf8");
|
||||
|
||||
// These two lines are the whole story behind the weekly backup's "job
|
||||
// timed out": a pass with no send options gets fifteen minutes and two
|
||||
// redeliveries, so a snapshot that cannot finish in fifteen minutes gives
|
||||
// up 45 minutes later having never once completed. If a pg-boss upgrade
|
||||
// changes either default, `DATA_BACKUP_SEND_OPTIONS` deserves a re-read.
|
||||
expect(plans).toContain("expire_seconds: FIFTEEN_MINUTES");
|
||||
expect(plans).toContain("retry_limit: 2");
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createCipheriv, randomBytes } from "node:crypto";
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
import {
|
||||
BACKUP_SCHEMA_VERSION,
|
||||
@@ -8,8 +10,20 @@ const mocks = vi.hoisted(() => ({
|
||||
buildFullBackupPayload: vi.fn(),
|
||||
}));
|
||||
|
||||
// The uploader takes the already-serialised form, so the stand-in for
|
||||
// `buildFullBackupJson` serialises whatever the payload mock was told to
|
||||
// return. Every case below keeps stubbing the PAYLOAD, which is the thing
|
||||
// these tests are actually about.
|
||||
vi.mock("@/lib/export/full-backup-payload", () => ({
|
||||
buildFullBackupPayload: mocks.buildFullBackupPayload,
|
||||
buildFullBackupJson: async (...args: unknown[]) =>
|
||||
JSON.stringify(
|
||||
(
|
||||
(await mocks.buildFullBackupPayload(...args)) as {
|
||||
payload: unknown;
|
||||
}
|
||||
).payload,
|
||||
),
|
||||
}));
|
||||
import {
|
||||
encryptBackup,
|
||||
@@ -27,10 +41,33 @@ describe("offhost-backup envelope", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const payload = JSON.stringify({ hello: "world", n: 42 });
|
||||
const buf = encryptBackup(payload, key);
|
||||
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x01");
|
||||
// Version 2: the plaintext is gzipped before the cipher sees it. This
|
||||
// assertion used to pin version 1, which is the version that could not
|
||||
// encrypt a large account's dump without four full copies of it resident
|
||||
// at once. The pin is flipped rather than dropped — the byte is the whole
|
||||
// reason a reader can tell the two apart.
|
||||
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x02");
|
||||
expect(decryptBackup(buf, key)).toBe(payload);
|
||||
});
|
||||
|
||||
it("still reads a version-1 object written before compression", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const payload = JSON.stringify({ hello: "world", n: 42 });
|
||||
// Hand-built v1 envelope: magic || 0x01 || iv || tag || ciphertext, the
|
||||
// exact bytes every object already sitting in an operator's bucket has.
|
||||
const iv = randomBytes(12);
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
||||
const ct = Buffer.concat([cipher.update(payload, "utf8"), cipher.final()]);
|
||||
const legacy = Buffer.concat([
|
||||
Buffer.from("HLBK\x01", "binary"),
|
||||
iv,
|
||||
cipher.getAuthTag(),
|
||||
ct,
|
||||
]);
|
||||
|
||||
expect(decryptBackup(legacy, key)).toBe(payload);
|
||||
});
|
||||
|
||||
it("rejects tampered ciphertext", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const buf = encryptBackup("data", key);
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Whether the weekly backup is actually still happening, in two facts.
|
||||
*
|
||||
* The pass stopped producing a new copy and nothing said so for a month and a
|
||||
* half. Everything that could have told the operator was either invisible or
|
||||
* short-lived: the handler's own error never ran (the job was killed, not
|
||||
* thrown), `recordError()` increments an unnamed counter, and the admin status
|
||||
* page's failing-queue panel only looks back 72 hours — which for a weekly
|
||||
* queue means a permanent failure is on screen three days in seven. The backups
|
||||
* page itself showed a perfectly ordinary row with a perfectly ordinary
|
||||
* timestamp, because a 46-day-old copy looks exactly like a fresh one until you
|
||||
* read the date.
|
||||
*
|
||||
* So the surface carries both legs, and neither is derived from the other:
|
||||
*
|
||||
* - the AGE of the newest scheduled copy, which never ages out of any
|
||||
* retention window and would have caught this on the second missed Sunday;
|
||||
* - the terminal state of the last scheduled RUN, which is the only place the
|
||||
* reason ("job timed out") is written down at all.
|
||||
*
|
||||
* Manual uploads are excluded by the shape of the input: the caller passes only
|
||||
* the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look
|
||||
* alive.
|
||||
*/
|
||||
import type { LastQueueRun } from "@/lib/jobs/job-failures";
|
||||
|
||||
/**
|
||||
* How old the newest scheduled copy may get before the page says so. The cron
|
||||
* is weekly, so seven days is normal and eight is a rounding error; ten means
|
||||
* at least one Sunday produced nothing.
|
||||
*/
|
||||
export const BACKUP_STALE_AFTER_DAYS = 10;
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
export interface BackupScheduleStatus {
|
||||
/** ISO instant of the newest scheduled copy, or null when there is none. */
|
||||
lastSuccessAt: string | null;
|
||||
/** Whole days since that copy was written. */
|
||||
lastSuccessAgeDays: number | null;
|
||||
/** The threshold behind `stale`, so the UI can name it. */
|
||||
staleAfterDays: number;
|
||||
/** A scheduled copy exists and is older than the threshold. */
|
||||
stale: boolean;
|
||||
/** The last scheduled run pg-boss still remembers. */
|
||||
lastRun: LastQueueRun | null;
|
||||
/** That run ended `failed` or `cancelled`. */
|
||||
lastRunFailed: boolean;
|
||||
}
|
||||
|
||||
export function summariseBackupSchedule(args: {
|
||||
/** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */
|
||||
scheduledCreatedAt: readonly Date[];
|
||||
lastRun: LastQueueRun | null;
|
||||
now: Date;
|
||||
}): BackupScheduleStatus {
|
||||
const newest = args.scheduledCreatedAt.reduce<Date | null>(
|
||||
(best, at) => (best === null || at > best ? at : best),
|
||||
null,
|
||||
);
|
||||
const ageDays =
|
||||
newest === null
|
||||
? null
|
||||
: Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS);
|
||||
|
||||
return {
|
||||
lastSuccessAt: newest?.toISOString() ?? null,
|
||||
lastSuccessAgeDays: ageDays,
|
||||
staleAfterDays: BACKUP_STALE_AFTER_DAYS,
|
||||
// An instance that has never produced a scheduled copy is not stale, it is
|
||||
// new — the page's empty state already says there is nothing there, and
|
||||
// calling a fresh install "overdue" would train the operator to ignore the
|
||||
// warning that matters.
|
||||
stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS,
|
||||
lastRun: args.lastRun,
|
||||
lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Queue name and pg-boss send policy for the weekly on-host data backup.
|
||||
*
|
||||
* A leaf module on purpose: the cron registrar, the admin "run now" route and
|
||||
* the policy test all need these two facts, and none of them should have to
|
||||
* pull in the worker's handler graph to get them.
|
||||
*
|
||||
* The queue was registered with no send options, so every job it minted took
|
||||
* pg-boss 12.26's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`
|
||||
* (`QUEUE_DEFAULTS` in `pg-boss/dist/plans.js`). Fifteen minutes is a fine
|
||||
* default for a cleanup tick and far too short for a pass that serialises every
|
||||
* account's whole record: the job was killed mid-run, redelivered twice against
|
||||
* the same record, and reached `failed` with `job timed out` about 46 minutes
|
||||
* after it started. Nothing about that says "too slow" — the pass never got to
|
||||
* finish once.
|
||||
*
|
||||
* Two hours is deliberately generous rather than tuned. The handler now costs a
|
||||
* fraction of that on a large record (see the CHANGELOG entry), but the pass
|
||||
* loops over EVERY account on the instance and a self-host's disk is whatever
|
||||
* it is; the window's job is to be impossible to hit for the right reasons, and
|
||||
* a failure that arrives at the two-hour mark is a real one worth reading.
|
||||
* pg-boss refuses anything at or past 24 hours.
|
||||
*
|
||||
* The default `retryLimit: 2` stays. The old redeliveries were pointless
|
||||
* because the first attempt could not succeed either; a transient failure — a
|
||||
* pool hiccup, a locked row — genuinely benefits from a second attempt, and the
|
||||
* alternative is losing the week.
|
||||
*
|
||||
* The options ride on the SEND, not on `createQueue`: pg-boss's `create_queue()`
|
||||
* ends in `ON CONFLICT DO NOTHING`, so a queue-level default would apply to a
|
||||
* fresh database and be inert on every instance that already has the queue —
|
||||
* exactly the trap `reconcileQueuePolicies` exists to work around.
|
||||
*/
|
||||
|
||||
export const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
export const DATA_BACKUP_SEND_OPTIONS = {
|
||||
expireInSeconds: 2 * 60 * 60,
|
||||
} as const;
|
||||
@@ -98,7 +98,11 @@ export async function readFailingQueues(
|
||||
max(completed_on) AS last_failed_at,
|
||||
(
|
||||
array_agg(
|
||||
COALESCE(output->>'message', output::text)
|
||||
COALESCE(
|
||||
output->>'message',
|
||||
output->'value'->>'message',
|
||||
output::text
|
||||
)
|
||||
ORDER BY completed_on DESC NULLS LAST
|
||||
)
|
||||
)[1] AS last_error
|
||||
@@ -124,6 +128,75 @@ export async function readFailingQueues(
|
||||
}
|
||||
}
|
||||
|
||||
/** The terminal state pg-boss recorded for the newest run of one queue. */
|
||||
export interface LastQueueRun {
|
||||
/** ISO instant the run reached its terminal state. */
|
||||
at: string;
|
||||
/** `failed` covers both a thrown handler and an expired (killed) job. */
|
||||
state: "completed" | "failed" | "cancelled";
|
||||
/** The failure text, redacted and truncated. Null when the run succeeded. */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The newest terminal run of one queue, whatever its outcome.
|
||||
*
|
||||
* `readFailingQueues` answers "what broke recently" and is deliberately scoped
|
||||
* to a 72-hour window, which is the right shape for a status page watching
|
||||
* thirty queues that tick by the minute. It is the wrong shape for a WEEKLY
|
||||
* pass: a Sunday-night failure has fallen out of the window by Wednesday, so a
|
||||
* job that fails every single week is visible for three days in seven and
|
||||
* invisible for four. This reader takes the newest run instead of a window, so
|
||||
* a weekly queue's last outcome is readable right up until the next one.
|
||||
*
|
||||
* Bounded by pg-boss's own retention: terminal rows live `deletion_seconds`
|
||||
* (seven days by default) past completion, so `null` here means "no run row
|
||||
* survives", which is not the same as "the pass never ran". Callers pair it
|
||||
* with a fact that does not age out — for backups, the age of the newest
|
||||
* stored copy.
|
||||
*/
|
||||
export async function readLastQueueRun(
|
||||
queue: string,
|
||||
): Promise<LastQueueRun | null> {
|
||||
try {
|
||||
const rows = await prisma.$queryRaw<
|
||||
Array<{ state: string; completed_on: Date | null; output: string | null }>
|
||||
>`
|
||||
SELECT
|
||||
state,
|
||||
completed_on,
|
||||
-- pg-boss wraps a thrown value: an expired job's row reads
|
||||
-- {"value":{"message":"job timed out"}}, so a plain ->>'message'
|
||||
-- misses it and the operator gets raw JSON instead of the sentence.
|
||||
COALESCE(
|
||||
output->>'message',
|
||||
output->'value'->>'message',
|
||||
output::text
|
||||
) AS output
|
||||
FROM pgboss.job
|
||||
WHERE name = ${queue}
|
||||
AND state IN ('completed', 'failed', 'cancelled')
|
||||
AND completed_on IS NOT NULL
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
const row = rows[0];
|
||||
if (!row || row.completed_on === null) return null;
|
||||
const state =
|
||||
row.state === "failed" || row.state === "cancelled"
|
||||
? row.state
|
||||
: "completed";
|
||||
return {
|
||||
at: row.completed_on.toISOString(),
|
||||
state,
|
||||
error: state === "completed" ? null : presentError(row.output),
|
||||
};
|
||||
} catch {
|
||||
// No `pgboss` schema (web-only deployment) or no permission to read it.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface QueueFailureForUser {
|
||||
/** ISO instant the user's newest job on this queue gave up. */
|
||||
lastFailedAt: string;
|
||||
|
||||
@@ -18,8 +18,9 @@
|
||||
* the backup history. See docs/ops/backup-restore.md.
|
||||
*/
|
||||
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
@@ -84,20 +85,43 @@ export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
* Encrypt a JSON payload with the dedicated backup key.
|
||||
*
|
||||
* Wire format (binary):
|
||||
* magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext
|
||||
* magic(4)="HLBK" || version(1) || iv(12) || tag(16) || ciphertext
|
||||
*
|
||||
* Version 1 encrypts the JSON directly. Version 2 gzips it first, which is
|
||||
* what the version byte was there for: a large account's dump is hundreds of
|
||||
* megabytes of extremely repetitive JSON, and encrypting it whole means the
|
||||
* string, the cipher's input copy, the ciphertext and the concatenation are
|
||||
* all resident at once — the same arithmetic that made the in-database weekly
|
||||
* backup unfinishable. Gzip takes an order of magnitude off every one of them
|
||||
* and off the object in the bucket. Objects written under version 1 stay
|
||||
* readable; both the restore drill and `scripts/restore-backup.ts` come
|
||||
* through `decryptBackup`, so neither needs to know which it got.
|
||||
*/
|
||||
const BACKUP_ENVELOPE_PLAIN = 0x01;
|
||||
const BACKUP_ENVELOPE_GZIP = 0x02;
|
||||
|
||||
export function encryptBackup(plaintext: string, key: Buffer): Buffer {
|
||||
const iv = randomBytes(IV_LENGTH);
|
||||
const cipher = createCipheriv(ALGORITHM, key, iv);
|
||||
const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
||||
const ct = Buffer.concat([
|
||||
cipher.update(gzipSync(plaintext)),
|
||||
cipher.final(),
|
||||
]);
|
||||
const tag = cipher.getAuthTag();
|
||||
return Buffer.concat([Buffer.from("HLBK\x01", "binary"), iv, tag, ct]);
|
||||
const header = Buffer.from([
|
||||
...Buffer.from("HLBK", "binary"),
|
||||
BACKUP_ENVELOPE_GZIP,
|
||||
]);
|
||||
return Buffer.concat([header, iv, tag, ct]);
|
||||
}
|
||||
|
||||
export function decryptBackup(buf: Buffer, key: Buffer): string {
|
||||
const magic = buf.subarray(0, 4).toString("binary");
|
||||
const version = buf[4];
|
||||
if (magic !== "HLBK" || version !== 0x01) {
|
||||
if (
|
||||
magic !== "HLBK" ||
|
||||
(version !== BACKUP_ENVELOPE_PLAIN && version !== BACKUP_ENVELOPE_GZIP)
|
||||
) {
|
||||
throw new Error("Invalid backup envelope (bad magic or version)");
|
||||
}
|
||||
const iv = buf.subarray(5, 5 + IV_LENGTH);
|
||||
@@ -105,7 +129,10 @@ export function decryptBackup(buf: Buffer, key: Buffer): string {
|
||||
const ct = buf.subarray(5 + IV_LENGTH + TAG_LENGTH);
|
||||
const dec = createDecipheriv(ALGORITHM, key, iv);
|
||||
dec.setAuthTag(tag);
|
||||
return Buffer.concat([dec.update(ct), dec.final()]).toString("utf8");
|
||||
const plaintext = Buffer.concat([dec.update(ct), dec.final()]);
|
||||
return version === BACKUP_ENVELOPE_GZIP
|
||||
? gunzipSync(plaintext).toString("utf8")
|
||||
: plaintext.toString("utf8");
|
||||
}
|
||||
|
||||
export interface S3Like {
|
||||
@@ -215,12 +242,13 @@ export async function runOffhostBackup(
|
||||
const evt = getEvent();
|
||||
for (const user of users) {
|
||||
try {
|
||||
const { payload } = await buildFullBackupPayload(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
});
|
||||
const plaintext = JSON.stringify(payload);
|
||||
const ciphertext = encryptBackup(plaintext, cfg.encryptionKey);
|
||||
const ciphertext = encryptBackup(
|
||||
await buildFullBackupJson(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
}),
|
||||
cfg.encryptionKey,
|
||||
);
|
||||
const key = `${dateKey}/user-${user.id}.json.enc`;
|
||||
await s3.putObject(key, ciphertext);
|
||||
uploaded++;
|
||||
|
||||
@@ -4,16 +4,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
buildFullBackupPayload: vi.fn(),
|
||||
encrypt: vi.fn((value: string) => value),
|
||||
packBlob: vi.fn((value: string) => value),
|
||||
getWorkerPrisma: vi.fn(),
|
||||
upsert: vi.fn(),
|
||||
}));
|
||||
|
||||
// The handler takes the already-serialised form; the stand-in serialises
|
||||
// whatever the payload mock was told to return, so this file keeps stubbing
|
||||
// and asserting the PAYLOAD, which is what it is about.
|
||||
vi.mock("@/lib/export/full-backup-payload", () => ({
|
||||
buildFullBackupPayload: mocks.buildFullBackupPayload,
|
||||
buildFullBackupJson: async (...args: unknown[]) =>
|
||||
JSON.stringify(
|
||||
((await mocks.buildFullBackupPayload(...args)) as { payload: unknown })
|
||||
.payload,
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: mocks.encrypt }));
|
||||
// The envelope is exercised end-to-end in
|
||||
// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so
|
||||
// the stored bytes can be read back as JSON.
|
||||
vi.mock("@/lib/export/backup-blob", () => ({
|
||||
packBackupBlob: mocks.packBlob,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/background", () => ({
|
||||
withBackgroundEvent: vi.fn(
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
*/
|
||||
import { type Job } from "pg-boss";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import {
|
||||
@@ -79,15 +79,21 @@ export async function handleDataBackup(
|
||||
|
||||
let backed = 0;
|
||||
let usersFailed = 0;
|
||||
let largestBlobBytes = 0;
|
||||
for (const user of users) {
|
||||
try {
|
||||
const { payload } = await buildFullBackupPayload(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
});
|
||||
const backupJson = JSON.stringify(payload);
|
||||
|
||||
// Encrypt the backup data (contains sensitive health information)
|
||||
const encryptedBackup = encrypt(backupJson);
|
||||
// Compressed, then encrypted (the record contains sensitive health
|
||||
// information). Both legs live in `packBackupBlob`, which is also
|
||||
// what the restore path reads back through.
|
||||
const encryptedBackup = packBackupBlob(
|
||||
await buildFullBackupJson(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
}),
|
||||
);
|
||||
largestBlobBytes = Math.max(
|
||||
largestBlobBytes,
|
||||
Buffer.byteLength(encryptedBackup, "utf8"),
|
||||
);
|
||||
|
||||
await prisma.dataBackup.upsert({
|
||||
where: {
|
||||
@@ -113,6 +119,9 @@ export async function handleDataBackup(
|
||||
}
|
||||
}
|
||||
|
||||
// The stored size is the one number that says whether this pass is
|
||||
// heading back towards the wall it hit before: it tracks the record.
|
||||
evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes);
|
||||
evt.setBackground({
|
||||
task_name: "job.data_backup",
|
||||
result: { backed, total: users.length },
|
||||
|
||||
@@ -138,6 +138,10 @@ import {
|
||||
} from "@/lib/jobs/environment-fetch";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared";
|
||||
import {
|
||||
createAndSchedule,
|
||||
@@ -221,8 +225,6 @@ import {
|
||||
type AchievementUnlockSweepPayload,
|
||||
} from "@/lib/jobs/achievement-unlock-sweep";
|
||||
|
||||
const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00
|
||||
|
||||
const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup";
|
||||
@@ -478,7 +480,11 @@ const schedules: ScheduleEntry[] = [
|
||||
// v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload)
|
||||
// that fans out one per-user environment fetch per opted-in account.
|
||||
[ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON],
|
||||
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON],
|
||||
// The send options are the point of this tuple: without them the weekly
|
||||
// snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on
|
||||
// any sizeable record. See `data-backup-policy.ts` for why the window rides
|
||||
// the schedule rather than the queue.
|
||||
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS],
|
||||
[RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry],
|
||||
[IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry],
|
||||
[AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON],
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
* both sides a route-handler-independent shared type home; the route
|
||||
* keeps owning the HTTP contract, the component keeps owning the UI.
|
||||
*/
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
|
||||
export interface BackupRow {
|
||||
id: string;
|
||||
@@ -25,6 +26,12 @@ export interface BackupRow {
|
||||
|
||||
export interface BackupsList {
|
||||
rows: BackupRow[];
|
||||
/**
|
||||
* Whether the weekly schedule is still producing copies, and how the last
|
||||
* scheduled run ended. A row's own timestamp cannot answer that — a copy
|
||||
* made six weeks ago and one made on Sunday look identical in a table.
|
||||
*/
|
||||
schedule: BackupScheduleStatus;
|
||||
/**
|
||||
* Soft retention hint — the worker is configured for weekly backups
|
||||
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
|
||||
|
||||
Reference in new issue
Block a user