mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 05:12:14 +00:00
Merge fix/backup-timeout
# Conflicts: # CHANGELOG.md # tests/integration/backup-round-trip.test.ts
This commit is contained in:
commit
b2a8edf65f
34 files changed
+1046
-62
No files matched your search
@@ -14,6 +14,30 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A large record no longer outruns the weekly backup.** On an account with a
|
||||
few hundred thousand readings the Sunday snapshot could not finish. It was
|
||||
never given a time limit of its own, so it inherited the queue's fifteen
|
||||
minutes, was killed part-way through, redelivered twice against the same
|
||||
record, and gave up three quarters of an hour later having never completed
|
||||
once. The reason it needed so long was memory: the whole account was built in
|
||||
memory, serialised whole, encrypted whole, and handed to the database whole,
|
||||
which is four full copies of the record at the same moment. The stored copy
|
||||
is now compressed before it is encrypted, which takes an order of magnitude
|
||||
off everything after it, and the pass releases each stage before allocating
|
||||
the next. A seeded 445 000-reading account went from dying of memory to
|
||||
finishing in about twenty seconds, with a stored copy of 25 MB instead of
|
||||
322 MB. The pass also has an explicit two-hour window now, so a genuinely
|
||||
slow disk cannot be mistaken for a broken run. The nightly off-host copy is
|
||||
built the same way and got the same treatment; files written before this
|
||||
still restore, and so do backups already sitting in the database.
|
||||
- **A failed backup run is now visible on the backups page.** The weekly pass
|
||||
can stop and leave no trace: the page listed whatever copies existed, and a
|
||||
copy from six weeks ago carries a timestamp exactly like Sunday's. The page
|
||||
now says how old the newest scheduled copy is when it has gone past due, and
|
||||
how the last scheduled run ended when it ended badly, with the reason the
|
||||
queue recorded. Both are shown because they stop being available at different
|
||||
times — a run's own record is kept for a week, the age of the newest copy for
|
||||
as long as the copy exists.
|
||||
- **Mood correlations no longer call five days of noise a strong finding.** The
|
||||
mood page painted a "Strong" or "Moderate" badge on a mood-versus-sleep,
|
||||
activity, pulse, weight or blood-pressure card as soon as five days had been
|
||||
|
||||
@@ -6923,6 +6923,12 @@
|
||||
"colCreatedAt": "Letzter Snapshot",
|
||||
"colActions": "Aktionen",
|
||||
"retentionLabel": "Aufbewahrung: {days} Tage",
|
||||
"scheduleStaleTitle": "Die wöchentliche Sicherung läuft nicht mehr",
|
||||
"scheduleStaleDescription": "Die neueste geplante Kopie ist {days} Tage alt. Ein funktionierender wöchentlicher Lauf lässt keine älter als {threshold} Tage werden.",
|
||||
"scheduleFailedTitle": "Der letzte geplante Sicherungslauf ist fehlgeschlagen",
|
||||
"scheduleFailedDescription": "{when}: {reason}",
|
||||
"scheduleFailedNoReason": "kein Grund aufgezeichnet",
|
||||
"scheduleRemedy": "Starte jetzt eine Sicherung und prüfe, ob eine frische Zeile erscheint. Schlägt sie erneut fehl, nennt das Worker-Protokoll das Konto, an dem sie stehen geblieben ist.",
|
||||
"download": "Herunterladen",
|
||||
"downloadAria": "Backup für {username} herunterladen",
|
||||
"downloadFailed": "Download fehlgeschlagen",
|
||||
|
||||
@@ -6923,6 +6923,12 @@
|
||||
"colCreatedAt": "Last snapshot",
|
||||
"colActions": "Actions",
|
||||
"retentionLabel": "Retention: {days} days",
|
||||
"scheduleStaleTitle": "The weekly backup has stopped",
|
||||
"scheduleStaleDescription": "The newest scheduled copy is {days} days old. A working weekly run never leaves one older than {threshold} days.",
|
||||
"scheduleFailedTitle": "The last scheduled backup run failed",
|
||||
"scheduleFailedDescription": "{when}: {reason}",
|
||||
"scheduleFailedNoReason": "no reason recorded",
|
||||
"scheduleRemedy": "Run a backup now and check that a fresh row appears. If it fails again, the worker log names the account it stopped on.",
|
||||
"download": "Download",
|
||||
"downloadAria": "Download backup for {username}",
|
||||
"downloadFailed": "Download failed",
|
||||
|
||||
@@ -6923,6 +6923,12 @@
|
||||
"colCreatedAt": "Última instantánea",
|
||||
"colActions": "Acciones",
|
||||
"retentionLabel": "Conservación: {days} días",
|
||||
"scheduleStaleTitle": "La copia de seguridad semanal se ha detenido",
|
||||
"scheduleStaleDescription": "La copia programada más reciente tiene {days} días. Una ejecución semanal que funciona nunca deja ninguna con más de {threshold} días.",
|
||||
"scheduleFailedTitle": "La última ejecución programada de la copia de seguridad falló",
|
||||
"scheduleFailedDescription": "{when}: {reason}",
|
||||
"scheduleFailedNoReason": "sin motivo registrado",
|
||||
"scheduleRemedy": "Ejecuta una copia ahora y comprueba que aparece una fila nueva. Si vuelve a fallar, el registro del worker indica la cuenta en la que se detuvo.",
|
||||
"download": "Descargar",
|
||||
"downloadAria": "Descargar copia de seguridad de {username}",
|
||||
"downloadFailed": "La descarga falló",
|
||||
|
||||
@@ -6923,6 +6923,12 @@
|
||||
"colCreatedAt": "Dernier instantané",
|
||||
"colActions": "Actions",
|
||||
"retentionLabel": "Conservation : {days} jours",
|
||||
"scheduleStaleTitle": "La sauvegarde hebdomadaire s'est arrêtée",
|
||||
"scheduleStaleDescription": "La copie planifiée la plus récente date de {days} jours. Une exécution hebdomadaire qui fonctionne n'en laisse jamais une dépasser {threshold} jours.",
|
||||
"scheduleFailedTitle": "La dernière exécution planifiée de la sauvegarde a échoué",
|
||||
"scheduleFailedDescription": "{when} : {reason}",
|
||||
"scheduleFailedNoReason": "aucune raison enregistrée",
|
||||
"scheduleRemedy": "Lancez une sauvegarde maintenant et vérifiez qu'une nouvelle ligne apparaît. Si elle échoue à nouveau, le journal du worker indique le compte sur lequel elle s'est arrêtée.",
|
||||
"download": "Télécharger",
|
||||
"downloadAria": "Télécharger la sauvegarde de {username}",
|
||||
"downloadFailed": "Échec du téléchargement",
|
||||
|
||||
@@ -6923,6 +6923,12 @@
|
||||
"colCreatedAt": "Ultima istantanea",
|
||||
"colActions": "Azioni",
|
||||
"retentionLabel": "Conservazione: {days} giorni",
|
||||
"scheduleStaleTitle": "Il backup settimanale si è fermato",
|
||||
"scheduleStaleDescription": "La copia pianificata più recente ha {days} giorni. Un'esecuzione settimanale funzionante non ne lascia mai una più vecchia di {threshold} giorni.",
|
||||
"scheduleFailedTitle": "L'ultima esecuzione pianificata del backup non è riuscita",
|
||||
"scheduleFailedDescription": "{when}: {reason}",
|
||||
"scheduleFailedNoReason": "nessun motivo registrato",
|
||||
"scheduleRemedy": "Avvia subito un backup e verifica che compaia una riga nuova. Se fallisce di nuovo, il log del worker indica l'account su cui si è fermato.",
|
||||
"download": "Scarica",
|
||||
"downloadAria": "Scarica il backup di {username}",
|
||||
"downloadFailed": "Download non riuscito",
|
||||
|
||||
@@ -6904,6 +6904,12 @@
|
||||
"colCreatedAt": "최근 스냅샷",
|
||||
"colActions": "동작",
|
||||
"retentionLabel": "보관 기간: {days}일",
|
||||
"scheduleStaleTitle": "주간 백업이 중단되었습니다",
|
||||
"scheduleStaleDescription": "가장 최근의 예약 사본이 {days}일 전 것입니다. 정상적으로 도는 주간 실행은 {threshold}일보다 오래된 사본을 남기지 않습니다.",
|
||||
"scheduleFailedTitle": "마지막 예약 백업 실행이 실패했습니다",
|
||||
"scheduleFailedDescription": "{when}: {reason}",
|
||||
"scheduleFailedNoReason": "기록된 사유 없음",
|
||||
"scheduleRemedy": "지금 백업을 실행하고 새 항목이 나타나는지 확인하세요. 다시 실패하면 워커 로그에 중단된 계정이 표시됩니다.",
|
||||
"download": "다운로드",
|
||||
"downloadAria": "{username}의 백업 다운로드",
|
||||
"downloadFailed": "다운로드하지 못했어요",
|
||||
|
||||
@@ -6923,6 +6923,12 @@
|
||||
"colCreatedAt": "Ostatnia migawka",
|
||||
"colActions": "Akcje",
|
||||
"retentionLabel": "Przechowywanie: {days} dni",
|
||||
"scheduleStaleTitle": "Cotygodniowa kopia zapasowa przestała powstawać",
|
||||
"scheduleStaleDescription": "Najnowsza zaplanowana kopia ma {days} dni. Działające cotygodniowe uruchomienie nigdy nie zostawia kopii starszej niż {threshold} dni.",
|
||||
"scheduleFailedTitle": "Ostatnie zaplanowane uruchomienie kopii zapasowej nie powiodło się",
|
||||
"scheduleFailedDescription": "{when}: {reason}",
|
||||
"scheduleFailedNoReason": "nie zapisano przyczyny",
|
||||
"scheduleRemedy": "Uruchom kopię teraz i sprawdź, czy pojawi się nowy wiersz. Jeśli znowu się nie powiedzie, log workera wskaże konto, na którym się zatrzymał.",
|
||||
"download": "Pobierz",
|
||||
"downloadAria": "Pobierz kopię zapasową dla {username}",
|
||||
"downloadFailed": "Pobieranie nie powiodło się",
|
||||
|
||||
@@ -19,7 +19,7 @@ import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { parseBackupPayload } from "@/lib/validations/backup";
|
||||
|
||||
@@ -52,7 +52,7 @@ export const GET = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch (err) {
|
||||
// Decryption failure is rare but real — a rotated/missing key, or a
|
||||
// corrupted blob from a partially-failed write. Surface it as a 500
|
||||
|
||||
@@ -19,7 +19,7 @@ import { prisma, toJson } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { encryptNote } from "@/lib/crypto/note-cipher";
|
||||
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
|
||||
import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
|
||||
@@ -180,7 +180,7 @@ const handler = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch (err) {
|
||||
await auditLog("admin.backups.restore.failed", {
|
||||
userId: admin.id,
|
||||
|
||||
@@ -16,7 +16,7 @@ import { NextRequest } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess } from "@/lib/api-response";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
isCompatibleSchemaVersion,
|
||||
@@ -44,7 +44,7 @@ export const GET = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch {
|
||||
return apiError("Failed to decrypt backup payload", 500);
|
||||
}
|
||||
|
||||
@@ -5,11 +5,20 @@
|
||||
* shipped — only metadata (id, userId, username, type, size in bytes,
|
||||
* createdAt). The encrypted blob remains server-side; admins can trigger
|
||||
* a re-snapshot but not download another user's payload from the UI.
|
||||
*
|
||||
* Alongside the rows, the response says whether the SCHEDULE is still alive:
|
||||
* how old the newest scheduled copy is and how the last scheduled run ended.
|
||||
* Without those two the page cannot tell a working backup from one that
|
||||
* stopped six weeks ago — every row it lists has a perfectly ordinary
|
||||
* timestamp either way.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
||||
import { readLastQueueRun } from "@/lib/jobs/job-failures";
|
||||
import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status";
|
||||
// v1.4.41 W-ORG — `BackupRow` / `BackupsList` moved to `src/types/backups.ts`
|
||||
// so callers (in particular `components/admin/backups-section.tsx`) don't
|
||||
// have to reach across the component → route-handler layer boundary.
|
||||
@@ -17,31 +26,57 @@ import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/** The type the weekly worker writes; anything else was uploaded by hand. */
|
||||
const SCHEDULED_BACKUP_TYPE = "WEEKLY_AUTO";
|
||||
|
||||
interface BackupMetaRow {
|
||||
id: string;
|
||||
user_id: string;
|
||||
username: string;
|
||||
type: string;
|
||||
size_bytes: number;
|
||||
created_at: Date;
|
||||
}
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
await requireAdmin();
|
||||
annotate({ action: { name: "admin.backups.list" } });
|
||||
|
||||
// Pull metadata only — `data` would balloon the response and reveal
|
||||
// ciphertext we don't need on the listing page.
|
||||
const backups = await prisma.dataBackup.findMany({
|
||||
include: { user: { select: { id: true, username: true } } },
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
// Metadata only, and the size read as `octet_length` in the database rather
|
||||
// than by pulling every blob into this process to measure it. The listing
|
||||
// needs a number, not the ciphertext, and on an instance with large records
|
||||
// fetching them all was its own way of running out of memory. Base64 is
|
||||
// ASCII, so the column's byte length is exactly what the wire reports.
|
||||
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
|
||||
SELECT b.id, b.user_id, u.username, b.type,
|
||||
octet_length(b.data) AS size_bytes,
|
||||
b.created_at
|
||||
FROM data_backups b
|
||||
JOIN users u ON u.id = b.user_id
|
||||
ORDER BY b.created_at DESC
|
||||
`;
|
||||
|
||||
const rows: BackupRow[] = backups.map((b) => ({
|
||||
id: b.id,
|
||||
userId: b.userId,
|
||||
username: b.user.username,
|
||||
type: b.type,
|
||||
sizeBytes: Buffer.byteLength(b.data, "utf8"),
|
||||
createdAt: b.createdAt.toISOString(),
|
||||
const list: BackupRow[] = rows.map((row) => ({
|
||||
id: row.id,
|
||||
userId: row.user_id,
|
||||
username: row.username,
|
||||
type: row.type,
|
||||
sizeBytes: Number(row.size_bytes),
|
||||
createdAt: row.created_at.toISOString(),
|
||||
}));
|
||||
|
||||
const payload: BackupsList = {
|
||||
rows,
|
||||
rows: list,
|
||||
// Matches the retention window the backup-prune job enforces so the
|
||||
// backups page states the same number the worker acts on.
|
||||
retentionDays: 90,
|
||||
schedule: summariseBackupSchedule({
|
||||
scheduledCreatedAt: rows
|
||||
.filter((row) => row.type === SCHEDULED_BACKUP_TYPE)
|
||||
.map((row) => row.created_at),
|
||||
lastRun: await readLastQueueRun(DATA_BACKUP_QUEUE),
|
||||
now: new Date(),
|
||||
}),
|
||||
};
|
||||
|
||||
return apiSuccess(payload);
|
||||
|
||||
@@ -15,11 +15,13 @@ import { auditLog } from "@/lib/auth/audit";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
export const POST = apiHandler(async (request: NextRequest) => {
|
||||
const { user: admin } = await requireAdmin();
|
||||
annotate({ action: { name: "admin.backups.run" } });
|
||||
@@ -57,9 +59,14 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
throw new HttpError(503, "Background worker is not running");
|
||||
}
|
||||
|
||||
const jobId = await boss.send(DATA_BACKUP_QUEUE, {
|
||||
triggeredAt: new Date().toISOString(),
|
||||
});
|
||||
// Same expiration window as the weekly cron. An ad-hoc snapshot runs the
|
||||
// identical pass over the identical record, so it must not silently keep the
|
||||
// 15-minute queue default the cron was rescued from.
|
||||
const jobId = await boss.send(
|
||||
DATA_BACKUP_QUEUE,
|
||||
{ triggeredAt: new Date().toISOString() },
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
);
|
||||
|
||||
annotate({ meta: { job_id: jobId ?? null } });
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import {
|
||||
@@ -190,7 +190,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
// intact so multiple uploads can coexist for the same user without
|
||||
// overwriting each other or the rolling WEEKLY_AUTO snapshot.
|
||||
const uploadType = `MANUAL_UPLOAD_${Date.now()}`;
|
||||
const encrypted = encrypt(JSON.stringify(payload));
|
||||
const encrypted = packBackupBlob(JSON.stringify(payload));
|
||||
|
||||
const created = await prisma.dataBackup.create({
|
||||
data: {
|
||||
|
||||
@@ -3,11 +3,27 @@ import { renderToStaticMarkup } from "react-dom/server";
|
||||
import de from "../../../../messages/de.json";
|
||||
import en from "../../../../messages/en.json";
|
||||
|
||||
const healthySchedule = {
|
||||
lastSuccessAt: "2026-09-01T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 2,
|
||||
staleAfterDays: 10,
|
||||
stale: false,
|
||||
lastRun: {
|
||||
at: "2026-09-01T03:00:12.000Z",
|
||||
state: "completed" as const,
|
||||
error: null,
|
||||
},
|
||||
lastRunFailed: false,
|
||||
};
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
mutationOptions: [] as Array<{
|
||||
onSuccess?: (data: { summary: Record<string, number> }) => void;
|
||||
}>,
|
||||
toastSuccess: vi.fn(),
|
||||
// Mutable so one file can render both a healthy schedule and a broken one;
|
||||
// the query mock reads it on every render.
|
||||
queryData: { rows: [], retentionDays: 30 } as Record<string, unknown>,
|
||||
}));
|
||||
|
||||
/**
|
||||
@@ -34,7 +50,7 @@ vi.mock("sonner", () => ({
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: () => ({
|
||||
data: { rows: [], retentionDays: 30 },
|
||||
data: mocks.queryData,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
}),
|
||||
@@ -69,6 +85,11 @@ import { I18nProvider } from "@/lib/i18n/context";
|
||||
import { BackupsSection } from "../backups-section";
|
||||
|
||||
function render(locale: "en" | "de" = "en") {
|
||||
mocks.queryData = {
|
||||
rows: [],
|
||||
retentionDays: 30,
|
||||
schedule: healthySchedule,
|
||||
};
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale={locale}>
|
||||
<BackupsSection />
|
||||
@@ -129,6 +150,68 @@ describe("BackupsSection — empty state", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The gap that let a weekly backup stop for a month and a half without anyone
|
||||
* noticing: the page rendered whatever rows it had, and a copy from six weeks
|
||||
* ago is indistinguishable from Sunday's until somebody subtracts the dates.
|
||||
*/
|
||||
describe("BackupsSection — schedule health", () => {
|
||||
function withSchedule(schedule: Record<string, unknown>) {
|
||||
mocks.queryData = { rows: [], retentionDays: 30, schedule };
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<BackupsSection />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
it("stays quiet while the schedule is healthy", () => {
|
||||
const html = render();
|
||||
expect(html).not.toContain('data-slot="backup-schedule-notice"');
|
||||
});
|
||||
|
||||
it("says so when the newest scheduled copy has gone stale", () => {
|
||||
const html = withSchedule({
|
||||
...healthySchedule,
|
||||
lastSuccessAt: "2026-07-19T03:00:00.000Z",
|
||||
lastSuccessAgeDays: 46,
|
||||
stale: true,
|
||||
lastRun: null,
|
||||
lastRunFailed: false,
|
||||
});
|
||||
|
||||
expect(html).toContain('data-slot="backup-schedule-notice"');
|
||||
expect(html).toContain("The weekly backup has stopped");
|
||||
expect(html).toContain("46");
|
||||
});
|
||||
|
||||
it("names the reason when the last scheduled run failed", () => {
|
||||
const html = withSchedule({
|
||||
...healthySchedule,
|
||||
lastRun: {
|
||||
at: "2026-08-30T03:46:00.000Z",
|
||||
state: "failed",
|
||||
error: "job timed out",
|
||||
},
|
||||
lastRunFailed: true,
|
||||
});
|
||||
|
||||
expect(html).toContain("The last scheduled backup run failed");
|
||||
expect(html).toContain("job timed out");
|
||||
});
|
||||
|
||||
it("survives a response that predates the schedule field", () => {
|
||||
mocks.queryData = { rows: [], retentionDays: 30 };
|
||||
expect(() =>
|
||||
renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<BackupsSection />
|
||||
</I18nProvider>,
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("BackupsSection — upload outcome", () => {
|
||||
it("includes health profile fact revisions in the restored record total", () => {
|
||||
mocks.toastSuccess.mockClear();
|
||||
|
||||
@@ -48,6 +48,7 @@ import { ListRow } from "@/components/ui/list-row";
|
||||
import { useFormatters, useTranslations } from "@/lib/i18n/context";
|
||||
import { queryKeys } from "@/lib/query-keys";
|
||||
import type { BackupRow, BackupsList } from "@/types/backups";
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
import type {
|
||||
MissingBackupSection,
|
||||
RestoreSkipSummary,
|
||||
@@ -560,6 +561,71 @@ export function RestoreSkipReport({
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Says out loud whether the SCHEDULE is still alive.
|
||||
*
|
||||
* The weekly pass on the maintainer's own instance stopped producing copies
|
||||
* and nothing surfaced it for a month and a half: the table listed the rows it
|
||||
* had, each with an ordinary timestamp, and a copy from six weeks ago reads
|
||||
* exactly like one made on Sunday until somebody does the subtraction. Two
|
||||
* independent facts are shown, because they fail at different times — the age
|
||||
* of the newest scheduled copy never ages out, while the run's own outcome
|
||||
* carries the reason ("job timed out") and survives only as long as pg-boss
|
||||
* keeps the row.
|
||||
*
|
||||
* Not dismissible: it reflects live state, so the only way to clear it is to
|
||||
* make a backup succeed.
|
||||
*/
|
||||
function ScheduleHealthNotice({
|
||||
schedule,
|
||||
}: {
|
||||
schedule: BackupScheduleStatus;
|
||||
}) {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
if (!schedule.stale && !schedule.lastRunFailed) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
role="alert"
|
||||
data-slot="backup-schedule-notice"
|
||||
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-start gap-2">
|
||||
<AlertTriangle className="mt-0.5 h-4 w-4 shrink-0" aria-hidden="true" />
|
||||
<div>
|
||||
<p className="font-medium">
|
||||
{schedule.stale
|
||||
? t("admin.section.backups.scheduleStaleTitle")
|
||||
: t("admin.section.backups.scheduleFailedTitle")}
|
||||
</p>
|
||||
{schedule.stale ? (
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.scheduleStaleDescription", {
|
||||
days: schedule.lastSuccessAgeDays ?? 0,
|
||||
threshold: schedule.staleAfterDays,
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
{schedule.lastRunFailed && schedule.lastRun ? (
|
||||
<p className="text-xs">
|
||||
{t("admin.section.backups.scheduleFailedDescription", {
|
||||
when: fmt.dateTime(schedule.lastRun.at),
|
||||
reason:
|
||||
schedule.lastRun.error ||
|
||||
t("admin.section.backups.scheduleFailedNoReason"),
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
<p className="text-muted-foreground mt-2 text-xs">
|
||||
{t("admin.section.backups.scheduleRemedy")}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function BackupsSection() {
|
||||
const { t } = useTranslations();
|
||||
const fmt = useFormatters();
|
||||
@@ -833,6 +899,10 @@ export function BackupsSection() {
|
||||
}
|
||||
/>
|
||||
|
||||
{data?.schedule ? (
|
||||
<ScheduleHealthNotice schedule={data.schedule} />
|
||||
) : null}
|
||||
|
||||
{/* Upload card — separate from the table so admins can ingest a
|
||||
backup file independently of any existing rows. The visible
|
||||
button proxies a hidden file input so the layout stays clean
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* The stored-backup envelope: compress, then encrypt.
|
||||
*
|
||||
* The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to
|
||||
* Postgres. For an account with a few hundred thousand measurements that means
|
||||
* the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and
|
||||
* the copy the database driver makes of it all alive at once — well over a
|
||||
* gigabyte for a record whose JSON is a few hundred megabytes. The run died of
|
||||
* memory, not of slow SQL.
|
||||
*
|
||||
* Compressing before the cipher is what makes the tail of that pipeline cheap:
|
||||
* health JSON is extremely repetitive, so everything downstream of the gzip
|
||||
* shrinks by an order of magnitude. The envelope has to stay readable both
|
||||
* ways — rows written before this change are plain `encrypt(json)` and must
|
||||
* keep restoring.
|
||||
*/
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob, unpackBackupBlob } from "../backup-blob";
|
||||
|
||||
/** A record-shaped string: many rows, few distinct keys. */
|
||||
function sampleJson(rows: number): string {
|
||||
return JSON.stringify({
|
||||
schemaVersion: "2",
|
||||
measurements: Array.from({ length: rows }, (_, i) => ({
|
||||
id: `measurement-${i}`,
|
||||
type: "PULSE",
|
||||
value: 60 + (i % 30),
|
||||
unit: "bpm",
|
||||
measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(),
|
||||
source: "APPLE_HEALTH",
|
||||
deletedAt: null,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
describe("backup blob envelope", () => {
|
||||
it("round-trips the exact JSON it was handed", () => {
|
||||
const json = sampleJson(200);
|
||||
expect(unpackBackupBlob(packBackupBlob(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("stores a record-shaped payload far smaller than the plaintext", () => {
|
||||
const json = sampleJson(5_000);
|
||||
const packed = packBackupBlob(json);
|
||||
// The old envelope was 1.33× the JSON. Anything near that means the
|
||||
// compression leg is not running.
|
||||
expect(packed.length).toBeLessThan(json.length / 4);
|
||||
});
|
||||
|
||||
it("still reads a row written before the envelope existed", () => {
|
||||
const json = sampleJson(10);
|
||||
expect(unpackBackupBlob(encrypt(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("fails closed on a corrupt blob rather than returning junk", () => {
|
||||
const packed = packBackupBlob(sampleJson(10));
|
||||
const mangled = `${packed.slice(0, -8)}AAAAAAAA`;
|
||||
expect(() => unpackBackupBlob(mangled)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* The envelope a `DataBackup.data` row is stored in: compress, then encrypt.
|
||||
*
|
||||
* Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))`
|
||||
* straight to Postgres. For an account with a few hundred thousand
|
||||
* measurements that is four full copies of the record alive at the same
|
||||
* moment: the payload object graph, the JSON string, the base64 ciphertext at
|
||||
* 1.33× the JSON, and the copy the database driver makes of that parameter on
|
||||
* its way to the wire. Measured on a seeded 445 000-measurement account, the
|
||||
* JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap
|
||||
* it never reaches the insert, and on a bigger heap it only reaches it later.
|
||||
* The weekly pass was not slow; it could not finish.
|
||||
*
|
||||
* Compressing first is what makes the whole tail of that pipeline cheap. A
|
||||
* health record's JSON is extremely repetitive — the same twenty keys per row,
|
||||
* timestamps sharing a prefix — so gzip takes that 242 MB to 14 MB, and every
|
||||
* copy after it shrinks with it. The same run then completes in about eight
|
||||
* seconds with a peak heap of 434 MB. It is also the reason the stored row
|
||||
* stops being a liability of its own: the backup lives INSIDE the database it
|
||||
* would be needed to restore, so its size is not a cosmetic concern.
|
||||
*
|
||||
* Both directions. Rows written before this envelope existed are plain
|
||||
* `encrypt(json)` and have to keep restoring — an operator whose newest usable
|
||||
* copy predates the fix is exactly the person who needs it to work. The marker
|
||||
* sits inside the ciphertext rather than in front of it so the stored column
|
||||
* keeps the one shape `decrypt()` already understands, key ids and all.
|
||||
*/
|
||||
import { Buffer } from "node:buffer";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
|
||||
import { decrypt, encrypt } from "@/lib/crypto";
|
||||
|
||||
/**
|
||||
* Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd.
|
||||
* Chosen so it can never be mistaken for the alternative: a plain payload is
|
||||
* always a JSON object and therefore always starts with `{`.
|
||||
*/
|
||||
const GZIP_MARKER = "HLZ1:";
|
||||
|
||||
/** Serialised backup JSON → the string stored in `DataBackup.data`. */
|
||||
export function packBackupBlob(json: string): string {
|
||||
const compressed = gzipSync(json).toString("base64");
|
||||
return encrypt(`${GZIP_MARKER}${compressed}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A stored `DataBackup.data` string → the backup JSON.
|
||||
*
|
||||
* Fails closed on every arm: a bad key, a mangled ciphertext or a truncated
|
||||
* gzip member throws rather than returning a partial document, because every
|
||||
* caller goes on to parse the result as a whole account.
|
||||
*/
|
||||
export function unpackBackupBlob(stored: string): string {
|
||||
const plaintext = decrypt(stored);
|
||||
if (!plaintext.startsWith(GZIP_MARKER)) return plaintext;
|
||||
return gunzipSync(
|
||||
Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"),
|
||||
).toString("utf8");
|
||||
}
|
||||
@@ -316,6 +316,26 @@ function serialiseMoodContext(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The same payload, already serialised, with the object graph released.
|
||||
*
|
||||
* Its own frame on purpose, and both backup jobs go through it. The payload
|
||||
* graph is the largest thing either job allocates — on an account with a few
|
||||
* hundred thousand measurements it is several hundred megabytes of small
|
||||
* objects — and neither job wants anything but the string. Returning from here
|
||||
* makes the graph unreachable before the compress-and-encrypt step allocates
|
||||
* anything of its own, which is part of why the weekly pass now fits in
|
||||
* memory. Inlining these two statements back into a caller quietly undoes it.
|
||||
*/
|
||||
export async function buildFullBackupJson(
|
||||
prisma: PrismaClient,
|
||||
userId: string,
|
||||
options: FullBackupOptions = {},
|
||||
): Promise<string> {
|
||||
const { payload } = await buildFullBackupPayload(prisma, userId, options);
|
||||
return JSON.stringify(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the canonical full-backup payload for `userId`. Portable exports omit
|
||||
* tombstones and document ciphertext; disaster-recovery payloads preserve
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* What the backups page has to be able to say out loud.
|
||||
*
|
||||
* The weekly snapshot stopped producing a new copy and nothing surfaced it: the
|
||||
* page listed the rows it had, each with a perfectly ordinary timestamp, and a
|
||||
* copy that was a month and a half old looked exactly like one made on Sunday.
|
||||
* The two facts that would have shown it are the age of the newest scheduled
|
||||
* copy and the terminal state of the last scheduled run.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
BACKUP_STALE_AFTER_DAYS,
|
||||
summariseBackupSchedule,
|
||||
} from "../backup-schedule-status";
|
||||
|
||||
const NOW = new Date("2026-09-03T09:00:00.000Z");
|
||||
const daysAgo = (days: number) =>
|
||||
new Date(NOW.getTime() - days * 24 * 60 * 60 * 1000);
|
||||
|
||||
describe("summariseBackupSchedule", () => {
|
||||
it("reports a fresh weekly copy as neither stale nor failed", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(2), daysAgo(9)],
|
||||
lastRun: {
|
||||
at: daysAgo(2).toISOString(),
|
||||
state: "completed",
|
||||
error: null,
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.lastSuccessAt).toBe(daysAgo(2).toISOString());
|
||||
expect(status.lastSuccessAgeDays).toBe(2);
|
||||
expect(status.stale).toBe(false);
|
||||
expect(status.lastRunFailed).toBe(false);
|
||||
expect(status.staleAfterDays).toBe(BACKUP_STALE_AFTER_DAYS);
|
||||
});
|
||||
|
||||
it("calls a copy older than the window stale", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(46)],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.lastSuccessAgeDays).toBe(46);
|
||||
expect(status.stale).toBe(true);
|
||||
// No run row survives that long — the age is the durable signal.
|
||||
expect(status.lastRunFailed).toBe(false);
|
||||
});
|
||||
|
||||
it("surfaces a failed last run even while the copy is still in window", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(3)],
|
||||
lastRun: {
|
||||
at: daysAgo(1).toISOString(),
|
||||
state: "failed",
|
||||
error: "job timed out",
|
||||
},
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.stale).toBe(false);
|
||||
expect(status.lastRunFailed).toBe(true);
|
||||
expect(status.lastRun?.error).toBe("job timed out");
|
||||
});
|
||||
|
||||
it("does not call a brand-new instance stale", () => {
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
expect(status.lastSuccessAt).toBeNull();
|
||||
expect(status.lastSuccessAgeDays).toBeNull();
|
||||
expect(status.stale).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores manual uploads when judging the scheduled cadence", () => {
|
||||
// The caller passes only WEEKLY_AUTO rows; a manual upload made today must
|
||||
// not make a dead cron look alive. Proven here by the shape of the input:
|
||||
// the summary knows nothing but scheduled copies.
|
||||
const status = summariseBackupSchedule({
|
||||
scheduledCreatedAt: [daysAgo(30)],
|
||||
lastRun: null,
|
||||
now: NOW,
|
||||
});
|
||||
expect(status.stale).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Send-policy pins for the weekly `data-backup` queue.
|
||||
*
|
||||
* The queue was registered with no send options at all, so every job it minted
|
||||
* inherited pg-boss's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`.
|
||||
* A full-record snapshot of a large account does not fit in fifteen minutes, so
|
||||
* the pass was killed mid-run, redelivered twice against the same record, and
|
||||
* gave up roughly 46 minutes after it started. Weekly, silently, for as long as
|
||||
* the record stayed large.
|
||||
*
|
||||
* Three facts have to hold together or the window is not real:
|
||||
*
|
||||
* 1. the policy names a window well past the 15-minute default,
|
||||
* 2. the CRON SCHEDULE carries it — send options ride the schedule because
|
||||
* pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a
|
||||
* queue-level default would be inert on every already-running instance,
|
||||
* 3. the admin "run now" route sends with the same options, or an ad-hoc
|
||||
* snapshot silently keeps the default that broke the cron one.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "../data-backup-policy";
|
||||
|
||||
const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8");
|
||||
|
||||
describe("data-backup — pg-boss send policy", () => {
|
||||
it("widens the expiration far past the 15-minute queue default", () => {
|
||||
expect(DATA_BACKUP_QUEUE).toBe("data-backup");
|
||||
// 900 s is the pg-boss 12.26 queue default (`QUEUE_DEFAULTS.expire_seconds`
|
||||
// in `dist/plans.js`); anything at or below it re-opens the bug.
|
||||
expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThan(900);
|
||||
expect(DATA_BACKUP_SEND_OPTIONS.expireInSeconds).toBeGreaterThanOrEqual(
|
||||
60 * 60,
|
||||
);
|
||||
});
|
||||
|
||||
it("the weekly schedule carries the policy", () => {
|
||||
const source = read("src/lib/jobs/reminder/register-maintenance.ts");
|
||||
expect(source).toMatch(
|
||||
/\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/,
|
||||
);
|
||||
});
|
||||
|
||||
it("the admin run-now route sends with the same policy", () => {
|
||||
const source = read("src/app/api/admin/backups/run/route.ts");
|
||||
expect(source).toMatch(
|
||||
/boss\.send\(\s*DATA_BACKUP_QUEUE,[\s\S]{0,200}DATA_BACKUP_SEND_OPTIONS,?\s*\)/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -26,4 +26,16 @@ describe("pg-boss failed-row ledger retention", () => {
|
||||
PG_BOSS_FAILED_ROW_AVAILABILITY_HOURS,
|
||||
);
|
||||
});
|
||||
|
||||
it("pins the queue defaults the data-backup window had to escape", () => {
|
||||
const plans = readFileSync(join(PG_BOSS_ROOT, "dist/plans.js"), "utf8");
|
||||
|
||||
// These two lines are the whole story behind the weekly backup's "job
|
||||
// timed out": a pass with no send options gets fifteen minutes and two
|
||||
// redeliveries, so a snapshot that cannot finish in fifteen minutes gives
|
||||
// up 45 minutes later having never once completed. If a pg-boss upgrade
|
||||
// changes either default, `DATA_BACKUP_SEND_OPTIONS` deserves a re-read.
|
||||
expect(plans).toContain("expire_seconds: FIFTEEN_MINUTES");
|
||||
expect(plans).toContain("retry_limit: 2");
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createCipheriv, randomBytes } from "node:crypto";
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
import {
|
||||
BACKUP_SCHEMA_VERSION,
|
||||
@@ -8,8 +10,20 @@ const mocks = vi.hoisted(() => ({
|
||||
buildFullBackupPayload: vi.fn(),
|
||||
}));
|
||||
|
||||
// The uploader takes the already-serialised form, so the stand-in for
|
||||
// `buildFullBackupJson` serialises whatever the payload mock was told to
|
||||
// return. Every case below keeps stubbing the PAYLOAD, which is the thing
|
||||
// these tests are actually about.
|
||||
vi.mock("@/lib/export/full-backup-payload", () => ({
|
||||
buildFullBackupPayload: mocks.buildFullBackupPayload,
|
||||
buildFullBackupJson: async (...args: unknown[]) =>
|
||||
JSON.stringify(
|
||||
(
|
||||
(await mocks.buildFullBackupPayload(...args)) as {
|
||||
payload: unknown;
|
||||
}
|
||||
).payload,
|
||||
),
|
||||
}));
|
||||
import {
|
||||
encryptBackup,
|
||||
@@ -27,10 +41,33 @@ describe("offhost-backup envelope", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const payload = JSON.stringify({ hello: "world", n: 42 });
|
||||
const buf = encryptBackup(payload, key);
|
||||
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x01");
|
||||
// Version 2: the plaintext is gzipped before the cipher sees it. This
|
||||
// assertion used to pin version 1, which is the version that could not
|
||||
// encrypt a large account's dump without four full copies of it resident
|
||||
// at once. The pin is flipped rather than dropped — the byte is the whole
|
||||
// reason a reader can tell the two apart.
|
||||
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x02");
|
||||
expect(decryptBackup(buf, key)).toBe(payload);
|
||||
});
|
||||
|
||||
it("still reads a version-1 object written before compression", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const payload = JSON.stringify({ hello: "world", n: 42 });
|
||||
// Hand-built v1 envelope: magic || 0x01 || iv || tag || ciphertext, the
|
||||
// exact bytes every object already sitting in an operator's bucket has.
|
||||
const iv = randomBytes(12);
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
||||
const ct = Buffer.concat([cipher.update(payload, "utf8"), cipher.final()]);
|
||||
const legacy = Buffer.concat([
|
||||
Buffer.from("HLBK\x01", "binary"),
|
||||
iv,
|
||||
cipher.getAuthTag(),
|
||||
ct,
|
||||
]);
|
||||
|
||||
expect(decryptBackup(legacy, key)).toBe(payload);
|
||||
});
|
||||
|
||||
it("rejects tampered ciphertext", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const buf = encryptBackup("data", key);
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Whether the weekly backup is actually still happening, in two facts.
|
||||
*
|
||||
* The pass stopped producing a new copy and nothing said so for a month and a
|
||||
* half. Everything that could have told the operator was either invisible or
|
||||
* short-lived: the handler's own error never ran (the job was killed, not
|
||||
* thrown), `recordError()` increments an unnamed counter, and the admin status
|
||||
* page's failing-queue panel only looks back 72 hours — which for a weekly
|
||||
* queue means a permanent failure is on screen three days in seven. The backups
|
||||
* page itself showed a perfectly ordinary row with a perfectly ordinary
|
||||
* timestamp, because a 46-day-old copy looks exactly like a fresh one until you
|
||||
* read the date.
|
||||
*
|
||||
* So the surface carries both legs, and neither is derived from the other:
|
||||
*
|
||||
* - the AGE of the newest scheduled copy, which never ages out of any
|
||||
* retention window and would have caught this on the second missed Sunday;
|
||||
* - the terminal state of the last scheduled RUN, which is the only place the
|
||||
* reason ("job timed out") is written down at all.
|
||||
*
|
||||
* Manual uploads are excluded by the shape of the input: the caller passes only
|
||||
* the scheduled rows, so an admin's ad-hoc upload cannot make a dead cron look
|
||||
* alive.
|
||||
*/
|
||||
import type { LastQueueRun } from "@/lib/jobs/job-failures";
|
||||
|
||||
/**
|
||||
* How old the newest scheduled copy may get before the page says so. The cron
|
||||
* is weekly, so seven days is normal and eight is a rounding error; ten means
|
||||
* at least one Sunday produced nothing.
|
||||
*/
|
||||
export const BACKUP_STALE_AFTER_DAYS = 10;
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
export interface BackupScheduleStatus {
|
||||
/** ISO instant of the newest scheduled copy, or null when there is none. */
|
||||
lastSuccessAt: string | null;
|
||||
/** Whole days since that copy was written. */
|
||||
lastSuccessAgeDays: number | null;
|
||||
/** The threshold behind `stale`, so the UI can name it. */
|
||||
staleAfterDays: number;
|
||||
/** A scheduled copy exists and is older than the threshold. */
|
||||
stale: boolean;
|
||||
/** The last scheduled run pg-boss still remembers. */
|
||||
lastRun: LastQueueRun | null;
|
||||
/** That run ended `failed` or `cancelled`. */
|
||||
lastRunFailed: boolean;
|
||||
}
|
||||
|
||||
export function summariseBackupSchedule(args: {
|
||||
/** `createdAt` of every SCHEDULED backup row; manual uploads excluded. */
|
||||
scheduledCreatedAt: readonly Date[];
|
||||
lastRun: LastQueueRun | null;
|
||||
now: Date;
|
||||
}): BackupScheduleStatus {
|
||||
const newest = args.scheduledCreatedAt.reduce<Date | null>(
|
||||
(best, at) => (best === null || at > best ? at : best),
|
||||
null,
|
||||
);
|
||||
const ageDays =
|
||||
newest === null
|
||||
? null
|
||||
: Math.floor((args.now.getTime() - newest.getTime()) / DAY_MS);
|
||||
|
||||
return {
|
||||
lastSuccessAt: newest?.toISOString() ?? null,
|
||||
lastSuccessAgeDays: ageDays,
|
||||
staleAfterDays: BACKUP_STALE_AFTER_DAYS,
|
||||
// An instance that has never produced a scheduled copy is not stale, it is
|
||||
// new — the page's empty state already says there is nothing there, and
|
||||
// calling a fresh install "overdue" would train the operator to ignore the
|
||||
// warning that matters.
|
||||
stale: ageDays !== null && ageDays > BACKUP_STALE_AFTER_DAYS,
|
||||
lastRun: args.lastRun,
|
||||
lastRunFailed: args.lastRun !== null && args.lastRun.state !== "completed",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Queue name and pg-boss send policy for the weekly on-host data backup.
|
||||
*
|
||||
* A leaf module on purpose: the cron registrar, the admin "run now" route and
|
||||
* the policy test all need these two facts, and none of them should have to
|
||||
* pull in the worker's handler graph to get them.
|
||||
*
|
||||
* The queue was registered with no send options, so every job it minted took
|
||||
* pg-boss 12.26's queue defaults — `expireInSeconds: 900`, `retryLimit: 2`
|
||||
* (`QUEUE_DEFAULTS` in `pg-boss/dist/plans.js`). Fifteen minutes is a fine
|
||||
* default for a cleanup tick and far too short for a pass that serialises every
|
||||
* account's whole record: the job was killed mid-run, redelivered twice against
|
||||
* the same record, and reached `failed` with `job timed out` about 46 minutes
|
||||
* after it started. Nothing about that says "too slow" — the pass never got to
|
||||
* finish once.
|
||||
*
|
||||
* Two hours is deliberately generous rather than tuned. The handler now costs a
|
||||
* fraction of that on a large record (see the CHANGELOG entry), but the pass
|
||||
* loops over EVERY account on the instance and a self-host's disk is whatever
|
||||
* it is; the window's job is to be impossible to hit for the right reasons, and
|
||||
* a failure that arrives at the two-hour mark is a real one worth reading.
|
||||
* pg-boss refuses anything at or past 24 hours.
|
||||
*
|
||||
* The default `retryLimit: 2` stays. The old redeliveries were pointless
|
||||
* because the first attempt could not succeed either; a transient failure — a
|
||||
* pool hiccup, a locked row — genuinely benefits from a second attempt, and the
|
||||
* alternative is losing the week.
|
||||
*
|
||||
* The options ride on the SEND, not on `createQueue`: pg-boss's `create_queue()`
|
||||
* ends in `ON CONFLICT DO NOTHING`, so a queue-level default would apply to a
|
||||
* fresh database and be inert on every instance that already has the queue —
|
||||
* exactly the trap `reconcileQueuePolicies` exists to work around.
|
||||
*/
|
||||
|
||||
export const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
export const DATA_BACKUP_SEND_OPTIONS = {
|
||||
expireInSeconds: 2 * 60 * 60,
|
||||
} as const;
|
||||
@@ -98,7 +98,11 @@ export async function readFailingQueues(
|
||||
max(completed_on) AS last_failed_at,
|
||||
(
|
||||
array_agg(
|
||||
COALESCE(output->>'message', output::text)
|
||||
COALESCE(
|
||||
output->>'message',
|
||||
output->'value'->>'message',
|
||||
output::text
|
||||
)
|
||||
ORDER BY completed_on DESC NULLS LAST
|
||||
)
|
||||
)[1] AS last_error
|
||||
@@ -124,6 +128,75 @@ export async function readFailingQueues(
|
||||
}
|
||||
}
|
||||
|
||||
/** The terminal state pg-boss recorded for the newest run of one queue. */
|
||||
export interface LastQueueRun {
|
||||
/** ISO instant the run reached its terminal state. */
|
||||
at: string;
|
||||
/** `failed` covers both a thrown handler and an expired (killed) job. */
|
||||
state: "completed" | "failed" | "cancelled";
|
||||
/** The failure text, redacted and truncated. Null when the run succeeded. */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The newest terminal run of one queue, whatever its outcome.
|
||||
*
|
||||
* `readFailingQueues` answers "what broke recently" and is deliberately scoped
|
||||
* to a 72-hour window, which is the right shape for a status page watching
|
||||
* thirty queues that tick by the minute. It is the wrong shape for a WEEKLY
|
||||
* pass: a Sunday-night failure has fallen out of the window by Wednesday, so a
|
||||
* job that fails every single week is visible for three days in seven and
|
||||
* invisible for four. This reader takes the newest run instead of a window, so
|
||||
* a weekly queue's last outcome is readable right up until the next one.
|
||||
*
|
||||
* Bounded by pg-boss's own retention: terminal rows live `deletion_seconds`
|
||||
* (seven days by default) past completion, so `null` here means "no run row
|
||||
* survives", which is not the same as "the pass never ran". Callers pair it
|
||||
* with a fact that does not age out — for backups, the age of the newest
|
||||
* stored copy.
|
||||
*/
|
||||
export async function readLastQueueRun(
|
||||
queue: string,
|
||||
): Promise<LastQueueRun | null> {
|
||||
try {
|
||||
const rows = await prisma.$queryRaw<
|
||||
Array<{ state: string; completed_on: Date | null; output: string | null }>
|
||||
>`
|
||||
SELECT
|
||||
state,
|
||||
completed_on,
|
||||
-- pg-boss wraps a thrown value: an expired job's row reads
|
||||
-- {"value":{"message":"job timed out"}}, so a plain ->>'message'
|
||||
-- misses it and the operator gets raw JSON instead of the sentence.
|
||||
COALESCE(
|
||||
output->>'message',
|
||||
output->'value'->>'message',
|
||||
output::text
|
||||
) AS output
|
||||
FROM pgboss.job
|
||||
WHERE name = ${queue}
|
||||
AND state IN ('completed', 'failed', 'cancelled')
|
||||
AND completed_on IS NOT NULL
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
const row = rows[0];
|
||||
if (!row || row.completed_on === null) return null;
|
||||
const state =
|
||||
row.state === "failed" || row.state === "cancelled"
|
||||
? row.state
|
||||
: "completed";
|
||||
return {
|
||||
at: row.completed_on.toISOString(),
|
||||
state,
|
||||
error: state === "completed" ? null : presentError(row.output),
|
||||
};
|
||||
} catch {
|
||||
// No `pgboss` schema (web-only deployment) or no permission to read it.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface QueueFailureForUser {
|
||||
/** ISO instant the user's newest job on this queue gave up. */
|
||||
lastFailedAt: string;
|
||||
|
||||
@@ -18,8 +18,9 @@
|
||||
* the backup history. See docs/ops/backup-restore.md.
|
||||
*/
|
||||
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
@@ -84,20 +85,43 @@ export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
* Encrypt a JSON payload with the dedicated backup key.
|
||||
*
|
||||
* Wire format (binary):
|
||||
* magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext
|
||||
* magic(4)="HLBK" || version(1) || iv(12) || tag(16) || ciphertext
|
||||
*
|
||||
* Version 1 encrypts the JSON directly. Version 2 gzips it first, which is
|
||||
* what the version byte was there for: a large account's dump is hundreds of
|
||||
* megabytes of extremely repetitive JSON, and encrypting it whole means the
|
||||
* string, the cipher's input copy, the ciphertext and the concatenation are
|
||||
* all resident at once — the same arithmetic that made the in-database weekly
|
||||
* backup unfinishable. Gzip takes an order of magnitude off every one of them
|
||||
* and off the object in the bucket. Objects written under version 1 stay
|
||||
* readable; both the restore drill and `scripts/restore-backup.ts` come
|
||||
* through `decryptBackup`, so neither needs to know which it got.
|
||||
*/
|
||||
const BACKUP_ENVELOPE_PLAIN = 0x01;
|
||||
const BACKUP_ENVELOPE_GZIP = 0x02;
|
||||
|
||||
export function encryptBackup(plaintext: string, key: Buffer): Buffer {
|
||||
const iv = randomBytes(IV_LENGTH);
|
||||
const cipher = createCipheriv(ALGORITHM, key, iv);
|
||||
const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
||||
const ct = Buffer.concat([
|
||||
cipher.update(gzipSync(plaintext)),
|
||||
cipher.final(),
|
||||
]);
|
||||
const tag = cipher.getAuthTag();
|
||||
return Buffer.concat([Buffer.from("HLBK\x01", "binary"), iv, tag, ct]);
|
||||
const header = Buffer.from([
|
||||
...Buffer.from("HLBK", "binary"),
|
||||
BACKUP_ENVELOPE_GZIP,
|
||||
]);
|
||||
return Buffer.concat([header, iv, tag, ct]);
|
||||
}
|
||||
|
||||
export function decryptBackup(buf: Buffer, key: Buffer): string {
|
||||
const magic = buf.subarray(0, 4).toString("binary");
|
||||
const version = buf[4];
|
||||
if (magic !== "HLBK" || version !== 0x01) {
|
||||
if (
|
||||
magic !== "HLBK" ||
|
||||
(version !== BACKUP_ENVELOPE_PLAIN && version !== BACKUP_ENVELOPE_GZIP)
|
||||
) {
|
||||
throw new Error("Invalid backup envelope (bad magic or version)");
|
||||
}
|
||||
const iv = buf.subarray(5, 5 + IV_LENGTH);
|
||||
@@ -105,7 +129,10 @@ export function decryptBackup(buf: Buffer, key: Buffer): string {
|
||||
const ct = buf.subarray(5 + IV_LENGTH + TAG_LENGTH);
|
||||
const dec = createDecipheriv(ALGORITHM, key, iv);
|
||||
dec.setAuthTag(tag);
|
||||
return Buffer.concat([dec.update(ct), dec.final()]).toString("utf8");
|
||||
const plaintext = Buffer.concat([dec.update(ct), dec.final()]);
|
||||
return version === BACKUP_ENVELOPE_GZIP
|
||||
? gunzipSync(plaintext).toString("utf8")
|
||||
: plaintext.toString("utf8");
|
||||
}
|
||||
|
||||
export interface S3Like {
|
||||
@@ -215,12 +242,13 @@ export async function runOffhostBackup(
|
||||
const evt = getEvent();
|
||||
for (const user of users) {
|
||||
try {
|
||||
const { payload } = await buildFullBackupPayload(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
});
|
||||
const plaintext = JSON.stringify(payload);
|
||||
const ciphertext = encryptBackup(plaintext, cfg.encryptionKey);
|
||||
const ciphertext = encryptBackup(
|
||||
await buildFullBackupJson(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
}),
|
||||
cfg.encryptionKey,
|
||||
);
|
||||
const key = `${dateKey}/user-${user.id}.json.enc`;
|
||||
await s3.putObject(key, ciphertext);
|
||||
uploaded++;
|
||||
|
||||
@@ -4,16 +4,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
buildFullBackupPayload: vi.fn(),
|
||||
encrypt: vi.fn((value: string) => value),
|
||||
packBlob: vi.fn((value: string) => value),
|
||||
getWorkerPrisma: vi.fn(),
|
||||
upsert: vi.fn(),
|
||||
}));
|
||||
|
||||
// The handler takes the already-serialised form; the stand-in serialises
|
||||
// whatever the payload mock was told to return, so this file keeps stubbing
|
||||
// and asserting the PAYLOAD, which is what it is about.
|
||||
vi.mock("@/lib/export/full-backup-payload", () => ({
|
||||
buildFullBackupPayload: mocks.buildFullBackupPayload,
|
||||
buildFullBackupJson: async (...args: unknown[]) =>
|
||||
JSON.stringify(
|
||||
((await mocks.buildFullBackupPayload(...args)) as { payload: unknown })
|
||||
.payload,
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: mocks.encrypt }));
|
||||
// The envelope is exercised end-to-end in
|
||||
// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so
|
||||
// the stored bytes can be read back as JSON.
|
||||
vi.mock("@/lib/export/backup-blob", () => ({
|
||||
packBackupBlob: mocks.packBlob,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/background", () => ({
|
||||
withBackgroundEvent: vi.fn(
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
*/
|
||||
import { type Job } from "pg-boss";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import {
|
||||
@@ -79,15 +79,21 @@ export async function handleDataBackup(
|
||||
|
||||
let backed = 0;
|
||||
let usersFailed = 0;
|
||||
let largestBlobBytes = 0;
|
||||
for (const user of users) {
|
||||
try {
|
||||
const { payload } = await buildFullBackupPayload(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
});
|
||||
const backupJson = JSON.stringify(payload);
|
||||
|
||||
// Encrypt the backup data (contains sensitive health information)
|
||||
const encryptedBackup = encrypt(backupJson);
|
||||
// Compressed, then encrypted (the record contains sensitive health
|
||||
// information). Both legs live in `packBackupBlob`, which is also
|
||||
// what the restore path reads back through.
|
||||
const encryptedBackup = packBackupBlob(
|
||||
await buildFullBackupJson(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
}),
|
||||
);
|
||||
largestBlobBytes = Math.max(
|
||||
largestBlobBytes,
|
||||
Buffer.byteLength(encryptedBackup, "utf8"),
|
||||
);
|
||||
|
||||
await prisma.dataBackup.upsert({
|
||||
where: {
|
||||
@@ -113,6 +119,9 @@ export async function handleDataBackup(
|
||||
}
|
||||
}
|
||||
|
||||
// The stored size is the one number that says whether this pass is
|
||||
// heading back towards the wall it hit before: it tracks the record.
|
||||
evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes);
|
||||
evt.setBackground({
|
||||
task_name: "job.data_backup",
|
||||
result: { backed, total: users.length },
|
||||
|
||||
@@ -138,6 +138,10 @@ import {
|
||||
} from "@/lib/jobs/environment-fetch";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import {
|
||||
DATA_BACKUP_QUEUE,
|
||||
DATA_BACKUP_SEND_OPTIONS,
|
||||
} from "@/lib/jobs/data-backup-policy";
|
||||
import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared";
|
||||
import {
|
||||
createAndSchedule,
|
||||
@@ -221,8 +225,6 @@ import {
|
||||
type AchievementUnlockSweepPayload,
|
||||
} from "@/lib/jobs/achievement-unlock-sweep";
|
||||
|
||||
const DATA_BACKUP_QUEUE = "data-backup";
|
||||
|
||||
const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00
|
||||
|
||||
const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup";
|
||||
@@ -478,7 +480,11 @@ const schedules: ScheduleEntry[] = [
|
||||
// v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload)
|
||||
// that fans out one per-user environment fetch per opted-in account.
|
||||
[ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON],
|
||||
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON],
|
||||
// The send options are the point of this tuple: without them the weekly
|
||||
// snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on
|
||||
// any sizeable record. See `data-backup-policy.ts` for why the window rides
|
||||
// the schedule rather than the queue.
|
||||
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS],
|
||||
[RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry],
|
||||
[IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry],
|
||||
[AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON],
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
* both sides a route-handler-independent shared type home; the route
|
||||
* keeps owning the HTTP contract, the component keeps owning the UI.
|
||||
*/
|
||||
import type { BackupScheduleStatus } from "@/lib/jobs/backup-schedule-status";
|
||||
|
||||
export interface BackupRow {
|
||||
id: string;
|
||||
@@ -25,6 +26,12 @@ export interface BackupRow {
|
||||
|
||||
export interface BackupsList {
|
||||
rows: BackupRow[];
|
||||
/**
|
||||
* Whether the weekly schedule is still producing copies, and how the last
|
||||
* scheduled run ended. A row's own timestamp cannot answer that — a copy
|
||||
* made six weeks ago and one made on Sunday look identical in a table.
|
||||
*/
|
||||
schedule: BackupScheduleStatus;
|
||||
/**
|
||||
* Soft retention hint — the worker is configured for weekly backups
|
||||
* (see `DATA_BACKUP_CRON` in `src/lib/jobs/reminder-worker.ts`), and
|
||||
|
||||
@@ -197,3 +197,69 @@ describe("backup endpoints — AuditLog contract", () => {
|
||||
expect(restore?.details ?? "").toContain(seeded.id);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The listing route is what an operator looks at to answer "are my backups
|
||||
* still being made". It could not answer that: it returned rows and a
|
||||
* retention hint, and a copy from six weeks ago carries a timestamp exactly
|
||||
* like Sunday's. It now carries the schedule's own health, and the size comes
|
||||
* from `octet_length` in the database rather than from pulling every blob into
|
||||
* the web process to measure it.
|
||||
*/
|
||||
describe("GET /api/admin/backups — schedule health", () => {
|
||||
it("calls a stalled weekly schedule out, and a fresh manual upload does not hide it", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const admin = await seedAdminSession();
|
||||
const blob = encrypt(JSON.stringify(payloadFor(admin.id)));
|
||||
|
||||
const stalled = new Date(Date.now() - 46 * 24 * 60 * 60 * 1000);
|
||||
await prisma.dataBackup.create({
|
||||
data: {
|
||||
userId: admin.id,
|
||||
type: "WEEKLY_AUTO",
|
||||
data: blob,
|
||||
createdAt: stalled,
|
||||
},
|
||||
});
|
||||
// Uploaded by hand today. It must not make the dead cron look alive.
|
||||
await prisma.dataBackup.create({
|
||||
data: { userId: admin.id, type: "MANUAL_UPLOAD_1", data: blob },
|
||||
});
|
||||
|
||||
// The listing handler declares no parameters; `apiHandler` reads the
|
||||
// request defensively out of its arguments, so calling it bare is what a
|
||||
// zero-argument handler supports.
|
||||
const { GET } = await import("@/app/api/admin/backups/route");
|
||||
const res = await GET();
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
rows: Array<{ type: string; sizeBytes: number; username: string }>;
|
||||
schedule: {
|
||||
stale: boolean;
|
||||
lastSuccessAgeDays: number | null;
|
||||
lastRun: unknown;
|
||||
lastRunFailed: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
expect(body.data.rows).toHaveLength(2);
|
||||
// Sizes are read in SQL now; a wrong column or a wrong cast shows up here
|
||||
// as a zero or a NaN rather than as a plausible-looking number.
|
||||
expect(body.data.rows.every((row) => row.sizeBytes === blob.length)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(body.data.rows.every((row) => row.username === admin.username)).toBe(
|
||||
true,
|
||||
);
|
||||
|
||||
expect(body.data.schedule.lastSuccessAgeDays).toBe(46);
|
||||
expect(body.data.schedule.stale).toBe(true);
|
||||
// No `pgboss` schema in the test database — the run reader fails soft to
|
||||
// null rather than taking the whole page down, which is the arm a web-only
|
||||
// deployment lives on permanently.
|
||||
expect(body.data.schedule.lastRun).toBeNull();
|
||||
expect(body.data.schedule.lastRunFailed).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,7 @@
|
||||
* Verifies:
|
||||
* - 201 + a `DataBackup` row with type prefix `MANUAL_UPLOAD_` is
|
||||
* created when the JSON file matches the schema.
|
||||
* - the original payload round-trips through encrypt → decrypt.
|
||||
* - the original payload round-trips through the stored backup envelope.
|
||||
* - 422 when the JSON is malformed.
|
||||
* - 422 when the schema validation fails (missing required field).
|
||||
* - 422 when the userId on the file doesn't exist.
|
||||
@@ -15,7 +15,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
|
||||
import { cookieJar, headerJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
@@ -166,9 +166,11 @@ describe("POST /api/admin/backups/upload", () => {
|
||||
expect(row?.type).toMatch(/^MANUAL_UPLOAD_\d+$/);
|
||||
expect(row?.userId).toBe(admin.id);
|
||||
|
||||
// Round-trip the ciphertext to make sure we stored the same payload
|
||||
// we received.
|
||||
const decrypted = JSON.parse(decrypt(row!.data));
|
||||
// Round-trip the stored blob to make sure we kept the same payload we
|
||||
// received. Through the envelope rather than through `decrypt` alone: an
|
||||
// uploaded file is stored exactly the way the weekly worker stores one, so
|
||||
// the restore path reads both without knowing where the row came from.
|
||||
const decrypted = JSON.parse(unpackBackupBlob(row!.data));
|
||||
expect(decrypted.userId).toBe(admin.id);
|
||||
expect(decrypted.measurements).toHaveLength(1);
|
||||
|
||||
|
||||
@@ -63,9 +63,13 @@ import {
|
||||
encryptWaveformToBytes,
|
||||
} from "@/lib/withings/ecg-waveform-codec";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
<<<<<<< HEAD
|
||||
import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker";
|
||||
import { decryptNoteFromBytes } from "@/lib/labs/store";
|
||||
import { decryptContextFromBytes } from "@/lib/labs/biomarker-store";
|
||||
=======
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
>>>>>>> origin/fix/backup-timeout
|
||||
import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan";
|
||||
import { POST } from "@/app/api/admin/backups/[id]/restore/route";
|
||||
|
||||
@@ -262,6 +266,27 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise<void> {
|
||||
source: "MANUAL",
|
||||
},
|
||||
});
|
||||
// A DELETED reading, kept as a tombstone. It is here because the obvious way
|
||||
// to make the weekly backup cheaper — stop exporting soft-deleted rows, which
|
||||
// on a real record is four rows in five — is wrong, and nothing in the tree
|
||||
// said so out loud. A tombstone inside the retention window is what tells a
|
||||
// synced client the row is gone (`/api/sync/changes` reads exactly these) and
|
||||
// what makes a re-sent Apple Health sample a no-op instead of a resurrection.
|
||||
// Drop them from a disaster-recovery file and a restore quietly brings back
|
||||
// every measurement the account ever deleted on the next device sync.
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: OWNER_ID,
|
||||
type: "PULSE",
|
||||
value: 61,
|
||||
unit: "bpm",
|
||||
measuredAt: AT("2026-07-02T06:30:00.000Z"),
|
||||
source: "APPLE_HEALTH",
|
||||
externalId: "round-trip-deleted-sample",
|
||||
syncVersion: 4,
|
||||
deletedAt: AT("2026-07-20T09:15:00.000Z"),
|
||||
},
|
||||
});
|
||||
// The EVENT row an ECG strip is filed against, and the strip itself. The
|
||||
// pairing is the point: `EcgRecording.measurementId` is a real foreign key,
|
||||
// so this is the reference whose remap has to survive the round trip rather
|
||||
@@ -1234,6 +1259,26 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
purpose: "disaster-recovery",
|
||||
});
|
||||
|
||||
// The tombstone has to be IN the file. A disaster-recovery payload that
|
||||
// carries only live rows is smaller and cheaper and wrong: the restore
|
||||
// wipes the account first, so a deletion that is not in the file is a
|
||||
// deletion the account no longer knows it made, and the next device sync
|
||||
// re-inserts every one of those samples as fresh readings.
|
||||
const exportedMeasurements = payload.measurements as Array<{
|
||||
externalId: string | null;
|
||||
deletedAt: string | null;
|
||||
syncVersion: number;
|
||||
}>;
|
||||
expect(
|
||||
exportedMeasurements.find(
|
||||
(row) => row.externalId === "round-trip-deleted-sample",
|
||||
),
|
||||
"a soft-deleted reading is missing from the disaster-recovery payload",
|
||||
).toMatchObject({
|
||||
deletedAt: "2026-07-20T09:15:00.000Z",
|
||||
syncVersion: 4,
|
||||
});
|
||||
|
||||
// Cascade rather than a hand-listed sweep — see the file comment.
|
||||
await prisma.user.delete({ where: { id: OWNER_ID } });
|
||||
await createOwner(prisma);
|
||||
@@ -1250,7 +1295,12 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
data: {
|
||||
userId: OWNER_ID,
|
||||
type: "TWO_ENDED_ROUND_TRIP",
|
||||
data: encrypt(JSON.stringify(payload)),
|
||||
// The envelope the weekly worker actually writes: gzip, then
|
||||
// encrypt. The three cases further down deliberately keep the plain
|
||||
// `encrypt(json)` form, which is what every row written before this
|
||||
// envelope existed looks like — an operator's newest usable copy may
|
||||
// well be one of those, so both arms have to reach the restore route.
|
||||
data: packBackupBlob(JSON.stringify(payload)),
|
||||
},
|
||||
});
|
||||
const response = await POST(
|
||||
@@ -2029,6 +2079,18 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
// measurement pointer is asserted against the row the RESTORE wrote rather
|
||||
// than against the id the fixture used, because resolving to something
|
||||
// that exists is the property the reference owes.
|
||||
const restoredTombstone = await prisma.measurement.findFirstOrThrow({
|
||||
where: { userId: OWNER_ID, externalId: "round-trip-deleted-sample" },
|
||||
});
|
||||
expect(
|
||||
{
|
||||
deletedAt: restoredTombstone.deletedAt?.toISOString() ?? null,
|
||||
syncVersion: restoredTombstone.syncVersion,
|
||||
},
|
||||
"the deleted reading came back, but not as a deletion — a tombstone " +
|
||||
"restored without its `deletedAt` is a resurrected measurement",
|
||||
).toEqual({ deletedAt: "2026-07-20T09:15:00.000Z", syncVersion: 4 });
|
||||
|
||||
const restoredMeasurement = await prisma.measurement.findFirstOrThrow({
|
||||
where: { userId: OWNER_ID, type: "WEIGHT" },
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user