Merge branch 'feat/restore-background-job' into release/v1.39.1

This commit is contained in:
Marc-André Bombeck committed 2026-09-25 08:10:06 +02:00
commit 934131ee3e
52 files changed
+5684 -2439

No files matched your search

@@ -321,7 +321,7 @@ describe("every backed-up model travels both ways, or is named as debt", () => {
it("does not count a nested relation read as a restore write", () => {
expect(
touches(
["src/app/api/admin/backups/[id]/restore/route.ts"],
["src/lib/export/restore-backup.ts"],
"User",
WRITE_OPS,
WRITE_RELATION_OPS,
@@ -12,7 +12,7 @@ import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
const READERS = [
"src/app/api/admin/backups/[id]/restore/route.ts",
"src/lib/export/restore-backup.ts",
"src/app/api/admin/backups/[id]/summary/route.ts",
"src/app/api/admin/backups/[id]/download/route.ts",
"src/app/api/admin/backups/upload/route.ts",
@@ -114,7 +114,7 @@ const WRITERS: Record<string, Disposition> = {
},
// ── Restore ───────────────────────────────────────────────────────────
"app/api/admin/backups/[id]/restore/route.ts": {
"lib/export/restore-backup.ts": {
kind: "restore",
why: "replays the record's own export; dropping rows would break the restore contract",
},
@@ -177,7 +177,7 @@ describe("T2 — the set of files writing Workout rows is frozen", () => {
"app/api/workouts/batch/route.ts": "client-supplied enum",
// Backup restore replays rows this app previously wrote, provenance and
// all.
"app/api/admin/backups/[id]/restore/route.ts": "restore replay",
"lib/export/restore-backup.ts": "restore replay",
};
it("only the registered ingests and the declared non-mappers create Workout rows", () => {
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,34 @@
/**
* GET /api/admin/backups/restores/[jobId] — how one restore is going.
*
* The status, the phase and the counts while it runs; what it wrote, cleared
* and skipped once it succeeded; a stable code and a sentence an operator can
* act on once it failed.
*/
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import {
evictAfterRestore,
readBackupRestoreJob,
} from "@/lib/jobs/backup-restore";
import { annotate } from "@/lib/logging/context";
export const dynamic = "force-dynamic";
export const GET = apiHandler(
async (
_request: Request,
{ params }: { params: Promise<{ jobId: string }> },
) => {
await requireAdmin();
const { jobId } = await params;
annotate({
action: { name: "admin.backups.restores.read" },
meta: { jobId },
});
const job = await readBackupRestoreJob(jobId);
if (!job) throw new HttpError(404, "Restore job not found");
evictAfterRestore(job);
return apiSuccess(job);
},
);
@@ -0,0 +1,25 @@
/**
* GET /api/admin/backups/restores — the restores the backups console shows.
*
* Every queued or running restore, and those that finished in the last day,
* newest first. The console reads it when it opens, so a reload during a
* restore picks the progress up again, and a restore that finished while
* nobody was looking still says how it ended.
*/
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import {
evictAfterRestore,
listRecentBackupRestoreJobs,
} from "@/lib/jobs/backup-restore";
import { annotate } from "@/lib/logging/context";
export const dynamic = "force-dynamic";
export const GET = apiHandler(async () => {
await requireAdmin();
annotate({ action: { name: "admin.backups.restores.list" } });
const jobs = await listRecentBackupRestoreJobs();
for (const job of jobs) evictAfterRestore(job);
return apiSuccess({ jobs });
});
@@ -0,0 +1,179 @@
/**
* What the backups console shows while a restore runs and after it ends.
*
* The restore runs as a background job; this is the only place an operator
* learns how it is going. Pinned here:
* 1. a running restore names its phase with the counts and draws a bar
* only where the share is known, and says the account still shows its
* current data;
* 2. a failed restore says what went wrong and that nothing changed, except
* for the one failure after the commit, where saying so would be false;
* 3. a succeeded restore reports through the outcome module, so an empty
* file restored to an empty account is not drawn as a tick;
* 4. the panels shown are every active job and the newest finished one not
* dismissed;
* 5. the copy resolves in more than one locale.
*
* Mutation check: always print the "nothing was changed" line and case 2's
* after-commit test goes red; render the bar in every phase and case 1's
* clearing test goes red.
*/
import { describe, expect, it, vi } from "vitest";
import { renderToStaticMarkup } from "react-dom/server";
import { I18nProvider } from "@/lib/i18n/context";
import {
RestoreJobStatus,
visibleRestoreJobs,
} from "@/components/admin/backup-restore-status";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore";
vi.mock("next/navigation", () => ({
useRouter: () => ({ push: vi.fn(), replace: vi.fn() }),
usePathname: () => "/admin/backups",
}));
function job(overrides: Partial<BackupRestoreJobView>): BackupRestoreJobView {
return {
id: "job-1",
userId: "user-1",
username: "alex",
backupId: "backup-1",
restoreInstanceSettings: false,
status: "running",
phase: "measurements",
progress: {
measurementsChecked: 1_250_000,
measurementsTotal: 1_250_000,
measurementsWritten: 400_000,
sectionsDone: 0,
sectionsTotal: 17,
},
result: null,
failure: null,
attempts: 1,
createdAt: "2026-09-25T08:00:00.000Z",
startedAt: "2026-09-25T08:00:01.000Z",
completedAt: null,
...overrides,
};
}
function render(view: BackupRestoreJobView, locale: "en" | "de" = "en") {
return renderToStaticMarkup(
<I18nProvider initialLocale={locale}>
<RestoreJobStatus job={view} onDismiss={() => {}} />
</I18nProvider>,
);
}
describe("RestoreJobStatus while it runs", () => {
it("names the phase with its counts and draws the share done", () => {
const html = render(job({}));
expect(html).toContain("Restoring alex");
expect(html).toContain("Writing readings: 400,000 of 1,250,000");
expect(html).toContain('data-slot="restore-job-progress"');
expect(html).toContain("keeps showing its current data");
});
it("draws no bar where the share is not known", () => {
const html = render(job({ phase: "clearing" }));
expect(html).toContain("Clearing the account");
expect(html).not.toContain('data-slot="restore-job-progress"');
});
it("says a queued restore is waiting", () => {
const html = render(job({ status: "queued", phase: null, progress: null }));
expect(html).toContain("waiting to start");
});
it("says when it was started again after a restart", () => {
expect(render(job({ attempts: 2 }))).toContain("Started again");
});
});
describe("RestoreJobStatus once it failed", () => {
it("says why, and that nothing changed", () => {
const html = render(
job({
status: "failed",
phase: null,
failure: { code: "interrupted", message: "server text" },
}),
);
expect(html).toContain("did not complete");
expect(html).toContain("server restarted");
expect(html).toContain("Nothing was changed");
expect(html).toContain('data-failure-code="interrupted"');
});
it("does not claim nothing changed when the data was restored", () => {
const html = render(
job({
status: "failed",
phase: null,
failure: { code: "failed_after_commit", message: "server text" },
}),
);
expect(html).toContain("The data was restored");
expect(html).not.toContain("Nothing was changed");
});
});
describe("RestoreJobStatus once it succeeded", () => {
const result = {
summary: { measurements: 1_250_000, medications: 3 },
skipped: { links: 0, catalogueKeys: [] },
cleared: { measurements: 10 },
};
it("reports the count through the outcome module", () => {
const html = render(
job({
status: "succeeded",
phase: null,
result,
completedAt: "2026-09-25T08:02:00.000Z",
}),
);
expect(html).toContain("Restore of alex finished");
expect(html).toContain("1,250,003 records restored");
expect(html).toContain('data-outcome="success"');
});
it("does not draw a tick for a file that restored nothing", () => {
const html = render(
job({
status: "succeeded",
phase: null,
result: { ...result, summary: {} },
}),
);
expect(html).toContain('data-outcome="empty"');
});
});
describe("visibleRestoreJobs", () => {
it("shows every active job and the newest finished one not dismissed", () => {
const jobs = [
job({ id: "running", status: "running" }),
job({ id: "newest-done", status: "succeeded" }),
job({ id: "older-done", status: "failed" }),
];
expect(visibleRestoreJobs(jobs, new Set()).map((j) => j.id)).toEqual([
"running",
"newest-done",
]);
expect(
visibleRestoreJobs(jobs, new Set(["newest-done"])).map((j) => j.id),
).toEqual(["running", "older-done"]);
});
});
describe("RestoreJobStatus in another language", () => {
it("speaks German", () => {
const html = render(job({}), "de");
expect(html).toContain("alex wird wiederhergestellt");
expect(html).toContain("Messwerte werden geschrieben");
});
});
@@ -3,7 +3,7 @@
*
* The server refuses a file whose manifest claims a section the file does not
* carry, before it deletes anything. That refusal reaches a person through the
* `meta.errorCode` on the error envelope, and a bare "restore failed" toast
* `code` on the restore job's failure, and a bare "restore failed" toast
* would waste it: the operator on the other end is frequently holding the only
* copy of an account, and the two facts they can act on — which section is
* missing, and that nothing was changed — have to stay on screen.
@@ -14,7 +14,7 @@
* 3. no sections means no notice — a panel that appears after every restore
* trains the operator to ignore it,
* 4. the copy resolves in more than one locale,
* 5. `missingSectionsOf` keys on the errorCode and not on the message text,
* 5. `missingSectionsOf` keys on the failure code and not on the message text,
* so rewording the server's prose cannot silently remove the panel.
*
* Mutation check: render a count instead of the list and case 1 goes red; drop
@@ -30,7 +30,6 @@ import {
RestoreRefusalNotice,
missingSectionsOf,
} from "@/components/admin/backups-section";
import { ApiError } from "@/lib/api/api-fetch";
import type { MissingBackupSection } from "@/lib/export/restore-skips";
vi.mock("next/navigation", () => ({
@@ -79,33 +78,24 @@ describe("RestoreRefusalNotice", () => {
});
describe("missingSectionsOf", () => {
it("reads the sections off the refusal envelope", () => {
const err = new ApiError("Backup is missing documents", 422, {
errorCode: "backup.section.missing",
sections: ["documents"],
});
expect(missingSectionsOf(err)).toEqual(["documents"]);
it("reads the sections off the failed job", () => {
expect(
missingSectionsOf({
code: "backup.section.missing",
sections: ["documents"],
}),
).toEqual(["documents"]);
});
it("ignores an error whose message merely looks like one", () => {
it("ignores a failure whose message merely looks like one", () => {
// The message is prose the server may reword. Only the code counts.
const err = new ApiError("Backup is missing documents", 422, {
errorCode: "something_else",
sections: ["documents"],
});
expect(missingSectionsOf(err)).toEqual([]);
expect(missingSectionsOf(new Error("Backup is missing documents"))).toEqual(
[],
);
expect(
missingSectionsOf({ code: "something_else", sections: ["documents"] }),
).toEqual([]);
expect(missingSectionsOf(null)).toEqual([]);
});
it("survives a refusal with no section list", () => {
const err = new ApiError("Refused", 422, {
errorCode: "backup.section.missing",
});
expect(missingSectionsOf(err)).toEqual([]);
expect(missingSectionsOf({ code: "backup.section.missing" })).toEqual([]);
});
});
@@ -0,0 +1,311 @@
"use client";
/**
* `<RestoreJobStatus>` — one restore job as the backups console shows it.
*
* A restore runs on the server as a background job (see
* `src/lib/jobs/backup-restore.ts`). The console polls
* `GET /api/admin/backups/restores` while a job is queued or running and
* renders each one here: what it is doing and how far it has got, then how it
* ended. Because the list comes from the server, a reload during a restore
* picks it up again where the page left it.
*
* Three rules, each for a way the synchronous restore misled an operator:
*
* - A running restore says the account still shows its current data. It
* does: nothing the restore writes is visible until it commits.
* - A failed restore says whether anything changed, and every failure but
* one changed nothing. It also says the backup is kept, because the next
* question is "do I have to upload it again", and the answer is no.
* - Only the job's own row decides the outcome. A poll that fails on the
* network is not a failed restore and is never shown as one.
*/
import { AlertTriangle, Loader2 } from "lucide-react";
import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line";
import { Button } from "@/components/ui/button";
import { Progress } from "@/components/ui/progress";
import { useFormatters, useTranslations } from "@/lib/i18n/context";
import type {
BackupRestoreFailureCode,
BackupRestoreJobView,
} from "@/lib/jobs/backup-restore";
/** Whether a job still holds its account's one restore slot. */
export function isActiveRestore(job: BackupRestoreJobView): boolean {
return job.status === "queued" || job.status === "running";
}
/**
* The jobs worth a panel: every active one, and the newest finished one the
* operator has not dismissed.
*/
export function visibleRestoreJobs(
jobs: readonly BackupRestoreJobView[],
dismissed: ReadonlySet<string>,
): BackupRestoreJobView[] {
const active = jobs.filter(isActiveRestore);
const finished = jobs.find(
(job) => !isActiveRestore(job) && !dismissed.has(job.id),
);
return finished ? [...active, finished] : active;
}
/** Total records a succeeded restore wrote, from its summary counts. */
function restoredCount(job: BackupRestoreJobView): number {
let total = 0;
for (const value of Object.values(job.result?.summary ?? {})) {
if (typeof value === "number") total += value;
}
return total;
}
/**
* The failure as a sentence in the operator's language.
*
* Literal `t()` calls, so `i18n-call-site-coverage.test.ts` sees every key.
* The `never` at the foot makes the compiler say the chain is exhaustive: a
* failure code added on the server without a sentence here stops the build.
*/
function failureText(
code: BackupRestoreFailureCode,
t: ReturnType<typeof useTranslations>["t"],
): string {
switch (code) {
case "backup_not_found":
return t("admin.section.backups.restoreFailureNotFound");
case "backup_changed":
return t("admin.section.backups.restoreFailureChanged");
case "backup.payload.undecryptable":
return t("admin.section.backups.restoreFailureUndecryptable");
case "schema_invalid":
return t("admin.section.backups.restoreFailureSchema");
case "incompatible_schema_version":
return t("admin.section.backups.restoreFailureVersion");
case "owner_mismatch":
return t("admin.section.backups.restoreFailureOwnerMismatch");
case "owner_not_found":
return t("admin.section.backups.restoreFailureOwnerMissing");
case "backup.section.missing":
return t("admin.section.backups.restoreFailureSectionMissing");
case "document_ciphertext_missing":
return t("admin.section.backups.restoreFailureDocument");
case "time_budget":
return t("admin.section.backups.restoreFailureTimeBudget");
case "transaction_failed":
return t("admin.section.backups.restoreFailureTransaction");
case "interrupted":
return t("admin.section.backups.restoreFailureInterrupted");
case "not_started":
return t("admin.section.backups.restoreFailureNotStarted");
case "enqueue_failed":
return t("admin.section.backups.restoreFailureEnqueue");
case "failed_after_commit":
return t("admin.section.backups.restoreFailureAfterCommit");
case "unexpected":
return t("admin.section.backups.restoreFailureUnexpected");
default: {
const unhandled: never = code;
return unhandled;
}
}
}
/** What a running job is doing, and the share of it done when that is known. */
function runningLine(
job: BackupRestoreJobView,
t: ReturnType<typeof useTranslations>["t"],
fmt: ReturnType<typeof useFormatters>,
): { text: string; percent: number | null } {
const progress = job.progress;
switch (job.phase) {
case "clearing":
return {
text: t("admin.section.backups.restoreJobPhaseClearing"),
percent: null,
};
case "measurements": {
const total = progress?.measurementsTotal ?? 0;
const done = progress?.measurementsWritten ?? 0;
return {
text: t("admin.section.backups.restoreJobPhaseMeasurements", {
done: fmt.integer(done),
total: fmt.integer(total),
}),
percent: total > 0 ? Math.min(100, (done / total) * 100) : null,
};
}
case "sections": {
const total = progress?.sectionsTotal ?? 0;
const done = progress?.sectionsDone ?? 0;
return {
text: t("admin.section.backups.restoreJobPhaseSections", {
done: fmt.integer(done),
total: fmt.integer(total),
}),
percent: total > 0 ? Math.min(100, (done / total) * 100) : null,
};
}
case "rebuilding":
return {
text: t("admin.section.backups.restoreJobPhaseRebuilding"),
percent: null,
};
case "validating":
default:
return {
text: t("admin.section.backups.restoreJobPhaseValidating", {
checked: fmt.integer(progress?.measurementsChecked ?? 0),
}),
percent: null,
};
}
}
export function RestoreJobStatus({
job,
onDismiss,
}: {
job: BackupRestoreJobView;
onDismiss: () => void;
}) {
const { t } = useTranslations();
const fmt = useFormatters();
const username = job.username ?? job.userId;
if (isActiveRestore(job)) {
const line =
job.status === "running"
? runningLine(job, t, fmt)
: {
text: t("admin.section.backups.restoreJobQueuedNote"),
percent: null,
};
return (
<div
role="status"
aria-live="polite"
data-slot="restore-job-status"
data-status={job.status}
className="border-border bg-muted/30 space-y-2 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start gap-2">
<Loader2
className="mt-0.5 size-4 shrink-0 animate-spin motion-reduce:animate-none"
aria-hidden="true"
/>
<div className="min-w-0 flex-1 space-y-1">
<p className="font-medium">
{job.status === "queued"
? t("admin.section.backups.restoreJobQueuedTitle", {
username,
})
: t("admin.section.backups.restoreJobRunningTitle", {
username,
})}
</p>
<p>{line.text}</p>
{line.percent !== null ? (
<Progress
value={line.percent}
aria-label={line.text}
data-slot="restore-job-progress"
/>
) : null}
{job.attempts > 1 ? (
<p className="text-muted-foreground text-xs">
{t("admin.section.backups.restoreJobResumed")}
</p>
) : null}
<p className="text-muted-foreground text-xs">
{t("admin.section.backups.restoreJobRunningNote")}
</p>
</div>
</div>
</div>
);
}
if (job.status === "succeeded") {
const count = restoredCount(job);
return (
<div
data-slot="restore-job-status"
data-status={job.status}
className="border-success/40 bg-success/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start justify-between gap-2">
<div className="space-y-1">
<p className="font-medium">
{t("admin.section.backups.restoreJobSucceededTitle", {
username,
})}
</p>
{/* An empty file restores to an empty account: honest, and not a
tick. The outcome module decides which it looks like. */}
<WrittenOutcomeLine
outcome={count > 0 ? "success" : "empty"}
testId="restore-job-outcome"
message={t(
"admin.section.backups.restoreJobSucceededDescription",
{
count: fmt.integer(count),
when: job.completedAt ? fmt.dateTime(job.completedAt) : "",
},
)}
/>
</div>
<Button
size="sm"
variant="ghost"
onClick={onDismiss}
className="min-h-11"
>
{t("common.dismiss")}
</Button>
</div>
</div>
);
}
const code = job.failure?.code ?? "unexpected";
const changedNothing = code !== "failed_after_commit";
return (
<div
role="alert"
data-slot="restore-job-status"
data-status={job.status}
data-failure-code={code}
className="border-destructive/40 bg-destructive/10 rounded-md border px-3 py-2 text-sm"
>
<div className="flex items-start justify-between gap-2">
<div className="flex items-start gap-2">
<AlertTriangle
className="mt-0.5 size-4 shrink-0"
aria-hidden="true"
/>
<div>
<p className="font-medium">
{t("admin.section.backups.restoreJobFailedTitle", { username })}
</p>
<p className="text-xs">{failureText(code, t)}</p>
{changedNothing ? (
<p className="text-muted-foreground mt-1 text-xs">
{t("admin.section.backups.restoreJobFailedKept")}
</p>
) : null}
</div>
</div>
<Button
size="sm"
variant="ghost"
onClick={onDismiss}
className="min-h-11"
>
{t("common.dismiss")}
</Button>
</div>
</div>
);
}
+146 -45
View File
@@ -57,7 +57,13 @@ import type {
RestoreSkipSummary,
SkippedCatalogue,
} from "@/lib/export/restore-skips";
import type { BackupRestoreJobView } from "@/lib/jobs/backup-restore";
import { getApiErrorMessage } from "./_shared";
import {
isActiveRestore,
RestoreJobStatus,
visibleRestoreJobs,
} from "./backup-restore-status";
import {
ApiError,
apiFetch,
@@ -448,13 +454,41 @@ function missingSectionLabel(
* carry, and that nothing was changed — have to survive long enough to be read.
*/
/**
* The refused sections carried on an `ApiError`, or none.
* The refused sections a failed restore job names, or none.
*
* Keyed on `meta.errorCode` and not on the message text: the message is prose
* Keyed on the failure's `code` and not on its message: the message is prose
* the server may reword, and matching on it would turn a copy edit into a
* silent loss of the panel.
*/
const DISMISSED_RESTORES_KEY = "healthlog.admin.backups.dismissedRestores";
/** Dismissed restore panels, from this browser. Empty when storage is off. */
function readDismissedRestores(): string[] {
try {
if (typeof window === "undefined") return [];
const raw = window.localStorage.getItem(DISMISSED_RESTORES_KEY);
const parsed: unknown = raw ? JSON.parse(raw) : [];
return Array.isArray(parsed)
? parsed.filter((id): id is string => typeof id === "string")
: [];
} catch {
return [];
}
}
function writeDismissedRestores(ids: string[]): void {
try {
// The list only ever shows a day of jobs; fifty ids is weeks of them.
window.localStorage.setItem(
DISMISSED_RESTORES_KEY,
JSON.stringify(ids.slice(-50)),
);
} catch {
// A private window: the panel comes back on reload, nothing else breaks.
}
}
/**
* The upload body for a backup file: the file itself when it is already
* gzip, otherwise the file compressed in the browser. The server accepts
@@ -468,10 +502,11 @@ async function uploadBody(file: File): Promise<Blob> {
).blob();
}
export function missingSectionsOf(err: unknown): MissingBackupSection[] {
if (!(err instanceof ApiError)) return [];
if (err.meta?.errorCode !== "backup.section.missing") return [];
const sections = err.meta?.sections;
export function missingSectionsOf(
failure: { code: string; sections?: readonly unknown[] } | null | undefined,
): MissingBackupSection[] {
if (failure?.code !== "backup.section.missing") return [];
const sections = failure.sections;
if (!Array.isArray(sections)) return [];
return sections.filter(
(section): section is MissingBackupSection => typeof section === "string",
@@ -820,9 +855,77 @@ export function BackupsSection() {
MissingBackupSection[]
>([]);
// Restore: typed-confirmation dialog. The mutation is keyed by row id
// and used inline by `<RestoreRowDialog>` below — keeping the
// mutation here lets the parent invalidate the list query on success.
// A job this page watched while it ran, and has now seen finish. Only those
// raise a toast and refresh the page's data: a job that had already finished
// when the page opened is shown in its panel and nothing more. Read in the
// poll's own callback, where each answer arrives, rather than in an effect.
const watched = useRef<Map<string, string>>(new Map());
function noticeFinishedRestores(jobs: readonly BackupRestoreJobView[]) {
for (const job of jobs) {
const before = watched.current.get(job.id);
watched.current.set(job.id, job.status);
if (before !== "queued" && before !== "running") continue;
if (job.status === "succeeded") {
setRefusedSections([]);
const report = job.result?.skipped;
if (report && report.links > 0) {
setSkipped(report as RestoreSkipSummary);
toast.warning(
t("admin.section.backups.restoreSkippedToast", {
links: String(report.links),
}),
);
} else {
setSkipped(null);
toast.success(t("admin.section.backups.restoreSuccess"));
}
// Restore touches every personal-data table; nuke the broader cache
// so dashboards / lists rebuild against the new state.
void queryClient.invalidateQueries();
} else if (job.status === "failed") {
setRefusedSections(missingSectionsOf(job.failure));
toast.error(t("admin.section.backups.restoreFailed"));
}
}
}
// The restore jobs, read from the server so a reload during a restore finds
// it again. Polled every two seconds while any is queued or running, and
// not at all otherwise. A poll that fails keeps the last answer on screen:
// a network blip is not a failed restore.
const restores = useQuery({
queryKey: queryKeys.adminBackupRestores(),
queryFn: async () => {
const data = await apiGet<{ jobs: BackupRestoreJobView[] }>(
"/api/admin/backups/restores",
);
noticeFinishedRestores(data.jobs);
return data;
},
refetchInterval: (query) =>
query.state.data?.jobs.some(isActiveRestore) ? 2000 : false,
});
const restoreJobs = restores.data?.jobs ?? [];
const activeOwners = new Set(
restoreJobs.filter(isActiveRestore).map((job) => job.userId),
);
// Finished restores the operator dismissed. Remembered in this browser only,
// for convenience; the list itself forgets a job after a day.
const [dismissedRestores, setDismissedRestores] = useState<Set<string>>(
() => new Set(readDismissedRestores()),
);
function dismissRestore(id: string) {
setDismissedRestores((previous) => {
const next = new Set(previous);
next.add(id);
writeDismissedRestores([...next]);
return next;
});
}
// Restore: typed-confirmation dialog. The request only queues the restore;
// the panel above the list follows it from there.
const restore = useMutation({
mutationFn: async ({
row,
@@ -831,47 +934,35 @@ export function BackupsSection() {
row: BackupRow;
restoreInstanceSettings: boolean;
}) => {
// Idempotency-Key prevents a double-click from re-running the
// destructive transaction. Include the row id so two different
// backups can both be restored independently in the same minute.
// Idempotency-Key prevents a double-click from queuing a second
// restore. Include the row id so two different backups can both be
// restored independently in the same minute.
const idempotencyKey = `restore-${row.id}-${randomId()}`;
return apiPost<{ restored: true; skipped?: RestoreSkipSummary }>(
return apiPost<{ jobId: string; status: "queued" }>(
`/api/admin/backups/${row.id}/restore`,
{ confirm: "RESTORE", restoreInstanceSettings },
// No request timeout: rebuilding a large account takes minutes.
{ headers: { "Idempotency-Key": idempotencyKey }, signal: null },
{ headers: { "Idempotency-Key": idempotencyKey } },
);
},
onSuccess: (data) => {
// A restore that dropped links is not a plain success and must not read
// as one. The count and the exact keys go on screen; the same report is
// in the audit row for anyone asking later.
setRefusedSections([]);
const report = data.skipped;
if (report && report.links > 0) {
setSkipped(report);
toast.warning(
t("admin.section.backups.restoreSkippedToast", {
links: String(report.links),
}),
);
} else {
setSkipped(null);
toast.success(t("admin.section.backups.restoreSuccess"));
}
queryClient.invalidateQueries({ queryKey: queryKeys.adminBackups() });
// Restore touches every personal-data table; nuke the broader
// cache so dashboards / lists rebuild against the new state.
queryClient.invalidateQueries();
setSkipped(null);
watched.current.set(data.jobId, "queued");
// Neutral, not a success: nothing has been restored yet.
toast.info(t("admin.section.backups.restoreStarted"));
void queryClient.invalidateQueries({
queryKey: queryKeys.adminBackupRestores(),
});
},
onError: (err) => {
// The server refuses an incomplete file before it deletes anything, and
// names the sections in `meta`. That is the one restore failure the
// operator can act on, so it gets the panel rather than only a toast.
const sections = missingSectionsOf(err);
if (sections.length > 0) {
setRefusedSections(sections);
toast.error(t("admin.section.backups.restoreMissingTitle"));
if (
err instanceof ApiError &&
err.meta?.errorCode === "backup.restore.active"
) {
toast.error(t("admin.section.backups.restoreActive"));
void queryClient.invalidateQueries({
queryKey: queryKeys.adminBackupRestores(),
});
return;
}
setRefusedSections([]);
@@ -1002,6 +1093,14 @@ export function BackupsSection() {
</div>
</div>
{visibleRestoreJobs(restoreJobs, dismissedRestores).map((job) => (
<RestoreJobStatus
key={job.id}
job={job}
onDismiss={() => dismissRestore(job.id)}
/>
))}
<RestoreRefusalNotice
sections={refusedSections}
onDismiss={() => setRefusedSections([])}
@@ -1133,8 +1232,9 @@ export function BackupsSection() {
<RestoreRowDialog
row={row}
pending={
restore.isPending &&
restore.variables?.row.id === row.id
(restore.isPending &&
restore.variables?.row.id === row.id) ||
activeOwners.has(row.userId)
}
onConfirm={(options) =>
restore.mutate({ row, ...options })
@@ -1196,8 +1296,9 @@ export function BackupsSection() {
<RestoreRowDialog
row={row}
pending={
restore.isPending &&
restore.variables?.row.id === row.id
(restore.isPending &&
restore.variables?.row.id === row.id) ||
activeOwners.has(row.userId)
}
onConfirm={(options) =>
restore.mutate({ row, ...options })
+3
View File
@@ -211,6 +211,9 @@ export const WIPE_MODELS = [
// just returned.
"IdempotencyKey",
"DataBackup",
// The restores run from those copies: which copy, who asked, and how each
// ended. About this account's record, so they go with it.
"BackupRestoreJob",
// What this host last put in the operator's bucket for this account. The
// objects themselves stay — the worker holds no DeleteObject grant and the
// bucket's lifecycle rule owns their retirement — but the row saying when
@@ -1,11 +1,10 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
/**
* A restore writes into the account the BACKUP RECORD names, not the one the
* encrypted payload claims.
*
* The route took its target from `payload.userId` — a field inside the blob —
* The restore took its target from `payload.userId` — a field inside the blob —
* while logging `backup.userId` from the row. An admin selecting one user's
* backup could therefore write into a different account, and the audit trail
* would record the account they thought they picked. Both values are in hand at
@@ -15,19 +14,6 @@ import { NextRequest } from "next/server";
* rather than an escalation: the operator's intent was not honoured.
*/
vi.mock("@/lib/api-handler", () => ({
apiHandler: (fn: unknown) => fn,
HttpError: class extends Error {
constructor(
public status: number,
message: string,
) {
super(message);
}
},
requireAdmin: vi.fn(async () => ({ user: { id: "admin-1" } })),
}));
vi.mock("@/lib/db", () => ({
prisma: {
dataBackup: { findUnique: vi.fn() },
@@ -53,10 +39,6 @@ vi.mock("@/lib/validations/backup", () => ({
}));
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/idempotency", () => ({
withIdempotency: (fn: unknown) => fn,
defaultUserIdResolver: vi.fn(),
}));
vi.mock("@/lib/rollups/mood-rollups", () => ({
recomputeUserMoodRollups: vi.fn(),
}));
@@ -68,22 +50,25 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({
recomputeUserRollups: vi.fn(),
}));
import { POST } from "../route";
import { restoreBackup } from "../restore-backup";
import { prisma } from "@/lib/db";
import { parseBackupPayload } from "@/lib/validations/backup";
import { auditLog } from "@/lib/auth/audit";
function request(): NextRequest {
return new NextRequest("http://localhost/api/admin/backups/b-1/restore", {
method: "POST",
headers: { "content-type": "application/json" },
// The route requires a typed confirmation before it will restore anything.
body: JSON.stringify({ confirm: "RESTORE" }),
/**
* Run the restore the job runs, and answer with the status the synchronous
* route used to: 200, or the refusal's status.
*/
async function restore(): Promise<{ status: number }> {
const outcome = await restoreBackup({
backup: { id: "b-1", userId: "user-A", data: "cipher" },
actorUserId: "admin-1",
ipAddress: null,
restoreInstanceSettings: false,
});
return { status: outcome.ok ? 200 : outcome.status };
}
const params = Promise.resolve({ id: "b-1" });
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(prisma.dataBackup.findUnique).mockResolvedValue({
@@ -101,9 +86,7 @@ describe("admin backup restore — declared owner", () => {
userId: "user-B",
} as never);
const res = await (
POST as unknown as (r: NextRequest, c: unknown) => Promise<Response>
)(request(), { params });
const res = await restore();
expect(res.status).toBe(409);
// The restore must not have gone looking for the claimed user at all.
@@ -127,9 +110,7 @@ describe("admin backup restore — declared owner", () => {
} as never);
vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never);
const res = await (
POST as unknown as (r: NextRequest, c: unknown) => Promise<Response>
)(request(), { params });
const res = await restore();
// Reaching the owner lookup proves the mismatch branch did not fire; this
// case then fails on the deleted-user branch, which is the next guard.
@@ -9,7 +9,7 @@
*
* So this test does not assert on the payload. It builds a real payload from a
* populated account, serialises it, parses it back through the REAL
* `parseBackupPayload`, hands it to the REAL route, and asserts on the rows the
* `parseBackupPayload`, hands it to the REAL restore, and asserts on the rows the
* restore tried to write. A field that survives that survives a restore.
*
* The transaction stand-in is deliberately not inert: `cycleSymptom.findMany`
@@ -19,7 +19,6 @@
* inert v1.33.1 fix behaved in production.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
import { Buffer } from "node:buffer";
import { SCORE_VERSION } from "@/lib/analytics/score/types";
@@ -32,19 +31,6 @@ const mocks = vi.hoisted(() => ({
transaction: vi.fn(),
}));
vi.mock("@/lib/api-handler", () => ({
apiHandler: (fn: unknown) => fn,
HttpError: class extends Error {
constructor(
public status: number,
message: string,
) {
super(message);
}
},
requireAdmin: vi.fn(async () => ({ user: { id: "admin-1" } })),
}));
vi.mock("@/lib/db", () => ({
prisma: {
dataBackup: { findUnique: vi.fn() },
@@ -66,10 +52,6 @@ vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
getEvent: vi.fn(() => null),
}));
vi.mock("@/lib/idempotency", () => ({
withIdempotency: (fn: unknown) => fn,
defaultUserIdResolver: vi.fn(),
}));
vi.mock("@/lib/cache/invalidate", () => ({ invalidateUserData: vi.fn() }));
vi.mock("@/lib/rollups/mood-rollups", () => ({
recomputeUserMoodRollups: vi.fn(),
@@ -82,7 +64,7 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({
recomputeUserRollups: vi.fn(),
}));
import { POST } from "../route";
import { restoreBackup } from "../restore-backup";
import { prisma } from "@/lib/db";
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
@@ -437,15 +419,10 @@ function recordingTx(
);
return new Proxy({} as Record<string, unknown>, {
get: (_t, model: string) => delegate(model),
});
}
function request(): NextRequest {
return new NextRequest("http://localhost/api/admin/backups/b-1/restore", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ confirm: "RESTORE" }),
// `$executeRaw` and friends: the restore's per-account lock, and the
// measurement insert, which answer with a row count.
get: (_t, model: string) =>
model.startsWith("$") ? async () => 0 : delegate(model),
});
}
@@ -462,7 +439,7 @@ async function roundTrip(
seed: Record<string, { findUnique?: unknown; findMany?: unknown[] }> = {},
payloadOverride?: Record<string, unknown>,
): Promise<{
res: Response;
res: { status: number };
written: Written[];
payload: Record<string, unknown>;
}> {
@@ -483,13 +460,25 @@ async function roundTrip(
fn(recordingTx(written, seed)),
);
const res = await (
POST as unknown as (r: NextRequest, c: unknown) => Promise<Response>
)(request(), { params: Promise.resolve({ id: "b-1" }) });
const res = await restore();
return { res, written, payload };
}
/**
* Run the restore the job runs, and answer with the status the synchronous
* route used to: 200, or the refusal's status.
*/
async function restore(): Promise<{ status: number }> {
const outcome = await restoreBackup({
backup: { id: "b-1", userId: OWNER, data: "cipher" },
actorUserId: "admin-1",
ipAddress: null,
restoreInstanceSettings: false,
});
return { status: outcome.ok ? 200 : outcome.status };
}
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(prisma.dataBackup.findUnique).mockResolvedValue({
+3 -1
View File
@@ -254,7 +254,7 @@ export const BACKUP_WRITER_FILES: readonly string[] = [
];
export const BACKUP_RESTORE_FILES: readonly string[] = [
"src/app/api/admin/backups/[id]/restore/route.ts",
"src/lib/export/restore-backup.ts",
"src/lib/export/profile-backup.ts",
"src/lib/export/intraday-profile-backup.ts",
"src/lib/export/health-score-backup.ts",
@@ -666,6 +666,8 @@ export const NOT_IN_BACKUP_MODELS: Readonly<Record<string, string>> = {
"Replay protection for requests this host already answered. Restoring it would suppress a legitimate new request.",
DataBackup:
"The backup catalogue itself. A backup that contains the list of backups is a recursion with no reader.",
BackupRestoreJob:
"The record of a restore this host ran from one of its own stored copies. It names a backup row that exists only in this database, so carrying it would describe a restore the receiving host never performed.",
OffhostBackupState:
"When this host last put this account's copy in this operator's bucket, and how big it was. It describes one deployment's relationship with one bucket, so restoring it elsewhere would assert an off-host copy that host has never written.",
ImportJob:
File diff suppressed because it is too large. Load diff
+10
View File
@@ -79,8 +79,17 @@ function parseMeasurement(element: unknown, index: number): BackupMeasurement {
* is not a JSON object and with `StreamedBackupInvalidError` for a
* measurement the schema refuses.
*/
export interface ReadStreamedBackupOptions {
/**
* Called after each measurement passes the element schema, with how many
* have so far. For a caller that reports progress; keep it cheap.
*/
onMeasurementChecked?: (checked: number) => void;
}
export async function readStreamedBackup(
source: BackupSource,
options: ReadStreamedBackupOptions = {},
): Promise<StreamedBackup> {
let firstWithoutId: number | null = null;
let bytes = 0;
@@ -95,6 +104,7 @@ export async function readStreamedBackup(
onElement: (_key, element, index) => {
const row = parseMeasurement(element, index);
if (!row.id && firstWithoutId === null) firstWithoutId = index;
options.onMeasurementChecked?.(index + 1);
},
});
@@ -0,0 +1,79 @@
/**
* Every way a restore job can end serialises into a job outcome.
*
* `runJob` validates what a handler returns before pg-boss persists it: fact
* keys from an allowlist, string values lower-case words joined by
* underscores. Two of the restore's failure codes are the dotted envelope
* codes the console branches on (`backup.section.missing`,
* `backup.payload.undecryptable`). Returned as they are, the validation
* throws, the job fails with a TypeError, and a restore that correctly
* refused an incomplete file reads as a crash in the operator's job card.
* The integration tests call the handler body directly and cannot see it.
*
* Mutation check: return `outcome.code` unmapped in `runBackupRestoreJob`
* (drop `jobFactCode`) and the dotted cases here go red.
*/
import { describe, expect, it } from "vitest";
import {
jobFactCode,
type BackupRestoreFailureCode,
} from "@/lib/jobs/backup-restore";
import {
jobDone,
jobFailed,
serializeJobOutcome,
} from "@/lib/jobs/job-outcome";
// A record over the union, so a code added to it without a row here does not
// compile.
const CODE_TABLE: Record<BackupRestoreFailureCode, true> = {
backup_not_found: true,
backup_changed: true,
"backup.payload.undecryptable": true,
schema_invalid: true,
incompatible_schema_version: true,
owner_mismatch: true,
owner_not_found: true,
"backup.section.missing": true,
document_ciphertext_missing: true,
time_budget: true,
transaction_failed: true,
interrupted: true,
not_started: true,
enqueue_failed: true,
failed_after_commit: true,
unexpected: true,
};
const CODES = Object.keys(CODE_TABLE) as BackupRestoreFailureCode[];
describe("restore job outcomes serialise", () => {
it.each(CODES)("a refusal with code %s", (code) => {
expect(() =>
serializeJobOutcome(jobDone({ refused: jobFactCode(code) })),
).not.toThrow();
});
it("the raw dotted code would not, which is why it is mapped", () => {
expect(() =>
serializeJobOutcome(jobDone({ refused: "backup.section.missing" })),
).toThrow();
});
it("the success, claim and sweep facts", () => {
const facts: Array<Record<string, number | boolean>> = [
{ restore_measurements: 1_250_000, restore_skipped_links: 0 },
{ restore_claimed: false },
{ restore_requeued: 1, restore_failed: 0 },
];
for (const did of facts) {
expect(() => serializeJobOutcome(jobDone(did))).not.toThrow();
}
});
it("the transaction failure", () => {
expect(() =>
serializeJobOutcome(jobFailed("restore_transaction_failed")),
).not.toThrow();
});
});
+18 -1
View File
@@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
import { JOB_BUDGET_SHARE, jobBudget } from "@/lib/jobs/job-budget";
import {
JOB_BUDGET_SHARE,
jobBudget,
jobDeadline,
} from "@/lib/jobs/job-budget";
// pg-boss declares a job dead at its expiry but does not stop the handler;
// the long passes stop themselves on this budget instead (issue #1031).
@@ -43,3 +47,16 @@ describe("jobBudget", () => {
expect(shouldStop()).toBe(false);
});
});
describe("jobDeadline", () => {
it("is the same budget share of the expiry, as an instant", () => {
expect(jobDeadline(job(7200), () => 1_000)).toBe(
1_000 + 7200 * 1000 * JOB_BUDGET_SHARE,
);
});
it("is undefined for a job with no usable expiry", () => {
expect(jobDeadline(job(0))).toBeUndefined();
expect(jobDeadline(job(Number.NaN))).toBeUndefined();
});
});
+780
View File
@@ -0,0 +1,780 @@
/**
* Backup restores as a background job.
*
* A restore used to run inside `POST /api/admin/backups/[id]/restore`. On an
* account of 1.25 million readings that request took about a minute and a
* half, and a reverse proxy that cuts a request at 60 s (nginx's default)
* showed the operator an error for a restore that went on to finish on the
* server. So the request now does only what is quick: it checks the stored copy
* opens, admits one job for the account (`admitBackupRestore`) and answers 202.
* This queue runs the restore (`restoreBackup`, unchanged in what it checks and
* writes) and records its progress and its outcome on the job's row, which
* `GET /api/admin/backups/restores/{id}` reads back.
*
* What stays true from the synchronous restore:
*
* - Every check still happens before the first delete, and the replacement
* is still one transaction. A failed job leaves the account as it was.
* - While it runs, the account keeps reading its current data: nothing the
* transaction writes is visible to anyone until it commits. Other writes
* to the account are neither refused nor held, exactly as before; a reading
* that arrives mid-restore stays next to the restored ones.
*
* What the job adds:
*
* - One restore per account at a time. The job table's partial unique index
* refuses a second queued or running row for the same account, and the
* route answers that with 409.
* - A time budget. The job's expiry is two hours; `restoreBackup` refuses,
* before deleting anything, a file whose transaction limit would run past
* three quarters of it (`jobDeadline`), the same share the other long
* passes stop at. pg-boss does not retry the job (`retryLimit: 0`): a
* restore that failed on its checks fails the same way again.
* - Resuming after a restart. The job writes a heartbeat every few seconds.
* A worker that stops mid-restore rolls the transaction back with it, and
* the boot sweep finds the row whose heartbeat went stale and queues it
* again from the start, at most twice. The stored copy it reads is the
* `DataBackup` row, which stays where it is whatever happens, so no retry
* ever needs the file again.
*/
import { createHash } from "node:crypto";
import type { Job, PgBoss } from "pg-boss";
import { z } from "zod/v4";
import { invalidateUserData } from "@/lib/cache/invalidate";
import { prisma, toJson } from "@/lib/db";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { jobDeadline } from "@/lib/jobs/job-budget";
import { reportJobProgress } from "@/lib/jobs/job-observer";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { annotate } from "@/lib/logging/context";
import { isP2002 } from "@/lib/prisma-errors";
import {
restoreBackup,
RESTORE_SECTION_STEPS,
type RestoreFailureCode,
type RestorePhase,
type RestoreProgress,
} from "@/lib/export/restore-backup";
export const BACKUP_RESTORE_QUEUE = "backup-restore";
/** Stable code of the 409 a second restore of the same account gets. */
export const BACKUP_RESTORE_ACTIVE_CODE = "backup.restore.active";
/**
* Two hours. The restore of 1.25 million readings took a minute and a half;
* the limit is sized so a record many times that still fits, while a job whose
* worker died does not keep its pg-boss row active for longer than an operator
* would wait.
*/
export const BACKUP_RESTORE_EXPIRE_SECONDS = 2 * 60 * 60;
export const BACKUP_RESTORE_SEND_OPTIONS = {
retryLimit: 0,
expireInSeconds: BACKUP_RESTORE_EXPIRE_SECONDS,
} as const;
/** How often a running job writes its progress and heartbeat. */
export const BACKUP_RESTORE_PROGRESS_WRITE_MS = 2_000;
/** A running job whose heartbeat is older than this has lost its worker. */
export const BACKUP_RESTORE_STALE_AFTER_MS = 2 * 60_000;
/** A queued job nobody picked up in this long is reported as not started. */
export const BACKUP_RESTORE_UNCLAIMED_AFTER_MS = 60 * 60_000;
/** Starts a job may have before an interrupted one is failed, not re-queued. */
export const BACKUP_RESTORE_MAX_ATTEMPTS = 2;
/** Finished jobs older than this are deleted when the account restores again. */
export const BACKUP_RESTORE_RETENTION_DAYS = 30;
export type BackupRestorePayload = { restoreJobId: string } | { sweep: true };
export type BackupRestoreStatus = "queued" | "running" | "succeeded" | "failed";
/** Every reason a job can end in `failed`, as a stable code. */
export type BackupRestoreFailureCode =
RestoreFailureCode | "not_started" | "enqueue_failed" | "failed_after_commit";
export interface BackupRestoreFailure {
code: BackupRestoreFailureCode;
message: string;
/** The sections a file's manifest names and it does not carry. */
sections?: string[];
}
const failureSchema = z.object({
code: z.string(),
message: z.string(),
sections: z.array(z.string()).optional(),
});
const progressSchema = z.object({
measurementsChecked: z.number().int().nonnegative(),
measurementsTotal: z.number().int().nonnegative().nullable(),
measurementsWritten: z.number().int().nonnegative(),
sectionsDone: z.number().int().nonnegative(),
sectionsTotal: z.number().int().nonnegative(),
});
const resultSchema = z.object({
summary: z.record(z.string(), z.unknown()),
skipped: z.object({
links: z.number().int(),
catalogueKeys: z.array(
z.object({
catalogue: z.string(),
key: z.string(),
links: z.number().int(),
}),
),
}),
cleared: z.record(z.string(), z.number()),
});
export type BackupRestoreResult = z.infer<typeof resultSchema>;
/** What the status routes answer with. Counts and codes, never content. */
export interface BackupRestoreJobView {
id: string;
userId: string;
username: string | null;
backupId: string;
restoreInstanceSettings: boolean;
status: BackupRestoreStatus;
phase: RestorePhase | null;
progress: RestoreProgress | null;
result: BackupRestoreResult | null;
failure: BackupRestoreFailure | null;
attempts: number;
createdAt: string;
startedAt: string | null;
completedAt: string | null;
}
const PHASES: readonly RestorePhase[] = [
"validating",
"clearing",
"measurements",
"sections",
"rebuilding",
];
const STATUSES: readonly BackupRestoreStatus[] = [
"queued",
"running",
"succeeded",
"failed",
];
/**
* A failure code as a job fact. Job facts are lower-case words joined by
* underscores (`job-outcome.ts`), and two of the restore's codes are the
* dotted envelope codes the console branches on.
*/
export function jobFactCode(code: BackupRestoreFailureCode): string {
return code.replace(/\./g, "_");
}
/** SHA-256 of a stored copy, to tell whether it changed while a job waited. */
export function backupDigest(data: string): string {
return createHash("sha256").update(data).digest("hex");
}
const viewSelect = {
id: true,
userId: true,
backupId: true,
restoreInstanceSettings: true,
status: true,
phase: true,
progress: true,
result: true,
failure: true,
attempts: true,
createdAt: true,
startedAt: true,
completedAt: true,
user: { select: { username: true } },
} as const;
type ViewRow = {
id: string;
userId: string;
backupId: string;
restoreInstanceSettings: boolean;
status: string;
phase: string | null;
progress: unknown;
result: unknown;
failure: unknown;
attempts: number;
createdAt: Date;
startedAt: Date | null;
completedAt: Date | null;
user: { username: string } | null;
};
function toView(row: ViewRow): BackupRestoreJobView {
const status = (STATUSES as readonly string[]).includes(row.status)
? (row.status as BackupRestoreStatus)
: "failed";
const phase =
row.phase && (PHASES as readonly string[]).includes(row.phase)
? (row.phase as RestorePhase)
: null;
const progress = progressSchema.safeParse(row.progress);
const result = resultSchema.safeParse(row.result);
const failure = failureSchema.safeParse(row.failure);
return {
id: row.id,
userId: row.userId,
username: row.user?.username ?? null,
backupId: row.backupId,
restoreInstanceSettings: row.restoreInstanceSettings,
status,
phase: status === "running" ? phase : null,
progress: progress.success ? progress.data : null,
result: result.success ? result.data : null,
failure: failure.success
? (failure.data as BackupRestoreFailure)
: status === "failed"
? { code: "unexpected", message: "The restore failed." }
: null,
attempts: row.attempts,
createdAt: row.createdAt.toISOString(),
startedAt: row.startedAt?.toISOString() ?? null,
completedAt: row.completedAt?.toISOString() ?? null,
};
}
/** One job, for the status route. Null when the id is unknown. */
export async function readBackupRestoreJob(
id: string,
): Promise<BackupRestoreJobView | null> {
const row = await prisma.backupRestoreJob.findUnique({
where: { id },
select: viewSelect,
});
return row ? toView(row) : null;
}
/**
* The jobs the console shows: every queued or running one, and the finished
* ones of the last day, newest first. What a reload of the page needs to pick
* up a restore it was watching, and to show how the last one ended.
*/
export async function listRecentBackupRestoreJobs(
now: Date = new Date(),
): Promise<BackupRestoreJobView[]> {
const since = new Date(now.getTime() - 24 * 60 * 60 * 1000);
const rows = await prisma.backupRestoreJob.findMany({
where: {
OR: [
{ status: { in: ["queued", "running"] } },
{ createdAt: { gte: since } },
],
},
orderBy: { createdAt: "desc" },
take: 20,
select: viewSelect,
});
return rows.map(toView);
}
function failureJson(failure: BackupRestoreFailure) {
return toJson({
code: failure.code,
message: failure.message,
...(failure.sections ? { sections: failure.sections } : {}),
});
}
const INTERRUPTED: BackupRestoreFailure = {
code: "interrupted",
message:
"The restore stopped when the server restarted and was rolled back. Nothing was changed, and the backup is kept: start the restore again. If it stops again, the server likely ran out of memory.",
};
const NOT_STARTED: BackupRestoreFailure = {
code: "not_started",
message:
"The background worker did not pick the restore up. Nothing was changed. Check that the worker is running, then start the restore again.",
};
/**
* Fail the account's jobs that can no longer finish, so they stop holding the
* one-per-account slot: a running job whose worker is gone, and a queued job
* nobody picked up. Also deletes finished jobs past retention.
*/
async function releaseAbandonedJobs(userId: string, now: Date): Promise<void> {
const staleBefore = new Date(now.getTime() - BACKUP_RESTORE_STALE_AFTER_MS);
const unclaimedBefore = new Date(
now.getTime() - BACKUP_RESTORE_UNCLAIMED_AFTER_MS,
);
await prisma.backupRestoreJob.updateMany({
where: {
userId,
status: "running",
OR: [{ heartbeatAt: { lt: staleBefore } }, { heartbeatAt: null }],
},
data: {
status: "failed",
phase: null,
failure: failureJson(INTERRUPTED),
completedAt: now,
},
});
await prisma.backupRestoreJob.updateMany({
where: { userId, status: "queued", createdAt: { lt: unclaimedBefore } },
data: {
status: "failed",
failure: failureJson(NOT_STARTED),
completedAt: now,
},
});
await prisma.backupRestoreJob.deleteMany({
where: {
userId,
status: { in: ["succeeded", "failed"] },
createdAt: {
lt: new Date(
now.getTime() - BACKUP_RESTORE_RETENTION_DAYS * 24 * 60 * 60 * 1000,
),
},
},
});
}
export type BackupRestoreAdmission =
| { admitted: true; jobId: string }
| { admitted: false; reason: "active"; activeJobId: string | null }
| { admitted: false; reason: "enqueue_failed"; jobId: string };
/**
* Admit one restore of `backupId` over `userId`'s record and queue it.
*
* The partial unique index decides whether another restore of the account is
* already queued or running, so two requests arriving together cannot both
* pass a check and both insert.
*/
export async function admitBackupRestore(input: {
userId: string;
actorUserId: string;
backupId: string;
backupDigest: string;
restoreInstanceSettings: boolean;
boss?: PgBoss | null;
}): Promise<BackupRestoreAdmission> {
const now = new Date();
await releaseAbandonedJobs(input.userId, now);
let jobId: string;
try {
const created = await prisma.backupRestoreJob.create({
data: {
userId: input.userId,
actorUserId: input.actorUserId,
backupId: input.backupId,
backupDigest: input.backupDigest,
restoreInstanceSettings: input.restoreInstanceSettings,
status: "queued",
},
select: { id: true },
});
jobId = created.id;
} catch (err) {
if (!isP2002(err)) throw err;
const active = await prisma.backupRestoreJob.findFirst({
where: { userId: input.userId, status: { in: ["queued", "running"] } },
select: { id: true },
});
return {
admitted: false,
reason: "active",
activeJobId: active?.id ?? null,
};
}
const boss = input.boss === undefined ? getGlobalBoss() : input.boss;
try {
if (!boss) throw new Error("worker unavailable");
const bossJobId = await boss.send(
BACKUP_RESTORE_QUEUE,
{ restoreJobId: jobId } satisfies BackupRestorePayload,
BACKUP_RESTORE_SEND_OPTIONS,
);
if (!bossJobId) throw new Error("queue rejected job");
await prisma.backupRestoreJob.update({
where: { id: jobId },
data: { pgBossJobId: bossJobId },
});
return { admitted: true, jobId };
} catch {
// Written but never queued: fail it here, or it holds the account's slot
// until the unclaimed window runs out.
await prisma.backupRestoreJob.update({
where: { id: jobId },
data: {
status: "failed",
failure: failureJson({
code: "enqueue_failed",
message:
"The restore could not be handed to the background worker. Nothing was changed. Check that the worker is running, then start the restore again.",
}),
completedAt: new Date(),
},
});
return { admitted: false, reason: "enqueue_failed", jobId };
}
}
/**
* Run one queued restore job to its end. Exported for the tests, which drive
* the queue without a pg-boss worker.
*/
export async function runBackupRestoreJob(
restoreJobId: string,
options: { deadline?: number } = {},
): Promise<JobOutcome> {
const row = await prisma.backupRestoreJob.findUnique({
where: { id: restoreJobId },
});
if (!row || row.status !== "queued") {
// Already taken by another delivery, finished, or failed by the admission
// check while it waited. There is nothing for this delivery to do.
return jobDone({ restore_claimed: false });
}
const startedAt = new Date();
const claimed = await prisma.backupRestoreJob.updateMany({
where: { id: row.id, status: "queued" },
data: {
status: "running",
phase: "validating",
attempts: { increment: 1 },
heartbeatAt: startedAt,
startedAt: row.startedAt ?? startedAt,
},
});
if (claimed.count === 0) return jobDone({ restore_claimed: false });
annotate({
meta: {
restore_job_id: row.id,
restore_owner_id: row.userId,
restore_attempt: row.attempts + 1,
},
});
let phase: RestorePhase = "validating";
let progress: RestoreProgress = {
measurementsChecked: 0,
measurementsTotal: null,
measurementsWritten: 0,
sectionsDone: 0,
sectionsTotal: RESTORE_SECTION_STEPS.length,
};
let writing: Promise<unknown> | null = null;
const writeProgress = () => {
if (writing) return;
writing = prisma.backupRestoreJob
.updateMany({
where: { id: row.id, status: "running" },
data: { phase, progress: toJson(progress), heartbeatAt: new Date() },
})
.catch(() => undefined)
.finally(() => {
writing = null;
});
};
// A timer, not only the progress callback: a single long statement (the
// delete of a million readings) reports nothing for a while, and the
// heartbeat must not go stale while it runs.
const timer = setInterval(writeProgress, BACKUP_RESTORE_PROGRESS_WRITE_MS);
timer.unref?.();
const finish = async (
status: "succeeded" | "failed",
data: { result?: BackupRestoreResult; failure?: BackupRestoreFailure },
) => {
clearInterval(timer);
if (writing) await writing;
const completedAt = new Date();
await prisma.backupRestoreJob.updateMany({
where: { id: row.id, status: "running" },
data: {
status,
phase: null,
progress: toJson(progress),
...(data.result ? { result: toJson(data.result) } : {}),
...(data.failure ? { failure: failureJson(data.failure) } : {}),
heartbeatAt: completedAt,
completedAt,
},
});
};
try {
const backup = await prisma.dataBackup.findUnique({
where: { id: row.backupId },
select: { id: true, userId: true, data: true },
});
if (!backup || backup.userId !== row.userId) {
await finish("failed", {
failure: {
code: "backup_not_found",
message:
"The backup was deleted before the restore could read it. Nothing was changed.",
},
});
return jobDone({ refused: "backup_not_found" });
}
if (backupDigest(backup.data) !== row.backupDigest) {
await finish("failed", {
failure: {
code: "backup_changed",
message:
"The backup was replaced by a newer copy while the restore waited. Nothing was changed. Check the new copy, then start the restore again.",
},
});
return jobDone({ refused: "backup_changed" });
}
const outcome = await restoreBackup({
backup,
actorUserId: row.actorUserId,
ipAddress: null,
restoreInstanceSettings: row.restoreInstanceSettings,
deadline: options.deadline,
progress: (nextPhase, nextProgress) => {
const phaseChanged = nextPhase !== phase;
phase = nextPhase;
progress = nextProgress;
reportJobProgress({
phase,
measurements_written: progress.measurementsWritten,
sections_done: progress.sectionsDone,
});
if (phaseChanged) writeProgress();
},
});
if (outcome.ok) {
const { summary, skipped, cleared } = outcome.response;
await finish("succeeded", {
result: {
summary: { ...summary },
skipped,
cleared,
},
});
return jobDone({
restore_measurements: Number(summary.measurements ?? 0),
restore_skipped_links: skipped.links,
});
}
const sections = Array.isArray(outcome.meta?.sections)
? (outcome.meta.sections as unknown[]).map(String)
: undefined;
await finish("failed", {
failure: {
code: outcome.code,
message: outcome.message,
...(sections ? { sections } : {}),
},
});
// A file refused by its checks is the operator's to fix and fails the
// same way on every try; the job did what it was asked. A transaction
// that could not be written is a fault, and is reported as one.
return outcome.status >= 500 && outcome.code === "transaction_failed"
? jobFailed("restore_transaction_failed")
: jobDone({ refused: jobFactCode(outcome.code) });
} catch (err) {
// After "rebuilding" the transaction had committed: the data is restored
// and only a step after it failed, which the message has to say.
// (`phase` is written by the progress callback, which TypeScript cannot see.)
const committed = (phase as RestorePhase) === "rebuilding";
await finish("failed", {
failure: committed
? {
code: "failed_after_commit",
message:
"The data was restored, but a step after it failed, so charts may take until the next nightly run to catch up. The server log names the step.",
}
: {
code: "unexpected",
message:
"The restore stopped on an unexpected error and was rolled back. Nothing was changed, and the backup is kept. The server log names the error.",
},
}).catch(() => undefined);
throw err;
} finally {
clearInterval(timer);
}
}
/**
* Re-queue the restores a stopped worker left running, or fail them once they
* have been started {@link BACKUP_RESTORE_MAX_ATTEMPTS} times. The restore is
* one transaction, so a job whose worker died was rolled back with it and
* starts again from the beginning; there is no half-written account to finish.
*/
export async function sweepInterruptedRestores(
boss: PgBoss | null = getGlobalBoss(),
now: Date = new Date(),
): Promise<{ requeued: number; failed: number }> {
const staleBefore = new Date(now.getTime() - BACKUP_RESTORE_STALE_AFTER_MS);
const rows = await prisma.backupRestoreJob.findMany({
where: {
status: "running",
OR: [{ heartbeatAt: { lt: staleBefore } }, { heartbeatAt: null }],
},
select: {
id: true,
attempts: true,
pgBossJobId: true,
heartbeatAt: true,
},
});
let requeued = 0;
let failed = 0;
for (const row of rows) {
// The delivery the dead worker held stays active in pg-boss until its
// expiry and would then read as a timed-out job. It is superseded.
if (boss && row.pgBossJobId) {
await boss
.complete(BACKUP_RESTORE_QUEUE, row.pgBossJobId, { superseded: true })
.catch(() => undefined);
}
const guard = {
id: row.id,
status: "running",
heartbeatAt: row.heartbeatAt,
};
if (!boss || row.attempts >= BACKUP_RESTORE_MAX_ATTEMPTS) {
const done = await prisma.backupRestoreJob.updateMany({
where: guard,
data: {
status: "failed",
phase: null,
failure: failureJson(INTERRUPTED),
completedAt: now,
},
});
failed += done.count;
continue;
}
const reset = await prisma.backupRestoreJob.updateMany({
where: guard,
data: { status: "queued", phase: null, progress: toJson({}) },
});
if (reset.count === 0) continue;
try {
const bossJobId = await boss.send(
BACKUP_RESTORE_QUEUE,
{ restoreJobId: row.id } satisfies BackupRestorePayload,
BACKUP_RESTORE_SEND_OPTIONS,
);
if (!bossJobId) throw new Error("queue rejected job");
await prisma.backupRestoreJob.update({
where: { id: row.id },
data: { pgBossJobId: bossJobId },
});
requeued += 1;
} catch {
await prisma.backupRestoreJob.updateMany({
where: { id: row.id, status: "queued" },
data: {
status: "failed",
failure: failureJson(INTERRUPTED),
completedAt: now,
},
});
failed += 1;
}
}
return { requeued, failed };
}
/**
* Queue the sweep for after a stopped worker's heartbeat has gone stale. Sent
* at worker boot: a job the previous process was running still has a fresh
* heartbeat at that moment, and would otherwise be taken for a live one.
*/
export async function enqueueBackupRestoreSweepAtBoot(
boss: PgBoss,
): Promise<void> {
try {
await boss.send(
BACKUP_RESTORE_QUEUE,
{ sweep: true } satisfies BackupRestorePayload,
{ startAfter: Math.ceil(BACKUP_RESTORE_STALE_AFTER_MS / 1000) + 15 },
);
} catch {
// The admission check fails an abandoned job the next time the account
// is restored, so a missed sweep costs a retry, not a stuck account.
}
}
/** The queue's handler. */
export async function handleBackupRestore(
jobs: Job<BackupRestorePayload>[],
): Promise<JobOutcome> {
// pg-boss hands this queue one job at a time (no batch size is set), and a
// single job's own facts are what its row in pg-boss should keep. A batch
// reports the first failure, or how many it ran.
let failure: JobOutcome | null = null;
let last: JobOutcome = jobDone({ jobs: 0 });
for (const job of jobs) {
const outcome = await withBackgroundEvent(
"job.backup_restore",
async () => {
if (!("sweep" in job.data)) {
return runBackupRestoreJob(job.data.restoreJobId, {
deadline: jobDeadline(job),
});
}
const swept = await sweepInterruptedRestores();
return jobDone({
restore_requeued: swept.requeued,
restore_failed: swept.failed,
});
},
);
last = outcome;
if (!outcome.ok && failure === null) failure = outcome;
}
if (jobs.length === 1) return last;
return failure ?? jobDone({ jobs: jobs.length });
}
/**
* Evict the restored account's server caches in the process serving the
* console.
*
* The job evicts them in the worker's process when the restore commits. In a
* deployment that runs the worker apart from the web server
* (`HEALTHLOG_PROCESS_TYPE=web` beside an `app-worker`), those are different
* processes, and the web process would keep serving the account's pre-restore
* dashboard until its caches aged out. The first time this process reports a
* finished restore, it evicts them itself. Once per job per process; a second
* eviction would only cost the next reader a cache miss.
*/
const evictedRestoreJobs = new Set<string>();
export function evictAfterRestore(job: BackupRestoreJobView): void {
if (job.status !== "succeeded" && job.status !== "failed") return;
if (evictedRestoreJobs.has(job.id)) return;
evictedRestoreJobs.add(job.id);
// A failed restore after its commit (`failed_after_commit`) changed the data
// too; any other failure changed nothing, and evicting costs one cache miss.
if (
job.status === "succeeded" ||
job.failure?.code === "failed_after_commit"
) {
invalidateUserData(job.userId);
}
}
+19
View File
@@ -46,3 +46,22 @@ export function jobBudget(
return () =>
now() >= deadline || jobs.some((job) => job.signal?.aborted === true);
}
/**
* The same budget as an instant, for a pass that cannot stop half-way.
*
* A pass that commits unit by unit asks `jobBudget` between units. A pass
* that is one transaction cannot stop part-way without undoing everything it
* did, so it asks once, before it starts, whether its own time limit ends
* before this instant, and refuses to start when it does. `undefined` when the
* job carries no expiry.
*/
export function jobDeadline(
job: BudgetedJob,
now: () => number = Date.now,
): number | undefined {
if (!Number.isFinite(job.expireInSeconds) || job.expireInSeconds <= 0) {
return undefined;
}
return now() + job.expireInSeconds * 1000 * JOB_BUDGET_SHARE;
}
+7
View File
@@ -166,6 +166,9 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"reminders_due",
"removed",
"rows_written",
// v1.39.1 — the background restore of a stored backup, and the sweep that
// re-queues one a stopped worker left running. Counts and a claim flag.
"restore_claimed",
"restore_drill_age_days",
"restore_drill_ciphertext_bytes",
"restore_drill_intake_events",
@@ -174,6 +177,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"restore_drill_mood_entries",
"restore_drill_plaintext_bytes",
"restore_drill_stale",
"restore_failed",
"restore_measurements",
"restore_requeued",
"restore_skipped_links",
"retried",
"reviewed",
"rollup_failed",
@@ -72,6 +72,12 @@ import {
RESTORE_DRILL_CRON,
RESTORE_DRILL_QUEUE,
} from "@/lib/jobs/restore-drill";
import {
BACKUP_RESTORE_QUEUE,
enqueueBackupRestoreSweepAtBoot,
handleBackupRestore,
type BackupRestorePayload,
} from "@/lib/jobs/backup-restore";
import {
NOTE_ENCRYPTION_BACKFILL_QUEUE,
NOTE_ENCRYPTION_BACKFILL_CONCURRENCY,
@@ -347,6 +353,10 @@ const allQueues = [
STEP_UP_ELEVATION_CLEANUP_QUEUE,
OFFHOST_BACKUP_QUEUE,
RESTORE_DRILL_QUEUE,
// v1.39.1 — an admin's restore of a stored backup, run off the request, and
// the boot sweep that re-queues one a stopped worker left running. Without
// this entry every restore request would queue a job nobody works.
BACKUP_RESTORE_QUEUE,
HOST_METRIC_QUEUE,
FEEDBACK_AGGREGATOR_QUEUE,
GEO_BACKFILL_QUEUE,
@@ -737,6 +747,18 @@ export async function registerMaintenanceQueues(
{ localConcurrency: 1 },
handleRestoreDrill,
);
// One restore at a time per worker: each holds a transaction over a whole
// account and a batch of readings in memory, and two at once on a small host
// is how a restore that fits alone would run out of heap.
await createAndWork<BackupRestorePayload>(
boss,
BACKUP_RESTORE_QUEUE,
{ localConcurrency: 1 },
handleBackupRestore,
);
// A restore the previous process was running is picked up again once its
// heartbeat has gone stale; see `sweepInterruptedRestores`.
await enqueueBackupRestoreSweepAtBoot(boss);
await createAndWork<HostMetricSamplePayload>(
boss,
HOST_METRIC_QUEUE,
+1
View File
@@ -101,6 +101,7 @@ const subsystemSurface = {
"step-up-elevation-cleanup": { audience: "system" },
"data-backup-offhost": { audience: "system" },
"data-restore-drill": { audience: "system" },
"backup-restore": { audience: "system" },
"host-metric-sample": { audience: "system" },
"feedback-aggregator": { audience: "system" },
"geo-backfill": { audience: "system" },
+5 -1
View File
@@ -65,7 +65,11 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
"auth.token.expired",
"auth.token.invalid",
],
backup: ["backup.payload.undecryptable", "backup.section.missing"],
backup: [
"backup.payload.undecryptable",
"backup.restore.active",
"backup.section.missing",
],
coach: [
"coach.budget.exceeded",
"coach.conversation.invalidTitle",
+149 -18
View File
@@ -1,5 +1,6 @@
/**
* OpenAPI route table — the two backups-console routes that read a stored copy.
* OpenAPI route table — the backups-console routes that read a stored copy, and
* the restore job they start.
*
* Part of the OpenAPI route table; aggregated in `./index.ts`.
*
@@ -13,6 +14,10 @@
* (422, `backup.payload.undecryptable`, nothing changed) rather than whatever
* the handler happens to answer, and a promise belongs somewhere it can be
* quoted from.
*
* The restore answers 202 and runs as a background job, so the two routes that
* report on that job are published with it: a 202 without its status route is
* half a contract.
*/
import { z } from "zod/v4";
import type { ZodOpenApiObject } from "zod-openapi";
@@ -42,20 +47,85 @@ const restoreSkipEntry = z
"One catalogue key the file referenced and this instance could not resolve, with the number of links it cost.",
});
const restoreResult = z
const restoreQueued = z
.object({
restored: z.literal(true),
summary: z.record(z.string(), z.unknown()),
skipped: z.object({
catalogueKeys: z.array(restoreSkipEntry),
links: z.number().int(),
}),
cleared: z.record(z.string(), z.number().int()),
jobId: z.string(),
status: z.literal("queued"),
statusUrl: z.string(),
})
.meta({
id: "AdminBackupRestoreResult",
id: "AdminBackupRestoreQueued",
description:
"What the transaction wrote. `cleared` counts the rows each class held before it was rebuilt from the file, `summary` counts what the file carried, and `skipped.links` is the number of links dropped because a catalogue key in the file does not exist here — zero is the normal answer.",
"The restore was accepted and queued. Poll `statusUrl` (`GET /api/admin/backups/restores/{jobId}`) for its progress and outcome.",
});
const restorePhase = z
.enum(["validating", "clearing", "measurements", "sections", "rebuilding"])
.meta({
id: "AdminBackupRestorePhase",
description:
"Where a running restore is: reading and checking the file, clearing the account's current data, writing the readings, writing the other sections, rebuilding the chart tiers after the commit.",
});
const restoreProgress = z
.object({
measurementsChecked: z.number().int(),
measurementsTotal: z.number().int().nullable(),
measurementsWritten: z.number().int(),
sectionsDone: z.number().int(),
sectionsTotal: z.number().int(),
})
.meta({
id: "AdminBackupRestoreProgress",
description:
"Counts only. `measurementsTotal` is null until the first read of the file has counted the readings.",
});
const restoreFailure = z
.object({
code: z.string().meta({
description:
"Stable reason: `backup_not_found`, `backup_changed`, `backup.payload.undecryptable`, `schema_invalid`, `incompatible_schema_version`, `owner_mismatch`, `owner_not_found`, `backup.section.missing`, `document_ciphertext_missing`, `time_budget`, `transaction_failed`, `interrupted`, `not_started`, `enqueue_failed`, `failed_after_commit`, `unexpected`. Every code but `failed_after_commit` means the account was not changed.",
}),
message: z.string(),
sections: z.array(z.string()).optional(),
})
.meta({
id: "AdminBackupRestoreFailure",
description:
"Why the restore did not happen, in a sentence an operator can act on. `sections` names the missing sections for `backup.section.missing`.",
});
const restoreJob = z
.object({
id: z.string(),
userId: z.string(),
username: z.string().nullable(),
backupId: z.string(),
restoreInstanceSettings: z.boolean(),
status: z.enum(["queued", "running", "succeeded", "failed"]),
phase: restorePhase.nullable(),
progress: restoreProgress.nullable(),
result: z
.object({
summary: z.record(z.string(), z.unknown()),
skipped: z.object({
catalogueKeys: z.array(restoreSkipEntry),
links: z.number().int(),
}),
cleared: z.record(z.string(), z.number().int()),
})
.nullable(),
failure: restoreFailure.nullable(),
attempts: z.number().int(),
createdAt: z.string(),
startedAt: z.string().nullable(),
completedAt: z.string().nullable(),
})
.meta({
id: "AdminBackupRestoreJob",
description:
"One restore job. `phase` and `progress` describe a running job; `result` is set once it `succeeded` (the same report the synchronous restore used to answer with, less `restored`); `failure` once it `failed`. `attempts` counts the starts: a job whose worker stopped is started again, at most twice.",
});
/**
@@ -122,9 +192,9 @@ export const adminBackupPaths: NonNullable<ZodOpenApiObject["paths"]> = {
"/api/admin/backups/{id}/restore": {
post: {
tags: ["Admin"],
summary: "Restore one stored backup over its owner's record",
summary: "Queue the restore of one stored backup over its owner's record",
description:
"Replaces the snapshot owner's data tables from the stored copy, in one transaction, under the ids the file carries. Replacing is not merging: every row the account gained after the snapshot was taken is deleted with the rest of its class. The target is the account the snapshot was taken for, never the admin running it, and a payload declaring a different owner than the stored row is refused. Admin session cookie required; Bearer tokens cannot reach admin endpoints.",
"Queues a restore that replaces the snapshot owner's data tables from the stored copy, in one transaction, under the ids the file carries, and answers 202 with the job's id. Replacing is not merging: every row the account gained after the snapshot was taken is deleted with the rest of its class. The target is the account the snapshot was taken for, never the admin running it. The request refuses a missing confirmation, an unknown backup and a copy that cannot be decrypted; the file's own checks (schema, declared owner, manifest, documents) run in the job before anything is deleted, and a refusal there ends the job as `failed` with the same reason. One restore per account at a time. Admin session cookie required; Bearer tokens cannot reach admin endpoints.",
requestParams: {
path: z.object({
id: z.string().meta({ description: "`DataBackup.id`." }),
@@ -139,11 +209,15 @@ export const adminBackupPaths: NonNullable<ZodOpenApiObject["paths"]> = {
content: { "application/json": { schema: restoreRequest } },
},
responses: {
"200": {
description: "The restore ran.",
"202": {
description:
"The restore is queued. Its progress and outcome are at `statusUrl`.",
content: {
"application/json": {
schema: dataEnvelope(restoreResult, "AdminBackupRestoreResponse"),
schema: dataEnvelope(
restoreQueued,
"AdminBackupRestoreQueuedResponse",
),
},
},
},
@@ -155,7 +229,7 @@ export const adminBackupPaths: NonNullable<ZodOpenApiObject["paths"]> = {
"404": notFoundResponse,
"409": {
description:
"Either the payload declares a different owner than the stored row, or a request under the same `Idempotency-Key` is still in flight. Nothing was changed.",
"A restore of the same account is already queued or running (`meta.errorCode` = `backup.restore.active`, `meta.jobId` names it), or a request under the same `Idempotency-Key` is still in flight. Nothing was changed.",
content: { "application/json": { schema: errorEnvelope } },
},
"413": {
@@ -164,7 +238,64 @@ export const adminBackupPaths: NonNullable<ZodOpenApiObject["paths"]> = {
},
"422": {
description:
"The request or the file was refused before anything was written: `confirm` missing, the copy undecryptable (`meta.errorCode` = `backup.payload.undecryptable`), the payload failing schema validation, an unsupported `schemaVersion`, a section the file's own manifest says it carries and does not (`meta.errorCode` = `backup.section.missing`), a metadata-only document entry, or an owner who no longer exists here.",
"The request was refused before anything was queued: `confirm` missing, or the copy undecryptable (`meta.errorCode` = `backup.payload.undecryptable`).",
content: { "application/json": { schema: errorEnvelope } },
},
"503": {
description:
"The restore could not be handed to the background worker. Nothing was changed.",
content: { "application/json": { schema: errorEnvelope } },
},
},
},
},
"/api/admin/backups/restores": {
get: {
tags: ["Admin"],
summary: "List the restores the backups console shows",
description:
"Every queued or running restore, and those created in the last day, newest first, at most 20. What the console reads when it opens, so a reload during a restore picks its progress up again. Admin session cookie required; Bearer tokens cannot reach admin endpoints.",
responses: {
"200": {
description: "The restore jobs.",
content: {
"application/json": {
schema: dataEnvelope(
z.object({ jobs: z.array(restoreJob) }),
"AdminBackupRestoreJobListResponse",
),
},
},
},
"403": adminOnlyResponse,
},
},
},
"/api/admin/backups/restores/{jobId}": {
get: {
tags: ["Admin"],
summary: "Read one restore job",
description:
"The status, phase and counts of one restore, and its report or its failure once it finished. Admin session cookie required; Bearer tokens cannot reach admin endpoints.",
requestParams: {
path: z.object({
jobId: z.string().meta({
description: "The `jobId` the restore request answered with.",
}),
}),
},
responses: {
"200": {
description: "The restore job.",
content: {
"application/json": {
schema: dataEnvelope(restoreJob, "AdminBackupRestoreJobResponse"),
},
},
},
"403": adminOnlyResponse,
"404": {
description: "No restore job with this id.",
content: { "application/json": { schema: errorEnvelope } },
},
},
+5
View File
@@ -41,6 +41,11 @@ export const adminKeys = {
/** v1.37.20 — restore preview: the counts a stored backup file carries. */
adminBackupSummary: (id: string) =>
["admin", "backups", id, "summary"] as const,
/**
* v1.39.1 — the restore jobs the backups console shows. Its own prefix, so
* invalidating the backup list does not refetch the poll and vice versa.
*/
adminBackupRestores: () => ["admin", "backup-restores"] as const,
/** v1.23 — encryption coverage + rotation progress (admin Encryption section). */
adminEncryptionStatus: () => ["admin", "encryption", "status"] as const,
adminCoachFeedback: () => ["admin", "coach-feedback"] as const,