Merge branch 'fix/backup-chunked-storage' into release/v1.39.2

This commit is contained in:
Marc-André Bombeck committed 2026-09-26 01:03:25 +02:00
commit d5a39603ca
43 files changed
+1912 -607

No files matched your search

@@ -24,8 +24,11 @@ import { auditLog } from "@/lib/auth/audit";
import {
BACKUP_UNDECRYPTABLE_CODE,
BACKUP_UNDECRYPTABLE_ERROR,
openBackupBlob,
} from "@/lib/export/backup-blob";
import {
openStoredBackup,
STORED_BACKUP_SELECT,
} from "@/lib/export/stored-backup";
import {
readStreamedBackup,
type BackupSource,
@@ -47,7 +50,12 @@ export const GET = apiHandler(
const backup = await prisma.dataBackup.findUnique({
where: { id },
include: { user: { select: { id: true, username: true } } },
select: {
...STORED_BACKUP_SELECT,
type: true,
createdAt: true,
user: { select: { id: true, username: true } },
},
});
if (!backup) {
@@ -62,10 +70,11 @@ export const GET = apiHandler(
}
// Opened and streamed, never unpacked into one string: a large record's
// JSON is longer than any string V8 can hold (#1031).
// JSON is longer than any string V8 can hold (#1031). Opening checks
// every stored piece first.
let source: BackupSource;
try {
source = openBackupBlob(backup.data);
source = await openStoredBackup(prisma, backup);
} catch (err) {
// A rotated or dropped key, or a stored copy that is no longer the one
// that was written. Both are bad stored input rather than a fault in
@@ -27,8 +27,12 @@ import { prisma } from "@/lib/db";
import {
BACKUP_UNDECRYPTABLE_CODE,
BACKUP_UNDECRYPTABLE_ERROR,
openBackupBlob,
} from "@/lib/export/backup-blob";
import {
openStoredBackup,
STORED_BACKUP_SELECT,
storedBackupIdentity,
} from "@/lib/export/stored-backup";
import { defaultUserIdResolver, withIdempotency } from "@/lib/idempotency";
import {
admitBackupRestore,
@@ -82,7 +86,7 @@ const handler = apiHandler(
const backup = await prisma.dataBackup.findUnique({
where: { id },
select: { id: true, userId: true, data: true },
select: STORED_BACKUP_SELECT,
});
if (!backup) {
await auditLog("admin.backups.restore.denied", {
@@ -93,12 +97,13 @@ const handler = apiHandler(
throw new HttpError(404, "Backup not found");
}
// Opening authenticates the whole ciphertext (and decompresses nothing),
// so a copy written under a key this host no longer holds is refused now,
// in the answer the operator is looking at, rather than minutes later in
// a job. Bad stored input, not a broken server: 422, nothing queued.
// Opening authenticates every stored piece (and decompresses nothing), so
// a copy written under a key this host no longer holds, or one whose
// pieces do not add up, is refused now, in the answer the operator is
// looking at, rather than minutes later in a job. Bad stored input, not a
// broken server: 422, nothing queued.
try {
openBackupBlob(backup.data);
await openStoredBackup(prisma, backup);
} catch (err) {
await auditLog("admin.backups.restore.failed", {
userId: admin.id,
@@ -118,7 +123,7 @@ const handler = apiHandler(
userId: backup.userId,
actorUserId: admin.id,
backupId: backup.id,
backupDigest: backupDigest(backup.data),
backupDigest: backupDigest(storedBackupIdentity(backup)),
restoreInstanceSettings,
});
@@ -64,6 +64,8 @@ beforeEach(() => {
id: "b1",
userId: "u1",
data: "ciphertext",
chunkCount: null,
chunkStreamId: null,
createdAt: new Date("2026-08-01T00:00:00Z"),
user: { id: "u1", username: "self-hoster" },
});
@@ -19,8 +19,11 @@ import { apiError, apiSuccess } from "@/lib/api-response";
import {
BACKUP_UNDECRYPTABLE_CODE,
BACKUP_UNDECRYPTABLE_ERROR,
openBackupBlob,
} from "@/lib/export/backup-blob";
import {
openStoredBackup,
STORED_BACKUP_SELECT,
} from "@/lib/export/stored-backup";
import {
readStreamedBackup,
type BackupSource,
@@ -44,7 +47,11 @@ export const GET = apiHandler(
const backup = await prisma.dataBackup.findUnique({
where: { id },
include: { user: { select: { id: true, username: true } } },
select: {
...STORED_BACKUP_SELECT,
createdAt: true,
user: { select: { id: true, username: true } },
},
});
if (!backup) {
throw new HttpError(404, "Backup not found");
@@ -54,7 +61,7 @@ export const GET = apiHandler(
// string V8 can hold (#1031).
let source: BackupSource;
try {
source = openBackupBlob(backup.data);
source = await openStoredBackup(prisma, backup);
} catch {
// Same refusal the restore and the download give, for the same reason:
// a copy this instance cannot open is bad stored input, and the preview
+11 -3
View File
@@ -63,14 +63,22 @@ export const GET = apiHandler(async () => {
// Metadata only, and the size read as `octet_length` in the database rather
// than by pulling every blob into this process to measure it. The listing
// needs a number, not the ciphertext, and on an instance with large records
// fetching them all was its own way of running out of memory. Base64 is
// ASCII, so the column's byte length is exactly what the wire reports.
// fetching them all was its own way of running out of memory. A copy is
// either one value in `data` (written before v1.39.2) or a run of pieces in
// `data_backup_chunks`, so the size is whichever of the two it has: what
// the copy occupies in the database.
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
SELECT b.id, b.user_id, u.username, b.type,
octet_length(b.data) AS size_bytes,
COALESCE(octet_length(b.data), 0)
+ COALESCE(c.bytes, 0) AS size_bytes,
b.created_at
FROM data_backups b
JOIN users u ON u.id = b.user_id
LEFT JOIN (
SELECT backup_id, SUM(octet_length(data)) AS bytes
FROM data_backup_chunks
GROUP BY backup_id
) c ON c.backup_id = b.id
ORDER BY b.created_at DESC
`;
+8 -4
View File
@@ -220,7 +220,11 @@ export const POST = apiHandler(async (request: NextRequest) => {
try {
created = await storeBackupBlob(
prisma,
() => ({ userId: owner!.id, type: uploadType }),
{
userId: admin.id,
type: uploadType,
ownerAfterRead: () => owner!.id,
},
async (write) => {
// One pass: every chunk is stored as it is checked, and a check that
// fails throws, which rolls the store back.
@@ -336,9 +340,9 @@ export const POST = apiHandler(async (request: NextRequest) => {
await denied(err.details.reason as string, err.details);
return apiError(err.answer, err.status, err.meta);
}
// The file was fine; its encrypted copy is larger than one stored backup
// may be on this host (a fifth of the heap limit). That is the operator's
// to act on, with the numbers in the message, and not a server fault for
// The file was fine; its encrypted copy is larger than the stored-copy
// limit (`BACKUP_MAX_STORED_MB`). That is the operator's to act on, with
// the numbers and the setting in the message, and not a server fault for
// the error reporter. The store rolled back, so nothing was kept.
if (err instanceof BackupBlobTooLargeError) {
await denied("stored_copy_too_large", {