mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 23:33:29 +00:00
Merge branch 'fix/backup-chunked-storage' into release/v1.39.2
This commit is contained in:
commit
d5a39603ca
43 files changed
+1912
-607
No files matched your search
@@ -24,8 +24,11 @@ import { auditLog } from "@/lib/auth/audit";
|
||||
import {
|
||||
BACKUP_UNDECRYPTABLE_CODE,
|
||||
BACKUP_UNDECRYPTABLE_ERROR,
|
||||
openBackupBlob,
|
||||
} from "@/lib/export/backup-blob";
|
||||
import {
|
||||
openStoredBackup,
|
||||
STORED_BACKUP_SELECT,
|
||||
} from "@/lib/export/stored-backup";
|
||||
import {
|
||||
readStreamedBackup,
|
||||
type BackupSource,
|
||||
@@ -47,7 +50,12 @@ export const GET = apiHandler(
|
||||
|
||||
const backup = await prisma.dataBackup.findUnique({
|
||||
where: { id },
|
||||
include: { user: { select: { id: true, username: true } } },
|
||||
select: {
|
||||
...STORED_BACKUP_SELECT,
|
||||
type: true,
|
||||
createdAt: true,
|
||||
user: { select: { id: true, username: true } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!backup) {
|
||||
@@ -62,10 +70,11 @@ export const GET = apiHandler(
|
||||
}
|
||||
|
||||
// Opened and streamed, never unpacked into one string: a large record's
|
||||
// JSON is longer than any string V8 can hold (#1031).
|
||||
// JSON is longer than any string V8 can hold (#1031). Opening checks
|
||||
// every stored piece first.
|
||||
let source: BackupSource;
|
||||
try {
|
||||
source = openBackupBlob(backup.data);
|
||||
source = await openStoredBackup(prisma, backup);
|
||||
} catch (err) {
|
||||
// A rotated or dropped key, or a stored copy that is no longer the one
|
||||
// that was written. Both are bad stored input rather than a fault in
|
||||
|
||||
@@ -27,8 +27,12 @@ import { prisma } from "@/lib/db";
|
||||
import {
|
||||
BACKUP_UNDECRYPTABLE_CODE,
|
||||
BACKUP_UNDECRYPTABLE_ERROR,
|
||||
openBackupBlob,
|
||||
} from "@/lib/export/backup-blob";
|
||||
import {
|
||||
openStoredBackup,
|
||||
STORED_BACKUP_SELECT,
|
||||
storedBackupIdentity,
|
||||
} from "@/lib/export/stored-backup";
|
||||
import { defaultUserIdResolver, withIdempotency } from "@/lib/idempotency";
|
||||
import {
|
||||
admitBackupRestore,
|
||||
@@ -82,7 +86,7 @@ const handler = apiHandler(
|
||||
|
||||
const backup = await prisma.dataBackup.findUnique({
|
||||
where: { id },
|
||||
select: { id: true, userId: true, data: true },
|
||||
select: STORED_BACKUP_SELECT,
|
||||
});
|
||||
if (!backup) {
|
||||
await auditLog("admin.backups.restore.denied", {
|
||||
@@ -93,12 +97,13 @@ const handler = apiHandler(
|
||||
throw new HttpError(404, "Backup not found");
|
||||
}
|
||||
|
||||
// Opening authenticates the whole ciphertext (and decompresses nothing),
|
||||
// so a copy written under a key this host no longer holds is refused now,
|
||||
// in the answer the operator is looking at, rather than minutes later in
|
||||
// a job. Bad stored input, not a broken server: 422, nothing queued.
|
||||
// Opening authenticates every stored piece (and decompresses nothing), so
|
||||
// a copy written under a key this host no longer holds, or one whose
|
||||
// pieces do not add up, is refused now, in the answer the operator is
|
||||
// looking at, rather than minutes later in a job. Bad stored input, not a
|
||||
// broken server: 422, nothing queued.
|
||||
try {
|
||||
openBackupBlob(backup.data);
|
||||
await openStoredBackup(prisma, backup);
|
||||
} catch (err) {
|
||||
await auditLog("admin.backups.restore.failed", {
|
||||
userId: admin.id,
|
||||
@@ -118,7 +123,7 @@ const handler = apiHandler(
|
||||
userId: backup.userId,
|
||||
actorUserId: admin.id,
|
||||
backupId: backup.id,
|
||||
backupDigest: backupDigest(backup.data),
|
||||
backupDigest: backupDigest(storedBackupIdentity(backup)),
|
||||
restoreInstanceSettings,
|
||||
});
|
||||
|
||||
|
||||
@@ -64,6 +64,8 @@ beforeEach(() => {
|
||||
id: "b1",
|
||||
userId: "u1",
|
||||
data: "ciphertext",
|
||||
chunkCount: null,
|
||||
chunkStreamId: null,
|
||||
createdAt: new Date("2026-08-01T00:00:00Z"),
|
||||
user: { id: "u1", username: "self-hoster" },
|
||||
});
|
||||
|
||||
@@ -19,8 +19,11 @@ import { apiError, apiSuccess } from "@/lib/api-response";
|
||||
import {
|
||||
BACKUP_UNDECRYPTABLE_CODE,
|
||||
BACKUP_UNDECRYPTABLE_ERROR,
|
||||
openBackupBlob,
|
||||
} from "@/lib/export/backup-blob";
|
||||
import {
|
||||
openStoredBackup,
|
||||
STORED_BACKUP_SELECT,
|
||||
} from "@/lib/export/stored-backup";
|
||||
import {
|
||||
readStreamedBackup,
|
||||
type BackupSource,
|
||||
@@ -44,7 +47,11 @@ export const GET = apiHandler(
|
||||
|
||||
const backup = await prisma.dataBackup.findUnique({
|
||||
where: { id },
|
||||
include: { user: { select: { id: true, username: true } } },
|
||||
select: {
|
||||
...STORED_BACKUP_SELECT,
|
||||
createdAt: true,
|
||||
user: { select: { id: true, username: true } },
|
||||
},
|
||||
});
|
||||
if (!backup) {
|
||||
throw new HttpError(404, "Backup not found");
|
||||
@@ -54,7 +61,7 @@ export const GET = apiHandler(
|
||||
// string V8 can hold (#1031).
|
||||
let source: BackupSource;
|
||||
try {
|
||||
source = openBackupBlob(backup.data);
|
||||
source = await openStoredBackup(prisma, backup);
|
||||
} catch {
|
||||
// Same refusal the restore and the download give, for the same reason:
|
||||
// a copy this instance cannot open is bad stored input, and the preview
|
||||
|
||||
@@ -63,14 +63,22 @@ export const GET = apiHandler(async () => {
|
||||
// Metadata only, and the size read as `octet_length` in the database rather
|
||||
// than by pulling every blob into this process to measure it. The listing
|
||||
// needs a number, not the ciphertext, and on an instance with large records
|
||||
// fetching them all was its own way of running out of memory. Base64 is
|
||||
// ASCII, so the column's byte length is exactly what the wire reports.
|
||||
// fetching them all was its own way of running out of memory. A copy is
|
||||
// either one value in `data` (written before v1.39.2) or a run of pieces in
|
||||
// `data_backup_chunks`, so the size is whichever of the two it has: what
|
||||
// the copy occupies in the database.
|
||||
const rows = await prisma.$queryRaw<BackupMetaRow[]>`
|
||||
SELECT b.id, b.user_id, u.username, b.type,
|
||||
octet_length(b.data) AS size_bytes,
|
||||
COALESCE(octet_length(b.data), 0)
|
||||
+ COALESCE(c.bytes, 0) AS size_bytes,
|
||||
b.created_at
|
||||
FROM data_backups b
|
||||
JOIN users u ON u.id = b.user_id
|
||||
LEFT JOIN (
|
||||
SELECT backup_id, SUM(octet_length(data)) AS bytes
|
||||
FROM data_backup_chunks
|
||||
GROUP BY backup_id
|
||||
) c ON c.backup_id = b.id
|
||||
ORDER BY b.created_at DESC
|
||||
`;
|
||||
|
||||
|
||||
@@ -220,7 +220,11 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
created = await storeBackupBlob(
|
||||
prisma,
|
||||
() => ({ userId: owner!.id, type: uploadType }),
|
||||
{
|
||||
userId: admin.id,
|
||||
type: uploadType,
|
||||
ownerAfterRead: () => owner!.id,
|
||||
},
|
||||
async (write) => {
|
||||
// One pass: every chunk is stored as it is checked, and a check that
|
||||
// fails throws, which rolls the store back.
|
||||
@@ -336,9 +340,9 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
await denied(err.details.reason as string, err.details);
|
||||
return apiError(err.answer, err.status, err.meta);
|
||||
}
|
||||
// The file was fine; its encrypted copy is larger than one stored backup
|
||||
// may be on this host (a fifth of the heap limit). That is the operator's
|
||||
// to act on, with the numbers in the message, and not a server fault for
|
||||
// The file was fine; its encrypted copy is larger than the stored-copy
|
||||
// limit (`BACKUP_MAX_STORED_MB`). That is the operator's to act on, with
|
||||
// the numbers and the setting in the message, and not a server fault for
|
||||
// the error reporter. The store rolled back, so nothing was kept.
|
||||
if (err instanceof BackupBlobTooLargeError) {
|
||||
await denied("stored_copy_too_large", {
|
||||
|
||||
Reference in new issue
Block a user