mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 15:23:29 +00:00
feat(backup): restore a dump straight from the off-host bucket
A lost host takes its local dumps with it, and the copy left is the one in the bucket. `docker compose run --rm db-restore latest --from-bucket` now lists `database/` in the bucket the BACKUP_S3_* variables name, takes the newest dump whose manifest is there too (or the one named), downloads both with the S3 client the image already ships, and continues as for any off-host copy: opened with BACKUP_ENCRYPTION_KEY, checked against the manifest, restored into an empty database. `list --from-bucket` shows what is there. The sealed download is removed once it is opened. The image build now proves the S3 client resolves from the shared module's place in the tree, so the restore cannot lose it unnoticed.
This commit is contained in:
1 parent
1a4289545e
commit
82db3de30a
6 files changed
+298
-10
No files matched your search
@@ -25,6 +25,8 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
applyRetention,
|
||||
bucketConfigFromEnv,
|
||||
pickBucketDump,
|
||||
buildManifest,
|
||||
compareKeyFingerprints,
|
||||
compareRowCounts,
|
||||
@@ -628,3 +630,60 @@ describe("an off-host copy read back by the restore", () => {
|
||||
expect(await readdir(dir)).toEqual([name]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("finding a dump in the bucket", () => {
|
||||
const keys = [
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261023T013000Z-daily.manifest.json.enc",
|
||||
"database/healthlog-20261024T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
|
||||
// The newest dump whose manifest never arrived, and things that are
|
||||
// not dumps at all.
|
||||
"database/healthlog-20261025T013000Z-daily.dump.enc",
|
||||
"2026-10-25/user-abc.json.enc",
|
||||
"database/notes.txt",
|
||||
];
|
||||
|
||||
it("takes the newest dump whose manifest is there too", () => {
|
||||
expect(pickBucketDump(keys)).toEqual({
|
||||
dumpKey: "database/healthlog-20261024T013000Z-daily.dump.enc",
|
||||
manifestKey:
|
||||
"database/healthlog-20261024T013000Z-daily.manifest.json.enc",
|
||||
});
|
||||
});
|
||||
|
||||
it("finds a named one with or without its suffix, and nothing else", () => {
|
||||
for (const name of [
|
||||
"healthlog-20261023T013000Z-daily.dump",
|
||||
"healthlog-20261023T013000Z-daily.dump.enc",
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
]) {
|
||||
expect(pickBucketDump(keys, name)?.dumpKey).toBe(
|
||||
"database/healthlog-20261023T013000Z-daily.dump.enc",
|
||||
);
|
||||
}
|
||||
expect(
|
||||
pickBucketDump(keys, "healthlog-20261025T013000Z-daily.dump"),
|
||||
).toBeNull();
|
||||
expect(pickBucketDump([])).toBeNull();
|
||||
});
|
||||
|
||||
it("reads the bucket from the four required variables", () => {
|
||||
const env = {
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.test",
|
||||
BACKUP_S3_BUCKET: "b",
|
||||
BACKUP_S3_ACCESS_KEY: "a",
|
||||
BACKUP_S3_SECRET_KEY: "s",
|
||||
};
|
||||
expect(bucketConfigFromEnv(env)).toEqual({
|
||||
endpoint: "https://s3.example.test",
|
||||
bucket: "b",
|
||||
accessKey: "a",
|
||||
secretKey: "s",
|
||||
region: "auto",
|
||||
});
|
||||
expect(
|
||||
bucketConfigFromEnv({ ...env, BACKUP_S3_SECRET_KEY: "" }),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1127,6 +1127,140 @@ export function offhostKeysFromEnv(env = process.env) {
|
||||
.map((k) => /** @type {Buffer} */ (k));
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {{
|
||||
* endpoint: string;
|
||||
* bucket: string;
|
||||
* accessKey: string;
|
||||
* secretKey: string;
|
||||
* region: string;
|
||||
* }} BucketConfig
|
||||
*/
|
||||
|
||||
/**
|
||||
* The off-host bucket an environment names (`BACKUP_S3_*`), or null when one
|
||||
* of the four required variables is missing.
|
||||
*
|
||||
* @param {Env} env
|
||||
* @returns {BucketConfig | null}
|
||||
*/
|
||||
export function bucketConfigFromEnv(env = process.env) {
|
||||
const value = (/** @type {string} */ name) => (env[name] ?? "").trim();
|
||||
const config = {
|
||||
endpoint: value("BACKUP_S3_ENDPOINT"),
|
||||
bucket: value("BACKUP_S3_BUCKET"),
|
||||
accessKey: value("BACKUP_S3_ACCESS_KEY"),
|
||||
secretKey: value("BACKUP_S3_SECRET_KEY"),
|
||||
region: value("BACKUP_S3_REGION") || "auto",
|
||||
};
|
||||
return config.endpoint &&
|
||||
config.bucket &&
|
||||
config.accessKey &&
|
||||
config.secretKey
|
||||
? config
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The newest database dump in a bucket listing whose manifest is there too,
|
||||
* or the one `name` asks for. Pure. `name` may carry the `.enc` suffix or
|
||||
* not. Answers both object keys, or null.
|
||||
*
|
||||
* @param {string[]} keys every key under the `database/` prefix
|
||||
* @param {string | null} name
|
||||
* @returns {{ dumpKey: string; manifestKey: string } | null}
|
||||
*/
|
||||
export function pickBucketDump(keys, name = null) {
|
||||
const present = new Set(keys);
|
||||
const wanted = name
|
||||
? name.replace(/\.enc$/, "").replace(/^database\//, "")
|
||||
: null;
|
||||
const candidates = keys
|
||||
.map((key) => ({
|
||||
key,
|
||||
parsed: parseDumpFileName(key.slice(OFFHOST_DATABASE_PREFIX.length)),
|
||||
}))
|
||||
.filter(
|
||||
({ key, parsed }) =>
|
||||
key.startsWith(OFFHOST_DATABASE_PREFIX) &&
|
||||
parsed !== null &&
|
||||
parsed.role === "dump" &&
|
||||
parsed.variant === "encrypted" &&
|
||||
(wanted === null || `${parsed.base}.dump` === wanted),
|
||||
)
|
||||
.map(({ parsed }) => {
|
||||
const p = /** @type {ParsedDumpName} */ (parsed);
|
||||
return {
|
||||
timestamp: p.timestamp.getTime(),
|
||||
dumpKey: offhostObjectKey(`${p.base}.dump.enc`),
|
||||
manifestKey: offhostObjectKey(`${p.base}.manifest.json.enc`),
|
||||
};
|
||||
})
|
||||
.filter((c) => present.has(c.manifestKey))
|
||||
.sort((a, b) => b.timestamp - a.timestamp);
|
||||
return candidates[0]
|
||||
? { dumpKey: candidates[0].dumpKey, manifestKey: candidates[0].manifestKey }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A minimal reader of the off-host bucket for the restore: list a prefix and
|
||||
* download an object to a file, streamed. Uses the AWS SDK the app ships
|
||||
* (path-style requests, as the app's own uploader does).
|
||||
*
|
||||
* @param {BucketConfig} config
|
||||
*/
|
||||
export async function openBucket(config) {
|
||||
const sdk = await import("@aws-sdk/client-s3");
|
||||
const client = new sdk.S3Client({
|
||||
endpoint: config.endpoint,
|
||||
region: config.region,
|
||||
forcePathStyle: true,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKey,
|
||||
secretAccessKey: config.secretKey,
|
||||
},
|
||||
});
|
||||
return {
|
||||
/** @param {string} prefix */
|
||||
async list(prefix) {
|
||||
/** @type {string[]} */
|
||||
const keys = [];
|
||||
/** @type {string | undefined} */
|
||||
let token;
|
||||
for (let page = 0; page < 200; page++) {
|
||||
const out = await client.send(
|
||||
new sdk.ListObjectsV2Command({
|
||||
Bucket: config.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: token,
|
||||
}),
|
||||
);
|
||||
for (const c of out.Contents ?? []) if (c.Key) keys.push(c.Key);
|
||||
if (!out.IsTruncated || !out.NextContinuationToken) return keys;
|
||||
token = out.NextContinuationToken;
|
||||
}
|
||||
throw new Error(`the bucket listing under ${prefix} did not end`);
|
||||
},
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {string} outPath
|
||||
*/
|
||||
async download(key, outPath) {
|
||||
const out = await client.send(
|
||||
new sdk.GetObjectCommand({ Bucket: config.bucket, Key: key }),
|
||||
);
|
||||
if (!out.Body) throw new Error(`the bucket returned no body for ${key}`);
|
||||
await pipeline(
|
||||
/** @type {NodeJS.ReadableStream} */ (
|
||||
/** @type {unknown} */ (out.Body)
|
||||
),
|
||||
createWriteStream(outPath, { flags: "wx", mode: 0o600 }),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Restore ────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user