fix(scripts): skip documents the source already says are too large once HealthLog has named its limit, and show small sizes in KB

This commit is contained in:
Marc-André Bombeck committed 2026-09-26 14:47:12 +02:00
1 parent 057281998f
commit d69e12bac2
2 files changed
+60 -5

No files matched your search

+25 -3
View File
@@ -647,6 +647,8 @@ async function upload(options, source, doc, file) {
return {
outcome: "skipped",
reason: `larger than HealthLog accepts${meta.maxFileBytes ? ` (${mib(meta.maxFileBytes)})` : ""}`,
maxFileBytes:
typeof meta.maxFileBytes === "number" ? meta.maxFileBytes : undefined,
};
case 415:
return {
@@ -668,9 +670,10 @@ async function upload(options, source, doc, file) {
function mib(bytes) {
if (typeof bytes !== "number") return "?";
return bytes >= 1024 * 1024 * 1024
? `${(bytes / 1024 ** 3).toFixed(1)} GB`
: `${(bytes / 1024 ** 2).toFixed(1)} MB`;
if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
// Under 0.1 MB a size in MB reads as 0.0; say it in KB.
if (bytes < 0.1 * 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
}
// ── Run ────────────────────────────────────────────────────────────────────
@@ -705,6 +708,8 @@ async function run(argv, env = process.env) {
};
let listed = 0;
let totalBytes = 0;
/** HealthLog's per-file limit, once a 413 has said it. */
let maxFileBytes = null;
let unknownSizes = 0;
out(
@@ -743,6 +748,20 @@ async function run(argv, env = process.env) {
continue;
}
// HealthLog says its per-file limit only when it refuses a file. Once
// it has, a document the source already says is larger is skipped
// without downloading it.
if (
maxFileBytes !== null &&
typeof doc.size === "number" &&
doc.size > maxFileBytes
) {
const reason = `larger than HealthLog accepts (${mib(maxFileBytes)})`;
tally.skipped.push({ where, reason });
out(` skipped ${where}: ${reason}`);
continue;
}
const file = await source.download(doc);
if (file.skip) {
tally.skipped.push({ where, reason: file.skip });
@@ -750,6 +769,9 @@ async function run(argv, env = process.env) {
continue;
}
const result = await upload(options, source, doc, file);
if (typeof result.maxFileBytes === "number") {
maxFileBytes = result.maxFileBytes;
}
if (result.outcome === "imported") {
tally.imported++;
out(` imported ${where}`);