Files
HealthLog/src/lib/jobs/restore-drill.ts
T
Marc-André Bombeck 7c5ff89def feat(lint): refuse UTC day keys outside the time zone module
`instant.toISOString().slice(0, 10)` and parsing a key back as UTC
midnight are the idioms behind every day-boundary fix this year: the
evening west of UTC and the small hours east of it land on the
neighbouring day. A new rule, healthlog/no-utc-day-key, refuses them in
src/ outside src/lib/tz and tests, and refuses a disable directive for
itself that is not next-line or gives no reason.

The time zone module now names every legitimate form: userDayKey for a
person's day, dateOnlyKey / dateOnlyAtNoonUtc / dayKeyAsUtcMidnight for
date-only values and day labels, daysBetweenDateKeys / weekdayOfDateKey /
isCalendarDateKey / statedDateKey for key arithmetic and parsing.

Of the 133 sites the rule found, 84 are gone. Most moved onto a named
helper with no change in behaviour; these changed behaviour:

- cycle calendar, insights, Coach snapshot and prediction refresh start
  their lookbacks at the first instant of the user's day
- a model-transcribed date like "June 10, 2026" no longer lands a day
  early in a process east of UTC (documents and lab scans)
- a custom report range covers exactly the picked days in the profile
  zone instead of starting the evening before west of UTC
- date-of-birth pickers cap at the user's today, not the UTC one
- an allergy onset date is stored at noon UTC like other date-only values
- the report's cycle summary, the Coach's usual BP range and the intake
  projector's rollup recompute use the user's day
- vaccination dates render as dates instead of the previous evening

49 remain, each with its reason on the line above: 39 UTC by design (15
file name stamps, rollup and object keys, the UTC-day score engines) and
10 marked baseline. A guard test requires a classified reason on every
directive and holds the baseline to a ceiling that only goes down.
2026-10-01 09:59:57 +02:00

261 lines
9.8 KiB
TypeScript

/**
* Monthly off-host backup restore drill (v1.16.4).
*
* A backup that has never been read back is a hope, not a backup. The
* off-host uploader (`offhost-backup.ts`) writes an encrypted snapshot
* per user every night, but nothing on the write path proves the
* objects are still fetchable, decryptable under the CURRENT
* `BACKUP_ENCRYPTION_KEY`, and structurally parseable. A silent key
* rotation mishap or a bucket-side corruption would only surface on
* the day a real restore is needed.
*
* This drill closes that gap once a month: fetch the most recent
* backup object, decrypt it, JSON-parse it, and sanity-check the
* payload shape. It deliberately performs NO database restore — the
* drill validates the artefact, not the import path. The result is
* surfaced through the wide-event meta on success and through
* `reportWorkerError` (stderr + GlitchTip) on any failure, so a broken
* backup chain pages the operator eleven months before it matters.
*
* Schedule: 04:11 on the 1st of each month (`11 4 1 * *`), after the
* 02:30 nightly upload and the 03:xx cleanup window, on a minute slot
* no other cron uses.
*/
import type { Job } from "pg-boss";
import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan";
import {
assessBackupKeys,
BackupKeyIdCollector,
describeBackupKeyProblem,
} from "@/lib/export/backup-key-ids";
import {
openBackupObject,
offhostKeyRing,
getS3Client,
loadOffhostConfig,
OffhostBackupNotConfiguredError,
type S3Like,
} from "@/lib/jobs/offhost-backup";
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
export const RESTORE_DRILL_QUEUE = "data-restore-drill";
export const RESTORE_DRILL_CRON = "11 4 1 * *";
/**
* A drill run that finds the newest backup older than this is reported
* as a failure even when decrypt + parse succeed: the nightly uploader
* has evidently stopped producing fresh objects (or the lifecycle rule
* is eating them faster than they are written).
*/
const MAX_BACKUP_AGE_DAYS = 3;
const BACKUP_KEY_PATTERN = /^(\d{4}-\d{2}-\d{2})\/user-.+\.json\.enc$/;
export interface RestoreDrillReport {
objectKey: string;
dateKey: string;
ageDays: number;
stale: boolean;
ciphertextBytes: number;
plaintextBytes: number;
/** Application key ids the object's inner ciphertext needs, each proven to open. */
innerKeyIds: string[];
recordCounts: {
measurements: number;
medications: number;
intakeEvents: number;
moodEntries: number;
};
}
function countArray(value: unknown): number {
return Array.isArray(value) ? value.length : 0;
}
/**
* Fetch → decrypt → parse the most recent off-host backup object.
*
* Read-only against the bucket (GetObject + ListObjects — both inside
* the uploader's existing IAM grant). Throws on every failure mode:
* not configured, empty bucket, fetch error, bad envelope / wrong key,
* malformed JSON, payload missing its core fields.
*/
export async function runRestoreDrill(
s3Override?: S3Like,
now: Date = new Date(),
): Promise<RestoreDrillReport> {
const cfg = loadOffhostConfig();
if (!cfg) {
throw new OffhostBackupNotConfiguredError(
"Off-host backup not configured — restore drill has nothing to verify.",
);
}
const s3 = s3Override ?? (await getS3Client(cfg));
// Date-prefixed keys (`YYYY-MM-DD/user-<id>.json.enc`) sort
// lexicographically in chronological order, so the newest object is
// simply the maximum matching key. `_healthcheck/` probes and any
// foreign objects in the bucket are filtered out by the pattern.
const objects = await s3.listObjects("");
const backupKeys = objects
.map((o) => o.key)
.filter((k) => BACKUP_KEY_PATTERN.test(k))
.sort();
if (backupKeys.length === 0) {
throw new Error(
`Restore drill found no backup objects in bucket "${cfg.bucket}" — the nightly off-host upload is not producing artefacts.`,
);
}
const objectKey = backupKeys[backupKeys.length - 1];
const dateKey = BACKUP_KEY_PATTERN.exec(objectKey)![1];
const ciphertext = await s3.getObject(objectKey);
// Read as a stream, never as one string: the JSON of a large record is
// longer than any string V8 can hold (#1031). The bulk tables are counted,
// not kept.
const source = openBackupObject(ciphertext, offhostKeyRing(cfg), objectKey);
let plaintextBytes = 0;
async function* counted() {
for await (const chunk of source()) {
plaintextBytes += chunk.byteLength;
yield chunk;
}
}
// The envelope opening proves the off-host key. It proves nothing about
// the application keys the ciphertext INSIDE was written under, and that
// is the half a key rotation breaks: a restore of this object writes those
// values back verbatim. So every inner key id is collected, and one value
// per key is actually decrypted.
const keys = new BackupKeyIdCollector();
let scanned;
try {
scanned = await scanBackupJson(counted(), {
streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]),
onElement: (key, element) => keys.visit(element, key),
});
} catch (err) {
if (err instanceof BackupJsonError) {
throw new Error(
`Restore drill: backup object "${objectKey}" decrypted but is not a JSON object: ${err.message}`,
);
}
throw err;
}
const payload = scanned.document;
const streamedCounts = scanned.streamedCounts;
if (
typeof payload.exportedAt !== "string" ||
typeof payload.userId !== "string" ||
streamedCounts.measurements === undefined
) {
throw new Error(
`Restore drill: backup object "${objectKey}" parses but is missing core fields (exportedAt / userId / measurements).`,
);
}
keys.visit(payload);
const keyVerdict = assessBackupKeys(keys, {
ignoreSections: new Set(["appSettings"]),
});
const keyProblem = describeBackupKeyProblem(keyVerdict);
if (keyProblem) {
throw new Error(
`Restore drill: backup object "${objectKey}" opens, but its content could not be restored on this server. ${keyProblem}`,
);
}
const ageDays = Math.floor(
// eslint-disable-next-line healthlog/no-utc-day-key -- UTC by design: age of a UTC-dated off-host object key
(now.getTime() - Date.parse(`${dateKey}T00:00:00Z`)) /
(24 * 60 * 60 * 1000),
);
return {
objectKey,
dateKey,
ageDays,
stale: ageDays > MAX_BACKUP_AGE_DAYS,
ciphertextBytes: ciphertext.length,
plaintextBytes,
innerKeyIds: keyVerdict.keyIds,
recordCounts: {
measurements: streamedCounts.measurements,
medications: countArray(payload.medications),
intakeEvents: streamedCounts.intakeEvents ?? 0,
moodEntries: streamedCounts.moodEntries ?? 0,
},
};
}
/**
* pg-boss handler. Mirrors the off-host uploader's posture: a
* not-configured deployment skips with a wide-event warning (most
* self-hosters never set the S3 vars and must not see a monthly error
* page); every other failure goes through `reportWorkerError` so the
* operator hears about a rotting backup chain.
*/
export async function handleRestoreDrill(
jobs: Job<object>[],
): Promise<JobOutcome> {
void jobs;
return withBackgroundEvent("job.restore_drill", async (evt) => {
try {
const report = await runRestoreDrill();
evt.addMeta("restore_drill_object_key", report.objectKey);
evt.addMeta("restore_drill_age_days", report.ageDays);
evt.addMeta("restore_drill_ciphertext_bytes", report.ciphertextBytes);
evt.addMeta("restore_drill_plaintext_bytes", report.plaintextBytes);
evt.addMeta(
"restore_drill_measurements",
report.recordCounts.measurements,
);
evt.addMeta("restore_drill_medications", report.recordCounts.medications);
evt.addMeta(
"restore_drill_intake_events",
report.recordCounts.intakeEvents,
);
evt.addMeta(
"restore_drill_mood_entries",
report.recordCounts.moodEntries,
);
evt.addMeta("restore_drill_stale", report.stale);
evt.addMeta("restore_drill_inner_key_ids", report.innerKeyIds.join(","));
if (report.stale) {
await reportWorkerError(
RESTORE_DRILL_QUEUE,
new Error(
`Newest off-host backup is ${report.ageDays} days old (threshold ${MAX_BACKUP_AGE_DAYS}) — the nightly upload chain has stalled.`,
),
{ objectKey: report.objectKey, ageDays: report.ageDays },
);
}
// A stale chain is a verdict about the uploader, not a failure of the
// drill: the artefact was fetched, decrypted and parsed, and the page
// above already reached the operator. Failing the job here would retry a
// reading that cannot change until the next upload lands.
return jobDone({
restore_drill_age_days: report.ageDays,
restore_drill_ciphertext_bytes: report.ciphertextBytes,
restore_drill_plaintext_bytes: report.plaintextBytes,
restore_drill_measurements: report.recordCounts.measurements,
restore_drill_medications: report.recordCounts.medications,
restore_drill_intake_events: report.recordCounts.intakeEvents,
restore_drill_mood_entries: report.recordCounts.moodEntries,
restore_drill_stale: report.stale,
});
} catch (err) {
if (err instanceof OffhostBackupNotConfiguredError) {
evt.addWarning(`restore-drill skipped: ${err.message}`);
// Most self-hosters never set the S3 vars. Nothing to verify is not a
// failed verification, and a monthly failed job would be noise.
return jobDone({ skipped: "offhost_backup_not_configured" });
}
evt.addWarning(`restore-drill failed: ${err}`);
await reportWorkerError(RESTORE_DRILL_QUEUE, err);
return jobFailed("restore drill failed", err);
}
});
}