mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 20:42:43 +00:00
Ask the newest values before the key check refuses
The probe decided on the eight oldest values alone. An install reinstalled under a fresh key long ago holds old rows sealed under a key that is gone and all current data under the key it runs with; it works today, and the check would refuse it with 503 on every request. When every oldest value fails, the newest values of the same columns are now sampled in reverse order. One that opens makes the verdict inconclusive (mixed): the process serves, warns and probes again at the next start. Only old and newest values that all fail are a mismatch. A newest-only open never seals; sealing stays with the oldest-first rule. Every probe query now runs in its own transaction under a transaction-local statement_timeout (2 s), so a large or locked table ends that column with an error and the probe as incomplete instead of holding up the boot. The newest order is the exact mirror of the oldest, so an index on the timestamp serves both directions. The module comment lists the three conditions that still yield a mismatch.
This commit is contained in:
1 parent
2c2e3fcc72
commit
2f3da10317
4 files changed
+426
-58
No files matched your search
@@ -25,8 +25,9 @@ const INCONCLUSIVE_EXPLANATION: Record<InconclusiveReason, string> = {
|
||||
"single-value": "the only stored value found under that id did not open.",
|
||||
mixed:
|
||||
"the oldest stored values under that id do not open, newer ones do. " +
|
||||
"That is what a wrong key that served for a while leaves behind; check " +
|
||||
"that this is the key the database was first written with.",
|
||||
"Either the old rows were sealed under a key that is gone, or a wrong " +
|
||||
"key served for a while and wrote the newer ones; check that this is " +
|
||||
"the key the database was first written with.",
|
||||
unsampled:
|
||||
"the values stored under that id are too large to probe at start-up.",
|
||||
incomplete:
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
keyMismatchLogBlock,
|
||||
probeValueOpens,
|
||||
resolveColumnLocation,
|
||||
sqlColumnSampler,
|
||||
type CanaryClient,
|
||||
type ColumnSampler,
|
||||
} from "../canary";
|
||||
@@ -62,6 +63,8 @@ function fakeClient(opts: {
|
||||
probe?: Record<string, Seeded[]>;
|
||||
/** Make the sampler throw for these columns. */
|
||||
unreadable?: string[];
|
||||
/** Make the sampler throw for every column when asked for its newest values. */
|
||||
newestUnreadable?: boolean;
|
||||
/** Another process seals this id (with this ciphertext) just before us. */
|
||||
race?: { keyId: string; ciphertext: string };
|
||||
}) {
|
||||
@@ -92,9 +95,12 @@ function fakeClient(opts: {
|
||||
return 1;
|
||||
},
|
||||
} as unknown as CanaryClient;
|
||||
const sampler: ColumnSampler = async (column, keyId, limit) => {
|
||||
const sampler: ColumnSampler = async (column, keyId, limit, order) => {
|
||||
const key = `${column.model}.${column.field}`;
|
||||
if (opts.unreadable?.includes(key)) throw new Error("relation missing");
|
||||
if (opts.newestUnreadable && order === "newest") {
|
||||
throw new Error("canceling statement due to statement timeout");
|
||||
}
|
||||
const rows = (opts.probe?.[key] ?? []).map((v) =>
|
||||
typeof v === "string" || v instanceof Uint8Array
|
||||
? { value: v, ts: null as Date | null, codec: null }
|
||||
@@ -113,15 +119,15 @@ function fakeClient(opts: {
|
||||
extractKeyIdFromBytes(b) === keyId
|
||||
);
|
||||
});
|
||||
return under
|
||||
const oldestFirst = under
|
||||
.map((r, i) => ({ r, i }))
|
||||
.sort((a, b) => {
|
||||
const ta = a.r.ts?.getTime() ?? Infinity;
|
||||
const tb = b.r.ts?.getTime() ?? Infinity;
|
||||
return ta === tb ? a.i - b.i : ta - tb;
|
||||
})
|
||||
.slice(0, limit)
|
||||
.map(({ r }) => r);
|
||||
});
|
||||
if (order === "newest") oldestFirst.reverse();
|
||||
return oldestFirst.slice(0, limit).map(({ r }) => r);
|
||||
};
|
||||
const check = (budget?: number) =>
|
||||
checkEncryptionKeyCanaries(client, { sampler, probeBudgetMs: budget });
|
||||
@@ -369,6 +375,103 @@ describe("encryption key canary", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the newest values are asked before a refusal", () => {
|
||||
/**
|
||||
* Eight old values (two in each of four columns, so the oldest-first
|
||||
* pool holds nothing else) sealed under `oldKey`, and two newer values
|
||||
* per column sealed under `newKey`. Returns the fake, configured with
|
||||
* KEY_B.
|
||||
*/
|
||||
function seedOldThenNew(oldKey: string, newKey: string) {
|
||||
configureKey(oldKey);
|
||||
const old = [
|
||||
encrypt("old 1"),
|
||||
encrypt("old 2"),
|
||||
encrypt("old 3"),
|
||||
encrypt("old 4"),
|
||||
encryptToBytes("old note 1"),
|
||||
encryptToBytes("old note 2"),
|
||||
encryptToBytes("old note 3"),
|
||||
encryptToBytes("old note 4"),
|
||||
];
|
||||
configureKey(newKey);
|
||||
const recent = [
|
||||
encrypt("new 1"),
|
||||
encrypt("new 2"),
|
||||
encrypt("new 3"),
|
||||
encrypt("new 4"),
|
||||
encryptToBytes("new note 1"),
|
||||
encryptToBytes("new note 2"),
|
||||
encryptToBytes("new note 3"),
|
||||
encryptToBytes("new note 4"),
|
||||
];
|
||||
configureKey(KEY_B);
|
||||
const columns = [
|
||||
"User.codexAccessTokenEncrypted",
|
||||
"User.codexRefreshTokenEncrypted",
|
||||
"MoodEntry.noteEncrypted",
|
||||
"Measurement.notesEncrypted",
|
||||
];
|
||||
const probe: Record<string, Seeded[]> = {};
|
||||
columns.forEach((key, i) => {
|
||||
probe[key] = [
|
||||
{ value: recent[2 * i], ts: `2026-09-0${i + 1}` },
|
||||
{ value: old[2 * i], ts: `2023-0${i + 1}-01` },
|
||||
{ value: recent[2 * i + 1], ts: `2026-09-1${i + 1}` },
|
||||
{ value: old[2 * i + 1], ts: `2023-0${i + 1}-02` },
|
||||
];
|
||||
});
|
||||
return probe;
|
||||
}
|
||||
|
||||
it("old rows under a lost key and current data under the configured key: mixed, not a mismatch", async () => {
|
||||
// A database reinstalled under a fresh key long ago: the old rows are
|
||||
// sealed under a key nobody has any more, and everything written since
|
||||
// opens under the configured one. Refusing would take a working
|
||||
// install down on upgrade.
|
||||
const { inserts, canaries, check } = fakeClient({
|
||||
probe: seedOldThenNew(KEY_A, KEY_B),
|
||||
});
|
||||
expect(await check()).toEqual(
|
||||
ok({ inconclusive: [{ keyId: "v1", reason: "mixed" }] }),
|
||||
);
|
||||
// A newest-only open never seals.
|
||||
expect(inserts).toEqual([]);
|
||||
expect(canaries.size).toBe(0);
|
||||
});
|
||||
|
||||
it("old rows under A and newer rows under B, configured B, is never sealed or verified", async () => {
|
||||
const { inserts, check } = fakeClient({
|
||||
probe: seedOldThenNew(KEY_A, KEY_B),
|
||||
});
|
||||
const outcome = await check();
|
||||
expect(outcome.state).toBe("ok");
|
||||
if (outcome.state !== "ok") return;
|
||||
expect(outcome.written).toEqual([]);
|
||||
expect(outcome.verified).toEqual([]);
|
||||
expect(inserts).toEqual([]);
|
||||
});
|
||||
|
||||
it("old AND newest values that all fail are a mismatch", async () => {
|
||||
const { inserts, check } = fakeClient({
|
||||
probe: seedOldThenNew(KEY_A, KEY_A),
|
||||
});
|
||||
expect(await check()).toEqual({ state: "mismatch", keyIds: ["v1"] });
|
||||
expect(inserts).toEqual([]);
|
||||
});
|
||||
|
||||
it("the newest values that cannot be read are inconclusive, never a mismatch", async () => {
|
||||
const { inserts, check } = fakeClient({
|
||||
probe: seedOldThenNew(KEY_A, KEY_A),
|
||||
newestUnreadable: true,
|
||||
});
|
||||
expect(await check()).toEqual(
|
||||
ok({ inconclusive: [{ keyId: "v1", reason: "incomplete" }] }),
|
||||
);
|
||||
expect(inserts).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("nothing unproven seals", () => {
|
||||
it("a probe out of time is inconclusive, even over data that opens", async () => {
|
||||
const { inserts, check } = fakeClient({
|
||||
@@ -518,6 +621,86 @@ describe("the SQL sampler's column map", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the SQL sampler's bounds", () => {
|
||||
beforeEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
configureKey(KEY_A);
|
||||
});
|
||||
|
||||
/** A client whose probe queries the server cancels at the timeout. */
|
||||
function timingOutClient() {
|
||||
const model = new PrismaClient({
|
||||
adapter: new PrismaPg({
|
||||
connectionString: "postgres://x:y@127.0.0.1:1/x",
|
||||
}),
|
||||
}) as unknown as { _runtimeDataModel: unknown };
|
||||
const settings: unknown[][] = [];
|
||||
const statements: string[] = [];
|
||||
const transactionOptions: unknown[] = [];
|
||||
const tx = {
|
||||
async $executeRaw(_q: TemplateStringsArray, ...values: unknown[]) {
|
||||
settings.push(values);
|
||||
return 1;
|
||||
},
|
||||
async $queryRawUnsafe(sql: string) {
|
||||
statements.push(sql);
|
||||
throw new Error("canceling statement due to statement timeout");
|
||||
},
|
||||
};
|
||||
const client = {
|
||||
_runtimeDataModel: model._runtimeDataModel,
|
||||
$queryRaw: async () => [],
|
||||
$executeRaw: async () => {
|
||||
throw new Error("nothing may be sealed");
|
||||
},
|
||||
$queryRawUnsafe: async () => [],
|
||||
async $transaction(
|
||||
fn: (t: typeof tx) => Promise<unknown>,
|
||||
options: unknown,
|
||||
) {
|
||||
transactionOptions.push(options);
|
||||
return fn(tx);
|
||||
},
|
||||
} as unknown as CanaryClient;
|
||||
return { client, settings, statements, transactionOptions };
|
||||
}
|
||||
|
||||
it("a probe query cancelled at its statement timeout is inconclusive, not a hang or a mismatch", async () => {
|
||||
const fake = timingOutClient();
|
||||
expect(await checkEncryptionKeyCanaries(fake.client)).toEqual(
|
||||
ok({ inconclusive: [{ keyId: "v1", reason: "incomplete" }] }),
|
||||
);
|
||||
// Every probe query runs under a transaction-local statement timeout.
|
||||
expect(fake.settings).toEqual([["2000"]]);
|
||||
expect(fake.statements).toHaveLength(1);
|
||||
expect(fake.transactionOptions[0]).toMatchObject({ timeout: 7_000 });
|
||||
});
|
||||
|
||||
it("reads the newest values in the exact mirror of the oldest order", async () => {
|
||||
const fake = timingOutClient();
|
||||
const sampler = sqlColumnSampler(fake.client, { statementTimeoutMs: 50 });
|
||||
const user = ENCRYPTED_COLUMNS.find((c) => c.model === "User")!;
|
||||
await expect(sampler(user, "v1", 2, "oldest")).rejects.toThrow();
|
||||
await expect(sampler(user, "v1", 2, "newest")).rejects.toThrow();
|
||||
expect(fake.settings).toEqual([["50"], ["50"]]);
|
||||
expect(fake.statements[0]).toContain(
|
||||
'ORDER BY "updated_at" ASC NULLS LAST, "id" ASC LIMIT',
|
||||
);
|
||||
expect(fake.statements[1]).toContain(
|
||||
'ORDER BY "updated_at" DESC NULLS FIRST, "id" DESC LIMIT',
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses to run without transactions rather than run unbounded", () => {
|
||||
const client = {
|
||||
$queryRaw: async () => [],
|
||||
$executeRaw: async () => 0,
|
||||
$queryRawUnsafe: async () => [],
|
||||
} as unknown as CanaryClient;
|
||||
expect(() => sqlColumnSampler(client)).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("key fingerprint and copy check", () => {
|
||||
beforeEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
|
||||
+161
-51
@@ -49,7 +49,8 @@
|
||||
* problem: a wrong key opens nothing. Values newer than the first opening
|
||||
* one do not count against it; they are what a wrong-key process would
|
||||
* have left behind. The canary is written.
|
||||
* - `fails`: two or more values were tried and none opened. That is what a
|
||||
* - `fails`: two or more of the oldest values were tried and none opened,
|
||||
* AND none of the newest values opens either (see below). That is what a
|
||||
* wrong key looks like; the process refuses (mismatch).
|
||||
* - `none`: every column answered, and none holds a single value under the
|
||||
* key id. A genuinely fresh install (or a new id added for rotation); the
|
||||
@@ -64,6 +65,41 @@
|
||||
* server down with 503 on every request; not sealing because that would
|
||||
* lock the right key out later.
|
||||
*
|
||||
* Before the oldest values may refuse, the newest are asked: the same columns,
|
||||
* the same `PROBE_ROWS_PER_COLUMN`, in reverse order, and the newest
|
||||
* `PROBE_SAMPLE_LIMIT` of those tried. If any opens, the verdict is
|
||||
* `inconclusive` (`mixed`), never `fails`. An install reinstalled under a
|
||||
* fresh key long ago holds old rows sealed under a key that is gone and all
|
||||
* current data under the key it runs with; it works today, and refusing it
|
||||
* would take it down on upgrade. The newest values only ever turn a refusal
|
||||
* into a warning. They never seal: a newest-only open is exactly what a wrong
|
||||
* key that served for a while leaves behind, so sealing stays with the
|
||||
* oldest-first rule above. A newest pass that cannot finish (time budget,
|
||||
* unreadable column) is `incomplete`, not `fails`.
|
||||
*
|
||||
* Every probe query is bounded on its own: it runs in a transaction with
|
||||
* `SET LOCAL statement_timeout` (`PROBE_STATEMENT_TIMEOUT_MS`), so one large
|
||||
* or locked table ends that column with an error, and the probe with
|
||||
* `incomplete`, instead of holding up the boot. The `ORDER BY` is the plain
|
||||
* timestamp-then-key order (and its exact mirror for the newest pass), which
|
||||
* a btree index on the timestamp serves in either direction; the primary key
|
||||
* is always indexed. The time budget is checked between columns.
|
||||
*
|
||||
* ## When the outcome is `mismatch`
|
||||
*
|
||||
* Exactly three ways, and no other:
|
||||
*
|
||||
* 1. a stored canary for a configured key id does not open under that key
|
||||
* (or opens to another value);
|
||||
* 2. no canary, and the probe says `fails`: at least two of the oldest
|
||||
* values were tried and none opened, and none of the newest values
|
||||
* opened either;
|
||||
* 3. the first seal was lost to another process and the canary it wrote
|
||||
* does not open under this process's key.
|
||||
*
|
||||
* Everything else (an unreadable or timed-out column, an exhausted budget,
|
||||
* a single value, mixed old and new) serves.
|
||||
*
|
||||
* The first seal can race another process. `ON CONFLICT DO NOTHING` keeps the
|
||||
* row the other process wrote; when this process's insert affected no row,
|
||||
* the stored canary is read back and must open under this process's key, or
|
||||
@@ -110,6 +146,9 @@ export const PROBE_VALUE_MAX_BYTES = 2 * 1024 * 1024;
|
||||
/** How long the probe may take before it gives up as `inconclusive`. */
|
||||
export const PROBE_BUDGET_MS = 10_000;
|
||||
|
||||
/** How long one probe query may run before the server cancels it. */
|
||||
export const PROBE_STATEMENT_TIMEOUT_MS = 2_000;
|
||||
|
||||
/** The key id the legacy bare-base64 string format belongs to. */
|
||||
const LEGACY_KEY_ID = "v1";
|
||||
|
||||
@@ -128,6 +167,19 @@ export interface CanaryClient {
|
||||
...values: unknown[]
|
||||
): Promise<T>;
|
||||
$executeRawUnsafe?(query: string, ...values: unknown[]): Promise<number>;
|
||||
$transaction?<R>(
|
||||
fn: (tx: ProbeTransaction) => Promise<R>,
|
||||
options?: { maxWait?: number; timeout?: number },
|
||||
): Promise<R>;
|
||||
}
|
||||
|
||||
/** The slice of a transaction client one bounded probe query needs. */
|
||||
export interface ProbeTransaction {
|
||||
$executeRaw(
|
||||
query: TemplateStringsArray,
|
||||
...values: unknown[]
|
||||
): Promise<number>;
|
||||
$queryRawUnsafe<T = unknown>(query: string, ...values: unknown[]): Promise<T>;
|
||||
}
|
||||
|
||||
/** One stored value under the key id, as the probe sees it. */
|
||||
@@ -140,15 +192,19 @@ export interface ProbeRow {
|
||||
ts: Date | null;
|
||||
}
|
||||
|
||||
/** Which end of a column's write-time order a sample is read from. */
|
||||
export type SampleOrder = "oldest" | "newest";
|
||||
|
||||
/**
|
||||
* Reads up to `limit` of the oldest values under `keyId` in one column, oldest
|
||||
* first. Throws when the column cannot be read; an empty array means the
|
||||
* column holds no value under that id.
|
||||
* Reads up to `limit` of the oldest (or newest) values under `keyId` in one
|
||||
* column, in that order. Throws when the column cannot be read; an empty
|
||||
* array means the column holds no value under that id.
|
||||
*/
|
||||
export type ColumnSampler = (
|
||||
column: EncryptedColumn,
|
||||
keyId: string,
|
||||
limit: number,
|
||||
order: SampleOrder,
|
||||
) => Promise<ProbeRow[]>;
|
||||
|
||||
export type InconclusiveReason =
|
||||
@@ -225,6 +281,46 @@ export function probeValueOpens(
|
||||
}
|
||||
}
|
||||
|
||||
type Pooled = { column: EncryptedColumn; row: ProbeRow; order: number };
|
||||
|
||||
/**
|
||||
* One value pool from every column, in `order`. Null when the budget ran out
|
||||
* or a column could not be read.
|
||||
*/
|
||||
async function samplePool(
|
||||
sampler: ColumnSampler,
|
||||
keyId: string,
|
||||
deadline: number,
|
||||
columns: readonly EncryptedColumn[],
|
||||
order: SampleOrder,
|
||||
): Promise<{ pool: Pooled[]; present: number } | null> {
|
||||
const pool: Pooled[] = [];
|
||||
let present = 0;
|
||||
for (const column of columns) {
|
||||
if (Date.now() > deadline) return null;
|
||||
let rows: ProbeRow[];
|
||||
try {
|
||||
rows = await sampler(column, keyId, PROBE_ROWS_PER_COLUMN, order);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
present += rows.length;
|
||||
for (const row of rows) {
|
||||
if (row.value !== null) pool.push({ column, row, order: pool.length });
|
||||
}
|
||||
}
|
||||
if (Date.now() > deadline) return null;
|
||||
// Oldest first; rows without a timestamp after every row with one. The
|
||||
// newest pass is the exact mirror.
|
||||
const sign = order === "oldest" ? 1 : -1;
|
||||
pool.sort((a, b) => {
|
||||
const ta = a.row.ts?.getTime() ?? Number.POSITIVE_INFINITY;
|
||||
const tb = b.row.ts?.getTime() ?? Number.POSITIVE_INFINITY;
|
||||
return ta === tb ? a.order - b.order : sign * (ta - tb);
|
||||
});
|
||||
return { pool, present };
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe the stored values under `keyId`; see the module comment for what each
|
||||
* verdict means and why.
|
||||
@@ -235,40 +331,14 @@ export async function probeExistingData(
|
||||
deadline: number,
|
||||
columns: readonly EncryptedColumn[] = ENCRYPTED_COLUMNS,
|
||||
): Promise<ProbeResult> {
|
||||
const pool: Array<{ column: EncryptedColumn; row: ProbeRow; order: number }> =
|
||||
[];
|
||||
let present = 0;
|
||||
for (const column of columns) {
|
||||
if (Date.now() > deadline) {
|
||||
return { verdict: "inconclusive", reason: "incomplete" };
|
||||
}
|
||||
let rows: ProbeRow[];
|
||||
try {
|
||||
rows = await sampler(column, keyId, PROBE_ROWS_PER_COLUMN);
|
||||
} catch {
|
||||
return { verdict: "inconclusive", reason: "incomplete" };
|
||||
}
|
||||
present += rows.length;
|
||||
for (const row of rows) {
|
||||
if (row.value !== null) pool.push({ column, row, order: pool.length });
|
||||
}
|
||||
}
|
||||
if (Date.now() > deadline) {
|
||||
return { verdict: "inconclusive", reason: "incomplete" };
|
||||
}
|
||||
if (present === 0) return { verdict: "none" };
|
||||
if (pool.length === 0) {
|
||||
const oldest = await samplePool(sampler, keyId, deadline, columns, "oldest");
|
||||
if (oldest === null) return { verdict: "inconclusive", reason: "incomplete" };
|
||||
if (oldest.present === 0) return { verdict: "none" };
|
||||
if (oldest.pool.length === 0) {
|
||||
return { verdict: "inconclusive", reason: "unsampled" };
|
||||
}
|
||||
|
||||
// Oldest first; rows without a timestamp after every row with one.
|
||||
pool.sort((a, b) => {
|
||||
const ta = a.row.ts?.getTime() ?? Number.POSITIVE_INFINITY;
|
||||
const tb = b.row.ts?.getTime() ?? Number.POSITIVE_INFINITY;
|
||||
return ta === tb ? a.order - b.order : ta - tb;
|
||||
});
|
||||
const tried = pool.slice(0, PROBE_SAMPLE_LIMIT);
|
||||
|
||||
const tried = oldest.pool.slice(0, PROBE_SAMPLE_LIMIT);
|
||||
let failedBeforeFirstOpen = 0;
|
||||
for (const { column, row } of tried) {
|
||||
if (probeValueOpens(column, row, keyId)) {
|
||||
@@ -278,9 +348,21 @@ export async function probeExistingData(
|
||||
}
|
||||
failedBeforeFirstOpen += 1;
|
||||
}
|
||||
return tried.length >= 2
|
||||
? { verdict: "fails" }
|
||||
: { verdict: "inconclusive", reason: "single-value" };
|
||||
if (tried.length < 2) {
|
||||
return { verdict: "inconclusive", reason: "single-value" };
|
||||
}
|
||||
|
||||
// Every oldest value failed. Before refusing, ask the newest: one that
|
||||
// opens means the configured key is the one the install writes with today.
|
||||
// It never seals; it only keeps a working install serving.
|
||||
const newest = await samplePool(sampler, keyId, deadline, columns, "newest");
|
||||
if (newest === null) return { verdict: "inconclusive", reason: "incomplete" };
|
||||
for (const { column, row } of newest.pool.slice(0, PROBE_SAMPLE_LIMIT)) {
|
||||
if (probeValueOpens(column, row, keyId)) {
|
||||
return { verdict: "inconclusive", reason: "mixed" };
|
||||
}
|
||||
}
|
||||
return { verdict: "fails" };
|
||||
}
|
||||
|
||||
// ─── The SQL sampler ────────────────────────────────────────────────────────
|
||||
@@ -359,6 +441,7 @@ function columnQuery(
|
||||
column: EncryptedColumn,
|
||||
keyId: string,
|
||||
mode: "sample" | "exists",
|
||||
order: SampleOrder = "oldest",
|
||||
): { sql: string; params: unknown[] } {
|
||||
if (!KEY_ID.test(keyId)) throw new Error("Malformed key id");
|
||||
const c = quoteIdentifier(loc.column);
|
||||
@@ -405,11 +488,16 @@ function columnQuery(
|
||||
const ts = loc.ts ? quoteIdentifier(loc.ts) : null;
|
||||
const codec = loc.codec ? quoteIdentifier(loc.codec) : "NULL";
|
||||
const max = Math.floor(PROBE_VALUE_MAX_BYTES);
|
||||
// The newest order is the exact mirror of the oldest, so an index on the
|
||||
// timestamp serves both (a backward scan for the newest).
|
||||
const [dir, nulls] =
|
||||
order === "oldest" ? ["ASC", "NULLS LAST"] : ["DESC", "NULLS FIRST"];
|
||||
const pk = quoteIdentifier(loc.pk);
|
||||
const sql =
|
||||
`SELECT CASE WHEN octet_length(${c}) <= ${max} THEN ${c} END AS value, ` +
|
||||
`${codec} AS codec, ${ts ?? "NULL::timestamp"} AS ts ` +
|
||||
`FROM ${table} WHERE ${where} ` +
|
||||
`ORDER BY ${ts ? `${ts} ASC NULLS LAST, ` : ""}${quoteIdentifier(loc.pk)} ASC ` +
|
||||
`ORDER BY ${ts ? `${ts} ${dir} ${nulls}, ` : ""}${pk} ${dir} ` +
|
||||
`LIMIT ${bind(0)}`;
|
||||
return { sql, params };
|
||||
}
|
||||
@@ -422,20 +510,42 @@ function requireUnsafe(client: CanaryClient) {
|
||||
return query.bind(client);
|
||||
}
|
||||
|
||||
/** The production sampler: one bounded query per column. */
|
||||
export function sqlColumnSampler(client: CanaryClient): ColumnSampler {
|
||||
const query = requireUnsafe(client);
|
||||
return async (column, keyId, limit) => {
|
||||
/**
|
||||
* The production sampler: one query per column, each in its own transaction
|
||||
* under `SET LOCAL statement_timeout`, so a large or locked table throws
|
||||
* (and the probe ends `incomplete`) instead of holding up the boot.
|
||||
*/
|
||||
export function sqlColumnSampler(
|
||||
client: CanaryClient,
|
||||
options: { statementTimeoutMs?: number } = {},
|
||||
): ColumnSampler {
|
||||
const transaction = client.$transaction;
|
||||
if (typeof transaction !== "function") {
|
||||
throw new Error("The database client cannot run the probe queries");
|
||||
}
|
||||
const run = transaction.bind(client);
|
||||
const timeoutMs = Math.max(
|
||||
1,
|
||||
Math.floor(options.statementTimeoutMs ?? PROBE_STATEMENT_TIMEOUT_MS),
|
||||
);
|
||||
return async (column, keyId, limit, order) => {
|
||||
const loc = resolveColumnLocation(client, column);
|
||||
const { sql, params } = columnQuery(loc, column, keyId, "sample");
|
||||
const { sql, params } = columnQuery(loc, column, keyId, "sample", order);
|
||||
params[params.length - 1] = limit;
|
||||
const rows = await query<
|
||||
Array<{
|
||||
value: string | Uint8Array | null;
|
||||
codec: string | null;
|
||||
ts: Date | null;
|
||||
}>
|
||||
>(sql, ...params);
|
||||
type Row = {
|
||||
value: string | Uint8Array | null;
|
||||
codec: string | null;
|
||||
ts: Date | null;
|
||||
};
|
||||
const rows = await run(
|
||||
async (tx) => {
|
||||
await tx.$executeRaw`SELECT set_config('statement_timeout', ${String(timeoutMs)}, true)`;
|
||||
return tx.$queryRawUnsafe<Row[]>(sql, ...params);
|
||||
},
|
||||
// The transaction outlives its statement by a margin, so the server's
|
||||
// cancellation, not the client's, is what ends a slow query.
|
||||
{ maxWait: timeoutMs, timeout: timeoutMs + 5_000 },
|
||||
);
|
||||
return rows.map((r) => ({
|
||||
value: r.value,
|
||||
codec: r.codec,
|
||||
|
||||
@@ -56,6 +56,7 @@ import {
|
||||
canaryPlaintext,
|
||||
checkEncryptionKeyCanaries,
|
||||
retireCanariesWithoutData,
|
||||
sqlColumnSampler,
|
||||
type CanaryClient,
|
||||
} from "@/lib/crypto/canary";
|
||||
import {
|
||||
@@ -359,6 +360,79 @@ describe("boot key check: what decides, and what may seal (real Postgres)", () =
|
||||
expect(await prisma.encryptionKeyCanary.count()).toBe(1);
|
||||
});
|
||||
|
||||
it("old rows under a lost key, newer rows under the configured key: serves, warns, seals nothing", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
// Sealed under a key that is gone: the eight oldest values the probe
|
||||
// reads first (two per column, four columns).
|
||||
await seedUser("lost-1", new Date("2023-01-01T00:00:00Z"), sealNow);
|
||||
await seedUser("lost-2", new Date("2023-02-01T00:00:00Z"), sealNow);
|
||||
// Everything since, under the key the install runs with today.
|
||||
useKey(KEY_B);
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await seedUser(
|
||||
`current-${i}`,
|
||||
new Date(Date.UTC(2025, 0, 1 + i)),
|
||||
sealNow,
|
||||
);
|
||||
}
|
||||
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const errors = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
expect(await runBootKeyCheck(prisma)).toBe(false);
|
||||
expect(isKeyMismatch()).toBe(false);
|
||||
expect(errors).not.toHaveBeenCalled();
|
||||
expect(warn.mock.calls[0]?.[0]).toContain("inconclusive");
|
||||
warn.mockRestore();
|
||||
errors.mockRestore();
|
||||
expect(await prisma.encryptionKeyCanary.count()).toBe(0);
|
||||
expect(await checkEncryptionKeyCanaries(prisma)).toEqual({
|
||||
state: "ok",
|
||||
written: [],
|
||||
verified: [],
|
||||
inconclusive: [{ keyId: "v1", reason: "mixed" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("old AND newest values that do not open still refuse", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await seedUser("a-1", new Date("2023-01-01T00:00:00Z"), sealNow);
|
||||
await seedUser("a-2", new Date("2023-02-01T00:00:00Z"), sealNow);
|
||||
for (let i = 0; i < 4; i++) {
|
||||
await seedUser(`a-new-${i}`, new Date(Date.UTC(2025, 0, 1 + i)), sealNow);
|
||||
}
|
||||
useKey(KEY_B);
|
||||
expect(await checkEncryptionKeyCanaries(prisma)).toEqual({
|
||||
state: "mismatch",
|
||||
keyIds: ["v1"],
|
||||
});
|
||||
expect(await prisma.encryptionKeyCanary.count()).toBe(0);
|
||||
});
|
||||
|
||||
it("a probe query held up by a lock gives up at its statement timeout: inconclusive, not a hang", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await seedUser("locked", new Date("2024-01-01T00:00:00Z"), sealNow);
|
||||
const outcome = await prisma.$transaction(
|
||||
async (tx) => {
|
||||
// Another session holds the table: every read of it waits.
|
||||
await tx.$executeRaw`LOCK TABLE users IN ACCESS EXCLUSIVE MODE`;
|
||||
const started = Date.now();
|
||||
const result = await checkEncryptionKeyCanaries(prisma, {
|
||||
sampler: sqlColumnSampler(prisma, { statementTimeoutMs: 200 }),
|
||||
});
|
||||
return { result, elapsed: Date.now() - started };
|
||||
},
|
||||
{ timeout: 20_000 },
|
||||
);
|
||||
expect(outcome.result).toEqual({
|
||||
state: "ok",
|
||||
written: [],
|
||||
verified: [],
|
||||
inconclusive: [{ keyId: "v1", reason: "incomplete" }],
|
||||
});
|
||||
expect(outcome.elapsed).toBeLessThan(5_000);
|
||||
expect(await prisma.encryptionKeyCanary.count()).toBe(0);
|
||||
});
|
||||
|
||||
it("losing the first-seal race to another key is a mismatch, not ok", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
// The other process, holding A, wins the insert between our probe and
|
||||
|
||||
Reference in new issue
Block a user