mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 07:42:23 +00:00
fix(ingest): hold the stability floor for identities no request schema sees
The dedup key on the medication intake surfaces is not only `externalId`. `MedicationIntakeEvent.idempotency_key` is a `@unique` column persisted with the dose row forever, with no TTL and no sweep, matched by plain equality on the per-medication POST, the bulk drain and the external ingest bridge. That makes it an identity, not a replay token, and it carries the same failure mode: a key that rotates per client launch never matches its own earlier post, so every retry logs the dose again. The blank case is sharper still — the column is unique table-wide, so an empty key resolves to whichever unrelated row stored it first. Refuse both shapes at the schema on the single-entry paths and per entry on the bulk drain. The MCP write path hashes a host-supplied key into the `externalId` half of its dedup key, so the shape is laundered before anything downstream could see it and the check has to run on the input. That closes a live one: every tool handler falls back to `""` when the host omits the argument, and `""` digests to a constant, so each keyless write after the first reported `already_logged` and was silently dropped. The HTTP `Idempotency-Key` header stays out, and now says why in the guard: `IdempotencyKey.expiresAt` bounds it and the cleanup job sweeps it, so it is a windowed replay token and rotation is its design. Extend the structural guard with a provenance rule for identities that never pass a Zod request schema — a spreadsheet column, an MCP tool argument. Inside the import and MCP trees, a module handling an identity must apply the floor itself or import a module that does. That is a real chain rather than a list of known offenders: a new importer that mints an `externalId` from file input and imports nothing checked fails it, with no allowlist to escape into.
This commit is contained in:
1 parent
6e4bcffadf
commit
15b5f6f92f
19 files changed
+1574
-43
No files matched your search
@@ -58,14 +58,23 @@ function body(rel: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* A REQUEST-side declaration of a client-supplied external identifier:
|
||||
* `externalId: z.…` / `externalSourceId: z.…` in a Zod object shape.
|
||||
* A REQUEST-side declaration of a client-supplied identity string:
|
||||
* `externalId: z.…` / `externalSourceId: z.…` / `idempotencyKey: z.…` in a
|
||||
* Zod object shape.
|
||||
*
|
||||
* Response DTOs under `lib/openapi/` are excluded below — those echo an
|
||||
* id the server already stored and validating them would be validating
|
||||
* `idempotencyKey` is here because `MedicationIntakeEvent.idempotency_key`
|
||||
* is NOT a windowed replay token — it is a `@unique` column persisted with
|
||||
* the dose row forever and matched by plain equality, which makes it an
|
||||
* identity with the same failure mode. The HTTP `Idempotency-Key` HEADER
|
||||
* is a different thing and stays out: `IdempotencyKey.expiresAt` bounds it
|
||||
* and `lib/jobs/idempotency-cleanup.ts` sweeps it.
|
||||
*
|
||||
* Response DTOs under `lib/openapi/` are excluded below — those echo a
|
||||
* value the server already stored and validating them would be validating
|
||||
* our own output.
|
||||
*/
|
||||
const DECLARATION = /^\s*(externalId|externalSourceId)\s*:\s*z\b/gm;
|
||||
const DECLARATION =
|
||||
/^\s*(externalId|externalSourceId|idempotencyKey)\s*:\s*z\b/gm;
|
||||
|
||||
/** `// @external-id-checked-per-entry: <path under repo root>` */
|
||||
const MARKER = /@external-id-checked-per-entry:\s*(\S+)/;
|
||||
@@ -83,6 +92,59 @@ const EXEMPT: Record<string, string> = {
|
||||
"restore payload is a server-written backup; refusing a stored id would block a restore",
|
||||
};
|
||||
|
||||
/**
|
||||
* Trees where a stored identity is derived from input that never passes
|
||||
* through a Zod request schema — an uploaded file's row, or an MCP host's
|
||||
* tool argument. The declaration rule above cannot see those, so this one
|
||||
* works on provenance instead: inside these trees, any module that touches
|
||||
* an identity must either apply the floor itself or import a module that
|
||||
* does.
|
||||
*
|
||||
* That is a real chain, not a list of known offenders: a NEW importer that
|
||||
* mints an `externalId` from file input and imports nothing checked fails
|
||||
* here, with no allowlist to add itself to.
|
||||
*/
|
||||
const DERIVED_IDENTITY_TREES = ["lib/import/", "app/api/import/", "lib/mcp/"];
|
||||
|
||||
const IDENTITY_MENTION = /\b(externalId|idempotencyKey)\b/;
|
||||
|
||||
/** Floor symbols that count as "this module applies the check itself". */
|
||||
const FLOOR_SYMBOLS = [
|
||||
"classifyExternalId",
|
||||
"isStableExternalId",
|
||||
"assertStableExternalId",
|
||||
"assertStableIdempotencyKey",
|
||||
];
|
||||
|
||||
/** Local `@/…` and relative modules this file imports, as src-relative paths. */
|
||||
function importedLocalModules(rel: string): string[] {
|
||||
const source = read(rel);
|
||||
const dir = rel.split("/").slice(0, -1).join("/");
|
||||
const out: string[] = [];
|
||||
for (const m of source.matchAll(/from\s+"([^"]+)"/g)) {
|
||||
const spec = m[1];
|
||||
let target: string | null = null;
|
||||
if (spec.startsWith("@/")) target = spec.slice(2);
|
||||
else if (spec.startsWith("./")) target = `${dir}/${spec.slice(2)}`;
|
||||
else if (spec.startsWith("../")) {
|
||||
const parts = dir.split("/");
|
||||
let rest = spec;
|
||||
while (rest.startsWith("../")) {
|
||||
parts.pop();
|
||||
rest = rest.slice(3);
|
||||
}
|
||||
target = [...parts, rest].join("/");
|
||||
}
|
||||
if (target) out.push(target.replace(/\.tsx?$/, ""));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function appliesFloor(rel: string): boolean {
|
||||
const b = body(rel);
|
||||
return FLOOR_SYMBOLS.some((sym) => b.includes(sym));
|
||||
}
|
||||
|
||||
/** Lines of `rel` (1-based) on which a request-side declaration appears. */
|
||||
function declarationLines(rel: string): number[] {
|
||||
const source = read(rel);
|
||||
@@ -121,13 +183,15 @@ describe("every client-supplied external id routes through the stability floor",
|
||||
|
||||
const usesFieldCheck =
|
||||
read(rel).includes(VALIDATOR_MODULE) &&
|
||||
body(rel).includes("assertStableExternalId");
|
||||
(body(rel).includes("assertStableExternalId") ||
|
||||
body(rel).includes("assertStableIdempotencyKey"));
|
||||
const marksPerEntry = MARKER.test(read(rel));
|
||||
expect(
|
||||
usesFieldCheck || marksPerEntry,
|
||||
`${rel} declares a client-supplied external id but neither applies ` +
|
||||
"`assertStableExternalId` to its request schema nor marks the declaration " +
|
||||
"`@external-id-checked-per-entry: <path>` naming the route that checks it per entry.",
|
||||
`${rel} declares a client-supplied identity string but neither applies ` +
|
||||
"`assertStableExternalId` / `assertStableIdempotencyKey` to its request schema " +
|
||||
"nor marks the declaration `@external-id-checked-per-entry: <path>` naming the " +
|
||||
"route that checks it per entry.",
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
@@ -156,6 +220,84 @@ describe("every client-supplied external id routes through the stability floor",
|
||||
);
|
||||
});
|
||||
|
||||
describe("identities derived outside a Zod schema stay inside the floor", () => {
|
||||
/**
|
||||
* The CSV importer builds an `externalId` from a spreadsheet column, and
|
||||
* the MCP write path hashes a host-supplied key into one. Neither goes
|
||||
* through a Zod request field, so the declaration rule above is blind to
|
||||
* both — which is exactly the gap this rule closes.
|
||||
*/
|
||||
function derivedIdentityFiles(): string[] {
|
||||
return sourceFiles()
|
||||
.filter((rel) => DERIVED_IDENTITY_TREES.some((t) => rel.startsWith(t)))
|
||||
.filter((rel) => IDENTITY_MENTION.test(read(rel)));
|
||||
}
|
||||
|
||||
it("finds the derived-identity modules at all (the matcher still matches)", () => {
|
||||
const files = derivedIdentityFiles();
|
||||
expect(files).toContain("lib/import/csv-measurements.ts");
|
||||
expect(files).toContain("lib/mcp/writes.ts");
|
||||
});
|
||||
|
||||
it.each(derivedIdentityFiles())(
|
||||
"%s applies the floor itself or imports a module that does",
|
||||
(rel) => {
|
||||
if (appliesFloor(rel)) return;
|
||||
const sources = importedLocalModules(rel).filter((target) => {
|
||||
try {
|
||||
return appliesFloor(`${target}.ts`);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
expect(
|
||||
sources.length,
|
||||
`${rel} handles an external identity but neither applies the stability floor ` +
|
||||
"nor imports a module that does. It derives an identity from input no Zod " +
|
||||
"request schema sees (a file row, an MCP tool argument), so nothing else " +
|
||||
"will catch an unstable value here.",
|
||||
).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("the CSV importer refuses an unstable id with its own row reason", () => {
|
||||
// The importer has no per-entry Zod field to hang a refine on; its
|
||||
// contract is a per-row `reason` code. Pin that the floor is wired to
|
||||
// that contract and not merely imported.
|
||||
const b = body("lib/import/csv-measurements.ts");
|
||||
expect(b).toContain("isStableExternalId");
|
||||
expect(b).toContain('skip("unstable_external_id")');
|
||||
expect(read("lib/import/csv-measurements.ts")).toContain(
|
||||
'| "unstable_external_id"',
|
||||
);
|
||||
});
|
||||
|
||||
it("the MCP write path checks the key BEFORE it is hashed into an identity", () => {
|
||||
// A SHA-256 of a pointer address is clean hex, so the externalId floor
|
||||
// downstream can never see the shape. The check has to run on the input.
|
||||
const b = body("lib/mcp/writes.ts");
|
||||
const guardAt = b.indexOf("checkMcpIdempotencyKey");
|
||||
const hashAt = b.indexOf("mcpExternalId");
|
||||
expect(guardAt).toBeGreaterThan(-1);
|
||||
expect(hashAt).toBeGreaterThan(-1);
|
||||
expect(b).toContain("classifyExternalId");
|
||||
// Every entry point returns the refusal before reaching the hash.
|
||||
for (const fn of [
|
||||
"logMcpMeasurement",
|
||||
"logMcpMood",
|
||||
"logMcpBloodPressure",
|
||||
]) {
|
||||
const start = b.indexOf(`export async function ${fn}`);
|
||||
expect(start, `${fn} not found`).toBeGreaterThan(-1);
|
||||
const scope = b.slice(start, start + 2000);
|
||||
expect(
|
||||
scope.indexOf("checkMcpIdempotencyKey"),
|
||||
`${fn} does not check the idempotency key before writing`,
|
||||
).toBeGreaterThan(-1);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("the batch surfaces refuse per entry, never whole-batch", () => {
|
||||
// Each of these drains a client sync queue. A schema-level refusal here
|
||||
// would stop a whole batch of good rows because of one bad one, which is
|
||||
|
||||
Reference in new issue
Block a user