mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
fix(ingest): refuse an external id that cannot be a stable identity
An external id is half of every ingest dedup key — (userId, source, externalId), (userId, externalSource, externalId), (userId, type, source, externalId). Re-posting the same triple is idempotent and returns the existing row, which holds only while the id is stable across client launches. A client that serialised an opaque platform object with its default object description sent a per-allocation memory address instead, so no sweep ever matched its own earlier row and each one minted a fresh record: a live instance grew 23 phantom medications in a day, none of which could collect a dose, because the dose side resolved through the same rotating string. Add one shared classifier for identifiers that provably cannot be an identity — an angle-bracketed object description carrying a hex pointer, a bare 0x address, and a blank value — and wire it at every surface that accepts a client-supplied externalId. Single-entry creates refuse through the schema, so the client gets the standard 422 multi-issue envelope with a message naming the shape it sent and what to send instead. Batch and bulk surfaces classify per entry and report the offending row as skipped with reason unstable_external_id, leaving the rest of the payload to land: one bad row must not stop a sync. Every refusal carries pinned wide-event meta (count, shape, surface) and never the id itself. A structural guard freezes both routes into the floor: a request-side externalId declaration either applies the field check or carries a marker naming the file that checks it per entry, and the guard verifies that file really does. The backup-restore payload is the one exemption, with its reason recorded — it replays ids the server itself wrote.
This commit is contained in:
1 parent
ef9891fec5
commit
2ab66ab2e3
26 files changed
+1232
-64
No files matched your search
@@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Structural guard on the external-id stability floor.
|
||||
*
|
||||
* A client-supplied `externalId` is half of every ingest dedup key. An id
|
||||
* that rotates between client launches never matches its own earlier row,
|
||||
* so each sync sweep mints a fresh record — the failure mode that filled a
|
||||
* live instance with phantom medications, one per sweep, none of which
|
||||
* could ever collect a dose.
|
||||
*
|
||||
* The fix is only durable if it cannot be bypassed by the NEXT route.
|
||||
* These guards freeze the two legal ways to accept the field:
|
||||
*
|
||||
* 1. the request schema routes it through `assertStableExternalId`, so
|
||||
* the refusal rides the standard 422 multi-issue envelope; or
|
||||
* 2. the field is deliberately permissive because a schema-level refusal
|
||||
* would fail a whole batch on one bad row — in which case the
|
||||
* declaration carries an `@external-id-checked-per-entry: <path>`
|
||||
* marker naming the file that runs `classifyExternalId` per entry,
|
||||
* and this guard verifies that file really does.
|
||||
*
|
||||
* They are tripwires, not proofs. A reviewer who waves through a bad
|
||||
* addition defeats them, and no test substitutes for that review.
|
||||
*/
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, sep } from "node:path";
|
||||
import { globSync } from "node:fs";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
const VALIDATOR_MODULE = "@/lib/validations/external-id";
|
||||
|
||||
function sourceFiles(): string[] {
|
||||
return globSync("**/*.{ts,tsx}", { cwd: SRC })
|
||||
.filter(
|
||||
(p) => !p.startsWith(`generated${sep}`) && !p.startsWith("generated/"),
|
||||
)
|
||||
.filter((p) => !p.includes("__tests__"))
|
||||
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
|
||||
.map((p) => p.split(sep).join("/"))
|
||||
.sort();
|
||||
}
|
||||
|
||||
function read(rel: string): string {
|
||||
return readFileSync(join(SRC, rel), "utf8");
|
||||
}
|
||||
|
||||
/**
|
||||
* A REQUEST-side declaration of a client-supplied external identifier:
|
||||
* `externalId: z.…` / `externalSourceId: z.…` in a Zod object shape.
|
||||
*
|
||||
* Response DTOs under `lib/openapi/` are excluded below — those echo an
|
||||
* id the server already stored and validating them would be validating
|
||||
* our own output.
|
||||
*/
|
||||
const DECLARATION = /^\s*(externalId|externalSourceId)\s*:\s*z\b/gm;
|
||||
|
||||
/** `// @external-id-checked-per-entry: <path under repo root>` */
|
||||
const MARKER = /@external-id-checked-per-entry:\s*(\S+)/;
|
||||
|
||||
/**
|
||||
* Files that declare the field but are deliberately outside the floor.
|
||||
* Each entry states WHY; an addition here is a decision, not a shortcut.
|
||||
*/
|
||||
const EXEMPT: Record<string, string> = {
|
||||
// Backup restore reads a payload the SERVER itself wrote and encrypted.
|
||||
// It has to reproduce the stored rows byte-for-byte — including ids that
|
||||
// predate this floor — or a self-hoster with an already-poisoned export
|
||||
// could never restore it. Nothing client-authored reaches this schema.
|
||||
"lib/validations/backup.ts":
|
||||
"restore payload is a server-written backup; refusing a stored id would block a restore",
|
||||
};
|
||||
|
||||
/** Lines of `rel` (1-based) on which a request-side declaration appears. */
|
||||
function declarationLines(rel: string): number[] {
|
||||
const source = read(rel);
|
||||
const lines: number[] = [];
|
||||
DECLARATION.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = DECLARATION.exec(source)) !== null) {
|
||||
lines.push(source.slice(0, m.index).split("\n").length);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function declaringFiles(): string[] {
|
||||
return sourceFiles()
|
||||
.filter((rel) => !rel.startsWith("lib/openapi/"))
|
||||
.filter((rel) => declarationLines(rel).length > 0);
|
||||
}
|
||||
|
||||
describe("every client-supplied external id routes through the stability floor", () => {
|
||||
it("finds the declaring files at all (the matcher still matches)", () => {
|
||||
// A guard whose matcher silently stops matching passes forever. Pin the
|
||||
// known set so a rename of the field or of the Zod idiom trips here.
|
||||
const files = declaringFiles();
|
||||
expect(files.length).toBeGreaterThanOrEqual(8);
|
||||
expect(files).toContain("app/api/measurements/batch/route.ts");
|
||||
expect(files).toContain("lib/validations/medication/create-update.ts");
|
||||
});
|
||||
|
||||
it.each(declaringFiles())(
|
||||
"%s either applies the field check or names the per-entry checker",
|
||||
(rel) => {
|
||||
if (rel in EXEMPT) {
|
||||
expect(EXEMPT[rel].length).toBeGreaterThan(20);
|
||||
return;
|
||||
}
|
||||
|
||||
const source = read(rel);
|
||||
const usesFieldCheck =
|
||||
source.includes(VALIDATOR_MODULE) &&
|
||||
source.includes("assertStableExternalId");
|
||||
const marksPerEntry = MARKER.test(source);
|
||||
expect(
|
||||
usesFieldCheck || marksPerEntry,
|
||||
`${rel} declares a client-supplied external id but neither applies ` +
|
||||
"`assertStableExternalId` to its request schema nor marks the declaration " +
|
||||
"`@external-id-checked-per-entry: <path>` naming the route that checks it per entry.",
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(declaringFiles())(
|
||||
"%s — every per-entry marker points at a file that really checks per entry",
|
||||
(rel) => {
|
||||
if (rel in EXEMPT) return;
|
||||
const source = read(rel);
|
||||
MARKER.lastIndex = 0;
|
||||
const targets = [...source.matchAll(new RegExp(MARKER, "g"))].map(
|
||||
(m) => m[1],
|
||||
);
|
||||
for (const target of targets) {
|
||||
expect(
|
||||
target.startsWith("src/"),
|
||||
`${rel}: marker target "${target}" must be a repo-root-relative src/ path`,
|
||||
).toBe(true);
|
||||
const checker = read(target.replace(/^src\//, ""));
|
||||
expect(
|
||||
checker.includes("classifyExternalId"),
|
||||
`${rel} points its per-entry marker at ${target}, but that file never calls classifyExternalId.`,
|
||||
).toBe(true);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("the batch surfaces refuse per entry, never whole-batch", () => {
|
||||
// Each of these drains a client sync queue. A schema-level refusal here
|
||||
// would stop a whole batch of good rows because of one bad one, which is
|
||||
// worse than the duplicate it prevents.
|
||||
const BATCH_ROUTES = [
|
||||
"app/api/measurements/batch/route.ts",
|
||||
"app/api/mood-entries/bulk/route.ts",
|
||||
"app/api/medications/intake/bulk/route.ts",
|
||||
"app/api/workouts/batch/route.ts",
|
||||
"app/api/cycle/day-logs/bulk/route.ts",
|
||||
"app/api/import/route.ts",
|
||||
];
|
||||
|
||||
it.each(BATCH_ROUTES)("%s classifies each entry itself", (rel) => {
|
||||
const source = read(rel);
|
||||
expect(source).toContain("classifyExternalId");
|
||||
expect(source).toContain("unstableExternalIdMeta");
|
||||
});
|
||||
|
||||
it.each(BATCH_ROUTES.filter((r) => r !== "app/api/import/route.ts"))(
|
||||
"%s reports the shared per-entry reason code",
|
||||
(rel) => {
|
||||
// The JSON importer has no per-entry status envelope — it reports a
|
||||
// skipped COUNT — so it is the one batch surface without the code.
|
||||
expect(read(rel)).toContain("UNSTABLE_EXTERNAL_ID_REASON");
|
||||
},
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user