mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 13:22:14 +00:00
Merge branch 'main' into feat/lab-source-reference-ranges
This commit is contained in:
41 files changed
+2839
-61
No files matched your search
@@ -0,0 +1,200 @@
|
||||
/**
|
||||
* Structural guard on the measurement ingest boundary.
|
||||
*
|
||||
* `VALUE_RANGES` is the band the application declares a metric can occupy.
|
||||
* Every path a person can drive enforced it from the start; the provider sync
|
||||
* writers never did, and nothing noticed, because each writer is correct on
|
||||
* its own terms — it wrote what the remote sent. The gap was only visible from
|
||||
* above, by listing every writer at once and asking which of them had ever
|
||||
* been told about the band.
|
||||
*
|
||||
* So the list lives here. Every file that writes a `Measurement` row is
|
||||
* enumerated with the reason it is allowed to, and the guard fails when the
|
||||
* discovered set stops matching. A provider added tomorrow lands a file that
|
||||
* is not in the map and has to answer the question before it can ship.
|
||||
*
|
||||
* This is a tripwire, not a proof. It cannot show a disposition is the right
|
||||
* one — only that nobody added a writer without writing down which gate covers
|
||||
* it. A reviewer who waves through a bogus `derived` label defeats it, and no
|
||||
* test substitutes for that review.
|
||||
*/
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { walkSourceFiles } from "./helpers/source-files";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
|
||||
/** The gate module every provider-fed writer has to reach. */
|
||||
const GATE_MODULE = "@/lib/measurements/plausibility-gate";
|
||||
/** The predicate every person-driven write path has always used. */
|
||||
const VALIDATOR = "validateMeasurementRange";
|
||||
|
||||
/**
|
||||
* How a writer satisfies the plausibility domain.
|
||||
*
|
||||
* - `gate` — provider-fed; must reach {@link GATE_MODULE} itself.
|
||||
* - `validator` — person-driven; must call {@link VALIDATOR} itself.
|
||||
* - `upstream` — validated in a named module before the row is built; the
|
||||
* named module is checked, so the label cannot be a shrug.
|
||||
* - `derived` — the value is arithmetic over rows already in the table, not a
|
||||
* number from outside. Gating here would drop a consolidation of data that
|
||||
* IS present, which is a different failure from admitting one that is not.
|
||||
* - `literal` — the written value is a constant in the file, so no external
|
||||
* number can reach the column.
|
||||
* - `restore` — replays the record's own backup verbatim. Fidelity is the
|
||||
* contract; a restore that silently dropped rows would be the worse bug,
|
||||
* and the diagnostic script is how a stored impossible row gets found.
|
||||
*/
|
||||
type Disposition =
|
||||
| { kind: "gate" }
|
||||
| { kind: "validator" }
|
||||
| { kind: "upstream"; validatedIn: string }
|
||||
| { kind: "derived"; why: string }
|
||||
| { kind: "literal"; why: string }
|
||||
| { kind: "restore"; why: string };
|
||||
|
||||
const WRITERS: Record<string, Disposition> = {
|
||||
// ── Provider sync writers, gated ──────────────────────────────────────
|
||||
"lib/withings/sync.ts": { kind: "gate" },
|
||||
"lib/withings/sync-activity.ts": { kind: "gate" },
|
||||
"lib/fitbit/sync-core.ts": { kind: "gate" },
|
||||
"lib/google-health/sync-core.ts": { kind: "gate" },
|
||||
"lib/nightscout/sync.ts": { kind: "gate" },
|
||||
// The shared identity seam. Polar, Oura, WHOOP, the Withings sleep leg,
|
||||
// the Apple export importer and the batch endpoint all write through it,
|
||||
// so the gate here is what makes a new provider fail closed.
|
||||
"lib/measurements/reconcile-external-measurement.ts": { kind: "gate" },
|
||||
|
||||
// ── Person-driven write paths, validated ──────────────────────────────
|
||||
"app/api/measurements/route.ts": {
|
||||
kind: "upstream",
|
||||
validatedIn: "lib/validations/measurement.ts",
|
||||
},
|
||||
"app/api/import/route.ts": { kind: "validator" },
|
||||
"lib/mcp/writes.ts": { kind: "validator" },
|
||||
"lib/measurements/create-from-telegram.ts": { kind: "validator" },
|
||||
"lib/measurements/import-apple-health-export.ts": { kind: "validator" },
|
||||
"app/api/import/csv/route.ts": {
|
||||
kind: "upstream",
|
||||
validatedIn: "lib/import/csv-measurements.ts",
|
||||
},
|
||||
|
||||
// ── Fixed values ──────────────────────────────────────────────────────
|
||||
"lib/withings/sync-ecg.ts": {
|
||||
kind: "literal",
|
||||
why: "an ECG row carries value 1 — the file's own constant, never a provider number",
|
||||
},
|
||||
"app/api/mental-health/assessments/route.ts": {
|
||||
kind: "literal",
|
||||
why: "the questionnaire score is summed from Zod-bounded item answers",
|
||||
},
|
||||
"lib/insights/score-row.ts": {
|
||||
kind: "literal",
|
||||
why: "the health score is computed in-app on a closed 0-100 scale",
|
||||
},
|
||||
|
||||
// ── Derived from rows already stored ──────────────────────────────────
|
||||
"lib/measurements/consolidate-daily-mean.ts": {
|
||||
kind: "derived",
|
||||
why: "writes the mean of the day's own stored rows",
|
||||
},
|
||||
"lib/measurements/consolidate-legacy-steps.ts": {
|
||||
kind: "derived",
|
||||
why: "folds stored per-sample step rows into the day's canonical row",
|
||||
},
|
||||
"lib/measurements/dense-intraday-retention.ts": {
|
||||
kind: "derived",
|
||||
why: "rebuilds hourly buckets from stored per-sample rows",
|
||||
},
|
||||
"lib/measurements/drain-per-sample-cumulative.ts": {
|
||||
kind: "derived",
|
||||
why: "reduces stored cumulative per-sample rows to one daily total",
|
||||
},
|
||||
|
||||
// ── Restore ───────────────────────────────────────────────────────────
|
||||
"app/api/admin/backups/[id]/restore/route.ts": {
|
||||
kind: "restore",
|
||||
why: "replays the record's own export; dropping rows would break the restore contract",
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Any Prisma client handle followed by a measurement write. Whitespace- and
|
||||
* handle-tolerant on purpose: the earlier generation of this repo's guards
|
||||
* demanded a literal `prisma.x.findUnique(` and went green because it matched
|
||||
* nothing, not because nothing was there. The transaction handles in use are
|
||||
* `tx` and `pc`; a future one is caught by the `[A-Za-z_$][\w$]*` head.
|
||||
*/
|
||||
const MEASUREMENT_WRITE =
|
||||
/[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/;
|
||||
|
||||
function sourceFiles(): string[] {
|
||||
return walkSourceFiles(SRC, { floor: 3000 })
|
||||
.filter((p) => !p.startsWith("generated/"))
|
||||
.filter((p) => !p.includes("__tests__"))
|
||||
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
|
||||
.sort();
|
||||
}
|
||||
|
||||
function read(rel: string): string {
|
||||
return readFileSync(join(SRC, rel), "utf8");
|
||||
}
|
||||
|
||||
function discoveredWriters(): string[] {
|
||||
return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel)));
|
||||
}
|
||||
|
||||
describe("every measurement writer declares how it meets the plausibility domain", () => {
|
||||
it("finds the writers at all — an empty sweep is a broken guard, not a clean repo", () => {
|
||||
// The map is the claim; this asserts the matcher can still see the
|
||||
// population it describes. A regex that silently stops matching would
|
||||
// otherwise turn every check below into a tautology.
|
||||
expect(discoveredWriters().length).toBeGreaterThanOrEqual(15);
|
||||
});
|
||||
|
||||
it("the discovered writer set matches the declared map exactly", () => {
|
||||
expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort());
|
||||
});
|
||||
|
||||
it("every provider-fed writer reaches the gate module", () => {
|
||||
const gated = Object.entries(WRITERS)
|
||||
.filter(([, d]) => d.kind === "gate")
|
||||
.map(([rel]) => rel);
|
||||
expect(gated.length).toBeGreaterThanOrEqual(6);
|
||||
for (const rel of gated) {
|
||||
expect(read(rel), `${rel} must import ${GATE_MODULE}`).toContain(
|
||||
GATE_MODULE,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("every person-driven writer calls the range validator", () => {
|
||||
for (const [rel, disposition] of Object.entries(WRITERS)) {
|
||||
if (disposition.kind === "validator") {
|
||||
expect(read(rel), `${rel} must call ${VALIDATOR}`).toContain(VALIDATOR);
|
||||
}
|
||||
if (disposition.kind === "upstream") {
|
||||
expect(
|
||||
read(disposition.validatedIn),
|
||||
`${disposition.validatedIn} must call ${VALIDATOR} on behalf of ${rel}`,
|
||||
).toContain(VALIDATOR);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("the exempt writers carry a written reason", () => {
|
||||
for (const [rel, disposition] of Object.entries(WRITERS)) {
|
||||
if (
|
||||
disposition.kind === "derived" ||
|
||||
disposition.kind === "literal" ||
|
||||
disposition.kind === "restore"
|
||||
) {
|
||||
expect(disposition.why.length, `${rel} needs a reason`).toBeGreaterThan(
|
||||
20,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* Pair guard binding `prisma/schema.prisma`'s `MoodEntry` and
|
||||
* `MoodEntryTagLink` models to the disaster-recovery backup selects.
|
||||
*
|
||||
* The sibling of `measurement-backup-completeness.test.ts`, written as its own
|
||||
* file rather than as a second describe inside it, because that guard's own
|
||||
* note asks for model-by-model growth and one file per model keeps every
|
||||
* failure message about one model.
|
||||
*
|
||||
* The failure it exists to catch, stated by the defect that prompted it: the
|
||||
* mood payload carried `id, date, mood, score, tags, source, loggedAt,
|
||||
* externalId, deletedAt, factors` and nothing else. The free-text note, the
|
||||
* row's timezone and the last-writer-wins counter were selected by the query
|
||||
* and written to no payload, so a restored entry came back without its text,
|
||||
* read its day boundaries under the legacy Europe/Berlin assumption, and lost
|
||||
* the next sync round to a paired device still holding a higher counter. The
|
||||
* backup validation schema is `.passthrough()`, so nothing complained at parse
|
||||
* time either. Nothing here was a wrong value; every one of them was silence.
|
||||
*
|
||||
* Both ends of the comparison are literals — a Prisma model block and a named
|
||||
* select object — so the matcher is exact. Every matcher also asserts a
|
||||
* non-zero match count: an empty match set fails rather than passing quietly,
|
||||
* which is how the bearer-scope guard stayed green for months while matching
|
||||
* nothing at all.
|
||||
*
|
||||
* Mutation check: comment out `tz: true` in `MOOD_ENTRY_BACKUP_SELECT` and
|
||||
* this goes red naming that column; add a scalar to the model without adding
|
||||
* it to the select and it goes red naming the new one; rename the mapper's
|
||||
* `moodEntry` parameter and the serialization assertion goes red on all of
|
||||
* them at once.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
const ROOT = process.cwd();
|
||||
const SCHEMA = join(ROOT, "prisma", "schema.prisma");
|
||||
const BACKUP_BUILDER = join(
|
||||
ROOT,
|
||||
"src",
|
||||
"lib",
|
||||
"export",
|
||||
"full-backup-payload.ts",
|
||||
);
|
||||
|
||||
/**
|
||||
* Columns the backup is correct to omit, each with the reason. The reasons are
|
||||
* length-checked below: an exclusion is a claim, and a claim needs saying.
|
||||
*/
|
||||
const EXCLUDED_MOOD_ENTRY_COLUMNS: Record<string, string> = {
|
||||
userId: "user-scoped backup; owner is re-derived from the restoring account",
|
||||
};
|
||||
|
||||
const EXCLUDED_TAG_LINK_COLUMNS: Record<string, string> = {
|
||||
moodEntryId: "re-bound to the restored parent row's id, never carried",
|
||||
moodTagId:
|
||||
"the link travels by the tag's key; a source-instance id need not exist in the target catalogue",
|
||||
createdAt:
|
||||
"the link's own row stamp; the entry carries the moment the person logged it, and a key-only BINARY tag has nowhere to put a second one",
|
||||
};
|
||||
|
||||
function modelBlock(schema: string, model: string): string {
|
||||
const start = schema.indexOf(`model ${model} {`);
|
||||
expect(start, `model ${model} not found`).toBeGreaterThan(-1);
|
||||
const end = schema.indexOf("\n}", start);
|
||||
expect(end).toBeGreaterThan(start);
|
||||
return schema.slice(start, end);
|
||||
}
|
||||
|
||||
/** Names of every `model` block, so relation fields can be told apart. */
|
||||
function declaredModels(schema: string): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const m of schema.matchAll(/^model\s+([A-Za-z_]\w*)\s*\{/gm)) {
|
||||
out.add(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scalar (column-backed) fields of a model block: everything that is not a
|
||||
* relation to another model. Enums and native scalars both map to columns and
|
||||
* therefore both belong in a backup.
|
||||
*/
|
||||
function scalarFields(block: string, models: Set<string>): string[] {
|
||||
const fields: string[] = [];
|
||||
for (const rawLine of block.split("\n").slice(1)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("//") || line.startsWith("@@")) continue;
|
||||
const m = /^([a-zA-Z_]\w*)\s+([A-Za-z_]\w*)(\[\])?(\?)?/.exec(line);
|
||||
if (!m) continue;
|
||||
const [, name, type, isList] = m;
|
||||
if (isList) continue;
|
||||
if (models.has(type)) continue;
|
||||
if (line.includes("@relation")) continue;
|
||||
fields.push(name);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
/** Keys of a named `const <NAME> = {` select literal. */
|
||||
function selectFields(source: string, constant: string): string[] {
|
||||
const start = source.indexOf(`const ${constant} = {`);
|
||||
expect(start, `${constant} not found`).toBeGreaterThan(-1);
|
||||
const end = source.indexOf("\n}", start);
|
||||
expect(end).toBeGreaterThan(start);
|
||||
const body = source.slice(start, end);
|
||||
return [...body.matchAll(/^ {2}([a-zA-Z_$][\w$]*): true,$/gm)].map(
|
||||
(m) => m[1],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The builder with its comment lines removed.
|
||||
*
|
||||
* Every matcher below runs against this rather than against the raw file,
|
||||
* because prose describing the defect reads exactly like the defect: the first
|
||||
* draft of this guard went red on its own explanation of the filter it exists
|
||||
* to forbid. Whole comment lines only — a trailing comment after code is left
|
||||
* alone, which is safe here because the only risk it carries is a matcher
|
||||
* accepting a column named in prose, and the select comparison beside it is
|
||||
* literal either way.
|
||||
*/
|
||||
function stripCommentLines(source: string): string {
|
||||
return source
|
||||
.split("\n")
|
||||
.filter((line) => {
|
||||
const trimmed = line.trim();
|
||||
return !(
|
||||
trimmed.startsWith("//") ||
|
||||
trimmed.startsWith("/*") ||
|
||||
trimmed.startsWith("*")
|
||||
);
|
||||
})
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Columns either serialization arm actually reads off the mapped row.
|
||||
*
|
||||
* The capture is the RIGHT-hand side — the column — not the payload key, which
|
||||
* matters here in a way it did not for `Measurement`: the mood payload renames
|
||||
* `moodLoggedAt` to `loggedAt` on the wire, so a left-hand matcher would report
|
||||
* the column as dropped while the value rides perfectly well.
|
||||
*
|
||||
* The two arms share one restorable wire format, so this is a union and
|
||||
* therefore a weaker claim than the select comparison: it catches a column that
|
||||
* is selected and then written to neither payload, not one carried by only a
|
||||
* single arm.
|
||||
*/
|
||||
function serialisedFields(source: string, variable: string): Set<string> {
|
||||
const matches = [
|
||||
...source.matchAll(
|
||||
new RegExp(`[:(,]\\s*${variable}\\.([a-zA-Z_$][\\w$]*)`, "g"),
|
||||
),
|
||||
];
|
||||
expect(
|
||||
matches.length,
|
||||
`no \`${variable}.<column>\` reads found — the mapper parameter was renamed and this matcher now proves nothing`,
|
||||
).toBeGreaterThan(0);
|
||||
return new Set(matches.map((m) => m[1]));
|
||||
}
|
||||
|
||||
describe("mood backup completeness", () => {
|
||||
const schema = readFileSync(SCHEMA, "utf8");
|
||||
const source = stripCommentLines(readFileSync(BACKUP_BUILDER, "utf8"));
|
||||
const models = declaredModels(schema);
|
||||
|
||||
const entryColumns = scalarFields(modelBlock(schema, "MoodEntry"), models);
|
||||
const entrySelected = selectFields(source, "MOOD_ENTRY_BACKUP_SELECT");
|
||||
const linkColumns = scalarFields(
|
||||
modelBlock(schema, "MoodEntryTagLink"),
|
||||
models,
|
||||
);
|
||||
const linkSelected = selectFields(source, "MOOD_ENTRY_TAG_LINK_SELECT");
|
||||
|
||||
it("parses both ends into a plausible shape", () => {
|
||||
expect(models.size).toBeGreaterThan(50);
|
||||
expect(entryColumns.length).toBeGreaterThan(10);
|
||||
expect(entryColumns).toContain("noteEncrypted");
|
||||
expect(entryColumns).toContain("syncVersion");
|
||||
expect(entryColumns).not.toContain("user");
|
||||
expect(entryColumns).not.toContain("tagLinks");
|
||||
expect(entrySelected.length).toBeGreaterThan(10);
|
||||
|
||||
expect(linkColumns).toEqual(
|
||||
expect.arrayContaining([
|
||||
"moodEntryId",
|
||||
"moodTagId",
|
||||
"createdAt",
|
||||
"rating",
|
||||
]),
|
||||
);
|
||||
expect(linkColumns).not.toContain("moodEntry");
|
||||
expect(linkColumns).not.toContain("moodTag");
|
||||
expect(linkSelected.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("the backup select covers every MoodEntry column", () => {
|
||||
const missing = entryColumns.filter(
|
||||
(c) => !entrySelected.includes(c) && !(c in EXCLUDED_MOOD_ENTRY_COLUMNS),
|
||||
);
|
||||
|
||||
if (missing.length > 0) {
|
||||
const report = missing.map((c) => ` ❌ ${c}`).join("\n");
|
||||
throw new Error(
|
||||
`MOOD_ENTRY_BACKUP_SELECT omits ${missing.length} column(s) of the MoodEntry model:\n${report}\n\n` +
|
||||
"A disaster-recovery restore brings these back at their column " +
|
||||
"default. Add each to the select and to both serialization arms, " +
|
||||
"extend `moodEntrySchema` and the restore write — or add it to " +
|
||||
"EXCLUDED_MOOD_ENTRY_COLUMNS with the reason the backup is correct " +
|
||||
"to drop it.",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("every selected MoodEntry column reaches a serialized payload", () => {
|
||||
const emitted = serialisedFields(source, "moodEntry");
|
||||
const dropped = entrySelected.filter((c) => !emitted.has(c));
|
||||
|
||||
if (dropped.length > 0) {
|
||||
const report = dropped.map((c) => ` ❌ ${c}`).join("\n");
|
||||
throw new Error(
|
||||
`${dropped.length} selected MoodEntry column(s) reach neither serialization arm:\n${report}\n\n` +
|
||||
"The query pays for them and no payload carries them, so a restore " +
|
||||
"cannot put them back. Emit each from the disaster-recovery arm " +
|
||||
"(and, where it is readable and not a tombstone, from the portable " +
|
||||
"arm too).",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("the tag-link select covers every MoodEntryTagLink column", () => {
|
||||
const missing = linkColumns.filter(
|
||||
(c) => !linkSelected.includes(c) && !(c in EXCLUDED_TAG_LINK_COLUMNS),
|
||||
);
|
||||
expect(
|
||||
missing,
|
||||
`MOOD_ENTRY_TAG_LINK_SELECT omits MoodEntryTagLink column(s): ${missing.join(", ")}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("every selected tag-link column reaches a serialized payload", () => {
|
||||
const emitted = serialisedFields(source, "tagLink");
|
||||
const dropped = linkSelected.filter((c) => !emitted.has(c));
|
||||
expect(
|
||||
dropped,
|
||||
`selected but serialized by neither arm: ${dropped.join(", ")}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("both structured-tag arms are carried, not just the rated one", () => {
|
||||
// The defect this file was written for: the tag-link read carried a
|
||||
// filter, and a BINARY link's rating is NULL by definition, so every
|
||||
// present/absent tag — the kind most people pick — was excluded from the
|
||||
// file and could not come back from it. The claim is therefore about the
|
||||
// read itself: it takes the links whole, unfiltered.
|
||||
//
|
||||
// Asserted on booleans rather than on the file contents so a failure reads
|
||||
// as one line instead of printing the whole builder back.
|
||||
const tagLinkRead = /tagLinks:\s*\{([\s\S]*?)\}/.exec(source);
|
||||
expect(
|
||||
tagLinkRead,
|
||||
"no `tagLinks:` read found in the backup builder — this matcher proves nothing",
|
||||
).not.toBeNull();
|
||||
expect(
|
||||
/where/.test(tagLinkRead![1]),
|
||||
"the tag-link read filters again; a filter here is what dropped every BINARY link from the backup",
|
||||
).toBe(false);
|
||||
expect(
|
||||
source.includes("structuredTags"),
|
||||
"no `structuredTags` key in the payload — the BINARY links have no carrier",
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("the exclusion lists carry no stale entry", () => {
|
||||
for (const [column, reason] of Object.entries(
|
||||
EXCLUDED_MOOD_ENTRY_COLUMNS,
|
||||
)) {
|
||||
expect(
|
||||
entryColumns,
|
||||
`${column} is excluded but is no longer a MoodEntry column`,
|
||||
).toContain(column);
|
||||
expect(reason.length).toBeGreaterThan(20);
|
||||
}
|
||||
for (const [column, reason] of Object.entries(EXCLUDED_TAG_LINK_COLUMNS)) {
|
||||
expect(
|
||||
linkColumns,
|
||||
`${column} is excluded but is no longer a MoodEntryTagLink column`,
|
||||
).toContain(column);
|
||||
expect(reason.length).toBeGreaterThan(20);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -684,6 +684,10 @@ const handler = apiHandler(
|
||||
entry.factors.map((factor) => factor.key),
|
||||
);
|
||||
const factorKeys = [...new Set(referencedFactorKeys)];
|
||||
const referencedTagKeys = payload.moodEntries.flatMap(
|
||||
(entry) => entry.structuredTags,
|
||||
);
|
||||
const tagKeys = [...new Set(referencedTagKeys)];
|
||||
// Re-create the account's own tag definitions first. The lookup
|
||||
// below used to ask only for the seeded catalogue (`userId: null`),
|
||||
// so a single custom rated tag made the whole restore throw
|
||||
@@ -740,6 +744,30 @@ const handler = apiHandler(
|
||||
referencedFactorKeys,
|
||||
);
|
||||
|
||||
// The BINARY half of the same taxonomy, looked up the same way and
|
||||
// reported the same way. It gets its own family because an
|
||||
// operator reading the report needs to know whether a rated
|
||||
// factor's score or a ticked tag went missing; they are different
|
||||
// losses on the same entry.
|
||||
const tagRows =
|
||||
tagKeys.length === 0
|
||||
? []
|
||||
: await tx.moodTag.findMany({
|
||||
where: {
|
||||
key: { in: tagKeys },
|
||||
kind: "BINARY",
|
||||
OR: [{ userId: null }, { userId: ownerId }],
|
||||
},
|
||||
select: { id: true, key: true },
|
||||
});
|
||||
const tagByKey = new Map(tagRows.map((tag) => [tag.key, tag.id]));
|
||||
recordUnknownKeys(
|
||||
skips,
|
||||
"moodTag",
|
||||
tagKeys.filter((key) => !tagByKey.has(key)),
|
||||
referencedTagKeys,
|
||||
);
|
||||
|
||||
for (const entry of payload.moodEntries) {
|
||||
const moodLoggedAt = new Date(entry.loggedAt);
|
||||
const restoredData = {
|
||||
@@ -747,10 +775,44 @@ const handler = apiHandler(
|
||||
mood: entry.mood,
|
||||
score: entry.score,
|
||||
tags: entry.tags ?? null,
|
||||
// The note comes back the way every other dual-column note
|
||||
// does: ciphertext verbatim when the file carries it, and a
|
||||
// portable file's plaintext re-encrypted on the way in, with
|
||||
// the legacy column left null either way.
|
||||
note: null,
|
||||
noteEncrypted:
|
||||
entry.noteEncrypted == null
|
||||
? encryptNote(entry.note ?? null)
|
||||
: decodeEncryptedBytes(entry.noteEncrypted),
|
||||
source: entry.source ?? "MOODLOG",
|
||||
externalId: entry.externalId ?? null,
|
||||
moodLoggedAt,
|
||||
// The zone the `date` string is anchored to. A file written
|
||||
// before this rode the payload carries nothing, and NULL is
|
||||
// what such a row meant anyway: the legacy Europe/Berlin
|
||||
// reading. So an old file keeps its old day boundaries and a
|
||||
// new one keeps the zone the person actually logged in.
|
||||
tz: entry.tz ?? null,
|
||||
// Mood reconciles last-writer-wins by `syncVersion`. Restoring
|
||||
// at the schema default would hand the next sync round to any
|
||||
// paired device still holding a higher number, and the device
|
||||
// would overwrite the recovered row with what it had — a
|
||||
// restore that loses to the client is worse than one that
|
||||
// fails. An old file carries nothing here and keeps the
|
||||
// default, which is the value those rows were written with.
|
||||
...(entry.syncVersion === undefined
|
||||
? {}
|
||||
: { syncVersion: entry.syncVersion }),
|
||||
deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null,
|
||||
...(entry.syncedAt
|
||||
? { syncedAt: new Date(entry.syncedAt) }
|
||||
: {}),
|
||||
...(entry.createdAt
|
||||
? { createdAt: new Date(entry.createdAt) }
|
||||
: {}),
|
||||
...(entry.updatedAt
|
||||
? { updatedAt: new Date(entry.updatedAt) }
|
||||
: {}),
|
||||
};
|
||||
const createData = { userId: ownerId, ...restoredData };
|
||||
const restored = entry.id
|
||||
@@ -798,8 +860,19 @@ const handler = apiHandler(
|
||||
]
|
||||
: [];
|
||||
});
|
||||
if (factorLinks.length > 0) {
|
||||
await tx.moodEntryTagLink.createMany({ data: factorLinks });
|
||||
// The other arm. A BINARY link carries no score — the presence
|
||||
// of the row is the signal — so `rating` is NULL by definition,
|
||||
// which is exactly what the backup's old tag-link filter took to
|
||||
// mean "nothing here".
|
||||
const tagLinks = entry.structuredTags.flatMap((key) => {
|
||||
const moodTagId = tagByKey.get(key);
|
||||
return moodTagId
|
||||
? [{ moodEntryId: restored.id, moodTagId, rating: null }]
|
||||
: [];
|
||||
});
|
||||
const links = [...factorLinks, ...tagLinks];
|
||||
if (links.length > 0) {
|
||||
await tx.moodEntryTagLink.createMany({ data: links });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -620,6 +620,17 @@ async function postBatch(request: NextRequest): Promise<Response> {
|
||||
};
|
||||
failedCount++;
|
||||
continue;
|
||||
case "rejected_range":
|
||||
// The reconciler carries the same plausibility gate this route
|
||||
// already applied above, so reaching here means the two disagreed —
|
||||
// report it exactly as the earlier guard would have, and let the
|
||||
// wide-event tally record which end refused.
|
||||
results[p.index] = {
|
||||
index: p.index,
|
||||
status: "skipped",
|
||||
reason: "value_out_of_range",
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isMergeableSource(p.row.source as string)) {
|
||||
|
||||
@@ -194,6 +194,9 @@ function catalogueLabel(
|
||||
if (catalogue === "illnessSymptom") {
|
||||
return t("admin.section.backups.restoreSkippedIllnessSymptom");
|
||||
}
|
||||
if (catalogue === "moodTag") {
|
||||
return t("admin.section.backups.restoreSkippedMoodTag");
|
||||
}
|
||||
return t("admin.section.backups.restoreSkippedMoodFactor");
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,8 @@ import { restoreProfileData } from "../profile-backup";
|
||||
const deletedAt = new Date("2026-07-19T12:00:00.000Z");
|
||||
const measurementNote = encryptToBytes("canonical measurement note");
|
||||
const sideEffectNote = encryptToBytes("nausea after the evening dose");
|
||||
const moodNotePlaintext = "quiet evening, slept badly";
|
||||
const moodNote = encryptToBytes(moodNotePlaintext);
|
||||
|
||||
const appSettings = {
|
||||
id: "singleton",
|
||||
@@ -193,11 +195,27 @@ function makePrisma() {
|
||||
mood: "OKAY",
|
||||
score: 3,
|
||||
tags: null,
|
||||
note: null,
|
||||
noteEncrypted: moodNote,
|
||||
source: "MOODLOG",
|
||||
externalId: "moodlog-42",
|
||||
moodLoggedAt: new Date("2026-07-19T08:00:00.000Z"),
|
||||
tz: "America/New_York",
|
||||
syncedAt: new Date("2026-07-19T08:00:01.000Z"),
|
||||
syncVersion: 3,
|
||||
deletedAt,
|
||||
tagLinks: [{ rating: 4, moodTag: { key: "sleep_quality" } }],
|
||||
createdAt: new Date("2026-07-19T08:00:00.000Z"),
|
||||
updatedAt: new Date("2026-07-19T09:00:00.000Z"),
|
||||
tagLinks: [
|
||||
{
|
||||
rating: 4,
|
||||
moodTag: { key: "sleep_quality", kind: "RATED" },
|
||||
},
|
||||
{
|
||||
rating: null,
|
||||
moodTag: { key: "headache", kind: "BINARY" },
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
@@ -409,7 +427,13 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
|
||||
expect(prisma.moodEntry.findMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: { userId: "user-1" },
|
||||
include: expect.any(Object),
|
||||
select: expect.objectContaining({
|
||||
id: true,
|
||||
noteEncrypted: true,
|
||||
tz: true,
|
||||
syncVersion: true,
|
||||
deletedAt: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mocks.buildRecordsBackupSection).toHaveBeenCalledWith(
|
||||
@@ -485,7 +509,16 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
|
||||
id: "mood-tombstone",
|
||||
externalId: "moodlog-42",
|
||||
deletedAt: deletedAt.toISOString(),
|
||||
tz: "America/New_York",
|
||||
syncedAt: "2026-07-19T08:00:01.000Z",
|
||||
syncVersion: 3,
|
||||
createdAt: "2026-07-19T08:00:00.000Z",
|
||||
updatedAt: "2026-07-19T09:00:00.000Z",
|
||||
note: null,
|
||||
noteEncrypted: Buffer.from(moodNote).toString("base64"),
|
||||
factors: [{ key: "sleep_quality", rating: 4 }],
|
||||
// The BINARY link the old `rating: { not: null }` filter excluded.
|
||||
structuredTags: ["headache"],
|
||||
},
|
||||
],
|
||||
nutrientDays: [
|
||||
@@ -1148,3 +1181,79 @@ describe("buildFullBackupPayload — profile and custom metrics", () => {
|
||||
expect(counts.intradayProfiles).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildFullBackupPayload — mood arms", () => {
|
||||
it("carries the ciphertext and the whole row on the disaster-recovery arm", async () => {
|
||||
installSectionMocks();
|
||||
const prisma = makePrisma();
|
||||
|
||||
const { payload } = await buildFullBackupPayload(
|
||||
prisma as never,
|
||||
"user-1",
|
||||
{
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: new Date("2026-07-20T00:00:00.000Z"),
|
||||
},
|
||||
);
|
||||
|
||||
const [entry] = (payload as { moodEntries: Array<Record<string, unknown>> })
|
||||
.moodEntries;
|
||||
expect(entry).toEqual(
|
||||
expect.objectContaining({
|
||||
id: "mood-tombstone",
|
||||
note: null,
|
||||
noteEncrypted: Buffer.from(moodNote).toString("base64"),
|
||||
tz: "America/New_York",
|
||||
syncedAt: "2026-07-19T08:00:01.000Z",
|
||||
syncVersion: 3,
|
||||
createdAt: "2026-07-19T08:00:00.000Z",
|
||||
updatedAt: "2026-07-19T09:00:00.000Z",
|
||||
deletedAt: deletedAt.toISOString(),
|
||||
factors: [{ key: "sleep_quality", rating: 4 }],
|
||||
structuredTags: ["headache"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("carries the readable note and no ciphertext on the portable arm", async () => {
|
||||
installSectionMocks();
|
||||
const prisma = makePrisma();
|
||||
|
||||
const { payload } = await buildFullBackupPayload(
|
||||
prisma as never,
|
||||
"user-1",
|
||||
{
|
||||
purpose: "portable-export",
|
||||
exportedAt: new Date("2026-07-20T00:00:00.000Z"),
|
||||
},
|
||||
);
|
||||
|
||||
const [entry] = (payload as { moodEntries: Array<Record<string, unknown>> })
|
||||
.moodEntries;
|
||||
// Same split the measurement arm makes: the readable artefact carries the
|
||||
// decrypted text and nothing an operator could not read, and the counters
|
||||
// that only mean something to this instance stay behind.
|
||||
expect(entry).toEqual(
|
||||
expect.objectContaining({
|
||||
note: moodNotePlaintext,
|
||||
tz: "America/New_York",
|
||||
structuredTags: ["headache"],
|
||||
}),
|
||||
);
|
||||
expect(entry).not.toHaveProperty("noteEncrypted");
|
||||
expect(entry).not.toHaveProperty("syncVersion");
|
||||
expect(entry).not.toHaveProperty("syncedAt");
|
||||
expect(JSON.stringify(entry)).not.toContain(
|
||||
Buffer.from(moodNote).toString("base64"),
|
||||
);
|
||||
// The tag-link read is unfiltered on both arms; a BINARY link is the one
|
||||
// most people have and it was the one the file never carried.
|
||||
expect(prisma.moodEntry.findMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
select: expect.objectContaining({
|
||||
tagLinks: { select: expect.any(Object) },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -103,6 +103,46 @@ const MEASUREMENT_BACKUP_SELECT = {
|
||||
updatedAt: true,
|
||||
} as const satisfies Prisma.MeasurementSelect;
|
||||
|
||||
/**
|
||||
* Every restorable `MoodEntry` scalar. `userId` is absent for the same reason
|
||||
* it is absent above: the backup is user-scoped and the owner is re-derived
|
||||
* from the restoring account.
|
||||
*
|
||||
* Named rather than inlined so `mood-backup-completeness.test.ts` can compare
|
||||
* it field-by-field against the model in `prisma/schema.prisma`, exactly as the
|
||||
* measurement guard does for the select above.
|
||||
*/
|
||||
const MOOD_ENTRY_BACKUP_SELECT = {
|
||||
id: true,
|
||||
date: true,
|
||||
mood: true,
|
||||
score: true,
|
||||
tags: true,
|
||||
note: true,
|
||||
noteEncrypted: true,
|
||||
source: true,
|
||||
externalId: true,
|
||||
moodLoggedAt: true,
|
||||
tz: true,
|
||||
syncedAt: true,
|
||||
syncVersion: true,
|
||||
deletedAt: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
} as const satisfies Prisma.MoodEntrySelect;
|
||||
|
||||
/**
|
||||
* A structured-tag link, as it rides the wire. The link is carried by the
|
||||
* tag's KEY, not by its id: a restore resolves the key against the target
|
||||
* instance's catalogue, where the source instance's row id may not exist at
|
||||
* all. `kind` comes along so the mapper can split RATED factors from BINARY
|
||||
* tags without a second query.
|
||||
*/
|
||||
const MOOD_ENTRY_TAG_LINK_SELECT = {
|
||||
rating: true,
|
||||
moodTag: { select: { key: true, kind: true } },
|
||||
} as const satisfies Prisma.MoodEntryTagLinkSelect;
|
||||
|
||||
/**
|
||||
* Build the canonical full-backup payload for `userId`. Portable exports omit
|
||||
* tombstones and document ciphertext; disaster-recovery payloads preserve
|
||||
@@ -207,14 +247,13 @@ export async function buildFullBackupPayload(
|
||||
prisma.moodEntry.findMany({
|
||||
where: disasterRecovery ? { userId } : { userId, deletedAt: null },
|
||||
orderBy: { moodLoggedAt: "desc" },
|
||||
include: {
|
||||
tagLinks: {
|
||||
where: { rating: { not: null } },
|
||||
select: {
|
||||
rating: true,
|
||||
moodTag: { select: { key: true } },
|
||||
},
|
||||
},
|
||||
select: {
|
||||
...MOOD_ENTRY_BACKUP_SELECT,
|
||||
// Every link, not only the rated ones. This read used to filter
|
||||
// `rating: { not: null }`, and a BINARY link carries a NULL rating by
|
||||
// definition — so the present/absent tags, the kind most people pick,
|
||||
// were absent from the file and could not come back from it.
|
||||
tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT },
|
||||
},
|
||||
}),
|
||||
// The account's OWN mood tags. The seeded catalogue is reference data every
|
||||
@@ -394,21 +433,66 @@ export async function buildFullBackupPayload(
|
||||
skipped: e.skipped,
|
||||
source: e.source,
|
||||
})),
|
||||
moodEntries: moodEntries.map((e) => ({
|
||||
id: e.id,
|
||||
date: e.date,
|
||||
mood: e.mood,
|
||||
score: e.score,
|
||||
tags: e.tags,
|
||||
source: e.source,
|
||||
loggedAt: e.moodLoggedAt.toISOString(),
|
||||
externalId: e.externalId,
|
||||
deletedAt: e.deletedAt?.toISOString() ?? null,
|
||||
factors: e.tagLinks.map((link) => ({
|
||||
key: link.moodTag.key,
|
||||
rating: link.rating!,
|
||||
})),
|
||||
})),
|
||||
// The parameter is spelled out rather than the `e` every other mapper here
|
||||
// uses, and deliberately: `mood-backup-completeness.test.ts` proves a
|
||||
// selected column reaches a payload by matching `: moodEntry.`. A matcher
|
||||
// on `: e.` would also match the intake-event mapper above it and could
|
||||
// therefore never fail.
|
||||
moodEntries: moodEntries.map((moodEntry) => {
|
||||
// The two link arms partition the entry's links exhaustively: a link is
|
||||
// a rated factor when its tag is RATED and it actually carries a score,
|
||||
// and a structured tag otherwise. Total on purpose — a malformed link
|
||||
// (a RATED tag whose rating went missing) rides as a tag key and comes
|
||||
// back NAMED as unresolvable on restore, rather than falling between the
|
||||
// two arms and disappearing the way every BINARY link used to.
|
||||
const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) =>
|
||||
link.moodTag.kind === "RATED" && link.rating !== null;
|
||||
|
||||
const factors = moodEntry.tagLinks
|
||||
.filter(isRatedFactor)
|
||||
.map((tagLink) => ({
|
||||
key: tagLink.moodTag.key,
|
||||
rating: tagLink.rating as number,
|
||||
}));
|
||||
// BINARY link keys. Their own array rather than a nullable rating inside
|
||||
// `factors`: the restore resolves `factors` against `kind: "RATED"` and
|
||||
// reports what it cannot resolve, so a rating-less entry in that array
|
||||
// would collide with both. A separate key leaves every backup file
|
||||
// written before this parsing and restoring exactly as it did.
|
||||
const structuredTags = moodEntry.tagLinks
|
||||
.filter((link) => !isRatedFactor(link))
|
||||
.map((tagLink) => tagLink.moodTag.key);
|
||||
|
||||
return {
|
||||
...(disasterRecovery
|
||||
? {
|
||||
note: moodEntry.note,
|
||||
noteEncrypted: moodEntry.noteEncrypted
|
||||
? Buffer.from(moodEntry.noteEncrypted).toString("base64")
|
||||
: null,
|
||||
syncedAt: moodEntry.syncedAt.toISOString(),
|
||||
syncVersion: moodEntry.syncVersion,
|
||||
createdAt: moodEntry.createdAt.toISOString(),
|
||||
updatedAt: moodEntry.updatedAt.toISOString(),
|
||||
}
|
||||
: { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }),
|
||||
id: moodEntry.id,
|
||||
date: moodEntry.date,
|
||||
mood: moodEntry.mood,
|
||||
score: moodEntry.score,
|
||||
tags: moodEntry.tags,
|
||||
source: moodEntry.source,
|
||||
loggedAt: moodEntry.moodLoggedAt.toISOString(),
|
||||
externalId: moodEntry.externalId,
|
||||
// The zone the `date` string is anchored to. Without it a restored row
|
||||
// falls back to the legacy Europe/Berlin reading and an account that
|
||||
// logged elsewhere gets its day boundaries quietly moved.
|
||||
tz: moodEntry.tz,
|
||||
deletedAt: moodEntry.deletedAt?.toISOString() ?? null,
|
||||
factors,
|
||||
structuredTags,
|
||||
};
|
||||
}),
|
||||
customMoodTags: customMoodTags.map((tag) => ({
|
||||
...(disasterRecovery ? { id: tag.id } : {}),
|
||||
key: tag.key,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* What a restore could not put back, and how much of it there was.
|
||||
*
|
||||
* Three of the restore's lookups resolve a key against a SEEDED CATALOGUE —
|
||||
* cycle symptoms, illness symptoms, mood factors. The catalogue is reference
|
||||
* Four of the restore's lookups resolve a key against a SEEDED CATALOGUE —
|
||||
* cycle symptoms, illness symptoms, rated mood factors and the mood tags a
|
||||
* person ticks present or absent. The catalogue is reference
|
||||
* data the instance owns, not data the file carries, so it legitimately drifts
|
||||
* between the day a backup is written and the day it is read: a key gets
|
||||
* renamed, a symptom is retired, the file is a year old, the instance is three
|
||||
@@ -28,7 +29,8 @@
|
||||
*/
|
||||
|
||||
/** Which seeded catalogue a key failed to resolve against. */
|
||||
export type SkippedCatalogue = "cycleSymptom" | "illnessSymptom" | "moodFactor";
|
||||
export type SkippedCatalogue =
|
||||
"cycleSymptom" | "illnessSymptom" | "moodFactor" | "moodTag";
|
||||
|
||||
/** One key this instance does not know, and the links it cost. */
|
||||
export interface SkippedCatalogueKey {
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* `upsertFitbitMeasurements` is the single write seam for every Fitbit
|
||||
* resource leg — activity, sleep, daily metrics — and it holds Prisma
|
||||
* directly, so it does not inherit the shared reconciler's gate. These cases
|
||||
* pin that the provider's numbers clear the app's own declared plausibility
|
||||
* band before any of them can become a row, and that a refusal is reported.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
|
||||
findManyMock: vi.fn(async () => [] as unknown[]),
|
||||
createManyMock: vi.fn<
|
||||
() => Promise<Array<{ id: string; type: string; measuredAt: Date }>>
|
||||
>(async () => []),
|
||||
updateMock: vi.fn(async () => ({})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findMany: findManyMock,
|
||||
createManyAndReturn: createManyMock,
|
||||
update: updateMock,
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn(async () => false),
|
||||
recordSyncFailure: vi.fn(async () => {}),
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
recomputeBucketsForMeasurement: vi.fn(async () => {}),
|
||||
recomputeUserRollups: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/integrations/oauth-refresh", () => ({
|
||||
persistRotatedToken: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserFitbitCredentials: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../client", () => ({ refreshAccessToken: vi.fn() }));
|
||||
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { upsertFitbitMeasurements } from "../sync-core";
|
||||
|
||||
/** RESTING_HEART_RATE is declared 25–220. */
|
||||
const REAL_READING = {
|
||||
type: "RESTING_HEART_RATE",
|
||||
value: 58,
|
||||
unit: "bpm",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
externalId: "rhr:2026-07-08",
|
||||
};
|
||||
|
||||
const IMPOSSIBLE_READING = {
|
||||
type: "RESTING_HEART_RATE",
|
||||
value: 4_915_200,
|
||||
unit: "bpm",
|
||||
measuredAt: new Date("2026-07-09T00:00:00.000Z"),
|
||||
externalId: "rhr:2026-07-09",
|
||||
};
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
findManyMock.mockReset().mockResolvedValue([]);
|
||||
createManyMock.mockReset().mockResolvedValue([]);
|
||||
updateMock.mockReset().mockResolvedValue({});
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("upsertFitbitMeasurements — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "RESTING_HEART_RATE",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertFitbitMeasurements("user-1", [REAL_READING], { deferRollup: true }),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ value: number }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.value).toBe(58);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "RESTING_HEART_RATE",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertFitbitMeasurements("user-1", [REAL_READING, IMPOSSIBLE_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ value: number; externalId: string }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.externalId).toBe("rhr:2026-07-08");
|
||||
// The refused reading never even reached the existence probe.
|
||||
expect(
|
||||
(
|
||||
(findManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
where: { externalId: { in: string[] } };
|
||||
}
|
||||
).where.externalId.in,
|
||||
).toEqual(["rhr:2026-07-08"]);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "fitbit",
|
||||
type: "RESTING_HEART_RATE",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("does not overwrite a stored reading with an out-of-band one", async () => {
|
||||
// The key already exists; ungated, the update branch rewrote its value.
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "existing",
|
||||
type: "RESTING_HEART_RATE",
|
||||
externalId: "rhr:2026-07-09",
|
||||
value: 58,
|
||||
unit: "bpm",
|
||||
measuredAt: IMPOSSIBLE_READING.measuredAt,
|
||||
sleepStage: null,
|
||||
deletedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertFitbitMeasurements("user-1", [IMPOSSIBLE_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(0);
|
||||
expect(updateMock).not.toHaveBeenCalled();
|
||||
expect(createManyMock).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -15,6 +15,7 @@ import { persistRotatedToken } from "@/lib/integrations/oauth-refresh";
|
||||
import { recordSyncFailure } from "@/lib/integrations/status";
|
||||
import type { FailureKind } from "@/lib/integrations/status";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
collapseToTypeDayKeys,
|
||||
recomputeBucketsForMeasurement,
|
||||
@@ -530,7 +531,7 @@ export async function replaceStaleFitbitSleep(
|
||||
*/
|
||||
export async function upsertFitbitMeasurements(
|
||||
userId: string,
|
||||
readings: FitbitMeasurementUpsert[],
|
||||
incoming: FitbitMeasurementUpsert[],
|
||||
opts: { deferRollup?: boolean } = {},
|
||||
): Promise<{
|
||||
imported: number;
|
||||
@@ -541,6 +542,15 @@ export async function upsertFitbitMeasurements(
|
||||
measuredAt: Date;
|
||||
}>;
|
||||
}> {
|
||||
// Every resource leg funnels through here, so this is the one place the
|
||||
// provider's numbers have to clear the app's declared plausibility band
|
||||
// before they can become rows. A refused reading is dropped and tallied on
|
||||
// the ambient event, never clamped and never written for a later reader to
|
||||
// discover.
|
||||
const readings = dropImplausibleMeasurements("fitbit", incoming, (r) => ({
|
||||
type: r.type,
|
||||
value: r.value,
|
||||
}));
|
||||
if (readings.length === 0) return { imported: 0, touched: [], inserted: [] };
|
||||
|
||||
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* `upsertGoogleHealthMeasurements` is the single write seam for every Google
|
||||
* Health resource leg and it holds Prisma directly, so it does not inherit the
|
||||
* shared reconciler's gate. These cases pin that the provider's numbers clear
|
||||
* the app's own declared plausibility band before any of them can become a
|
||||
* row, and that a refusal is reported rather than swallowed.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
|
||||
findManyMock: vi.fn(async () => [] as unknown[]),
|
||||
createManyMock: vi.fn<
|
||||
() => Promise<Array<{ id: string; type: string; measuredAt: Date }>>
|
||||
>(async () => []),
|
||||
updateMock: vi.fn(async () => ({})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findMany: findManyMock,
|
||||
createManyAndReturn: createManyMock,
|
||||
update: updateMock,
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn(async () => false),
|
||||
recordSyncFailure: vi.fn(async () => {}),
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
recomputeBucketsForMeasurement: vi.fn(async () => {}),
|
||||
recomputeUserRollups: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserGoogleHealthCredentials: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../client", async () => {
|
||||
const actual = await vi.importActual<typeof import("../client")>("../client");
|
||||
return {
|
||||
...actual,
|
||||
refreshAccessToken: vi.fn(),
|
||||
noteGoogleHealthMapped: vi.fn(),
|
||||
noteGoogleHealthWritten: vi.fn(),
|
||||
noteGoogleHealthOutcomeFailure: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { upsertGoogleHealthMeasurements } from "../sync-core";
|
||||
|
||||
/** OXYGEN_SATURATION is declared 50–100. */
|
||||
const REAL_READING = {
|
||||
type: "OXYGEN_SATURATION",
|
||||
value: 97,
|
||||
unit: "%",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
externalId: "spo2:2026-07-08",
|
||||
};
|
||||
|
||||
const IMPOSSIBLE_READING = {
|
||||
type: "OXYGEN_SATURATION",
|
||||
value: 9_700,
|
||||
unit: "%",
|
||||
measuredAt: new Date("2026-07-09T00:00:00.000Z"),
|
||||
externalId: "spo2:2026-07-09",
|
||||
};
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
findManyMock.mockReset().mockResolvedValue([]);
|
||||
createManyMock.mockReset().mockResolvedValue([]);
|
||||
updateMock.mockReset().mockResolvedValue({});
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("upsertGoogleHealthMeasurements — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "OXYGEN_SATURATION",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertGoogleHealthMeasurements("user-1", [REAL_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ value: number }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.value).toBe(97);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "OXYGEN_SATURATION",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertGoogleHealthMeasurements(
|
||||
"user-1",
|
||||
[REAL_READING, IMPOSSIBLE_READING],
|
||||
{ deferRollup: true },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ externalId: string }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.externalId).toBe("spo2:2026-07-08");
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "googleHealth",
|
||||
type: "OXYGEN_SATURATION",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("does not overwrite a stored reading with an out-of-band one", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "existing",
|
||||
type: "OXYGEN_SATURATION",
|
||||
externalId: "spo2:2026-07-09",
|
||||
value: 97,
|
||||
unit: "%",
|
||||
measuredAt: IMPOSSIBLE_READING.measuredAt,
|
||||
sleepStage: null,
|
||||
deletedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertGoogleHealthMeasurements("user-1", [IMPOSSIBLE_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(0);
|
||||
expect(updateMock).not.toHaveBeenCalled();
|
||||
expect(createManyMock).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,7 @@ import { prisma } from "@/lib/db";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { encrypt, decrypt } from "@/lib/crypto";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
|
||||
import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status";
|
||||
import {
|
||||
@@ -487,7 +488,7 @@ export async function replaceStaleGoogleHealthSleep(
|
||||
*/
|
||||
export async function upsertGoogleHealthMeasurements(
|
||||
userId: string,
|
||||
readings: GoogleHealthMeasurementUpsert[],
|
||||
incoming: GoogleHealthMeasurementUpsert[],
|
||||
opts: { deferRollup?: boolean } = {},
|
||||
): Promise<{
|
||||
imported: number;
|
||||
@@ -498,7 +499,17 @@ export async function upsertGoogleHealthMeasurements(
|
||||
measuredAt: Date;
|
||||
}>;
|
||||
}> {
|
||||
noteGoogleHealthMapped(readings.length);
|
||||
noteGoogleHealthMapped(incoming.length);
|
||||
// Every resource leg funnels through here, so this is the one place the
|
||||
// provider's numbers have to clear the app's declared plausibility band
|
||||
// before they can become rows. A refused reading is dropped and tallied on
|
||||
// the ambient event, never clamped and never written for a later reader to
|
||||
// discover.
|
||||
const readings = dropImplausibleMeasurements(
|
||||
"googleHealth",
|
||||
incoming,
|
||||
(r) => ({ type: r.type, value: r.value }),
|
||||
);
|
||||
if (readings.length === 0) return { imported: 0, touched: [], inserted: [] };
|
||||
|
||||
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/**
|
||||
* The ingest-side plausibility gate.
|
||||
*
|
||||
* `VALUE_RANGES` is the band the application declares a metric can occupy, and
|
||||
* every path a person can drive has always enforced it. The provider sync
|
||||
* writers did not, which is how a row the application itself calls impossible
|
||||
* reached the table and poisoned every mean and personal band derived from it.
|
||||
* These cases pin the two halves of the fix: the refusal, and the fact that a
|
||||
* refusal is never silent.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import {
|
||||
RANGE_REJECTED_META_KEY,
|
||||
classifyRangeRejection,
|
||||
dropImplausibleMeasurements,
|
||||
recordRangeRejections,
|
||||
type RangeRejectionTally,
|
||||
} from "../plausibility-gate";
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function withEvent<T>(run: () => T): T {
|
||||
return eventStorage.run(event, run);
|
||||
}
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("classifyRangeRejection", () => {
|
||||
it("passes a value inside the metric's declared band", () => {
|
||||
// PULSE is declared 20–300; both edges are inclusive.
|
||||
expect(classifyRangeRejection("PULSE", 60)).toBeNull();
|
||||
expect(classifyRangeRejection("PULSE", 20)).toBeNull();
|
||||
expect(classifyRangeRejection("PULSE", 300)).toBeNull();
|
||||
});
|
||||
|
||||
it("names the edge an out-of-band value broke", () => {
|
||||
expect(classifyRangeRejection("PULSE", 55643821.505)).toEqual({
|
||||
type: "PULSE",
|
||||
direction: "above_max",
|
||||
});
|
||||
expect(classifyRangeRejection("PULSE", 19)).toEqual({
|
||||
type: "PULSE",
|
||||
direction: "below_min",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses a non-finite value whatever the metric", () => {
|
||||
expect(classifyRangeRejection("PULSE", Number.NaN)).toEqual({
|
||||
type: "PULSE",
|
||||
direction: "not_finite",
|
||||
});
|
||||
// No declared band, so the value itself is the only thing to judge.
|
||||
expect(
|
||||
classifyRangeRejection("NOT_A_METRIC", Number.POSITIVE_INFINITY),
|
||||
).toEqual({ type: "NOT_A_METRIC", direction: "not_finite" });
|
||||
});
|
||||
|
||||
it("passes a metric with no declared band — an absent range is an absent fact", () => {
|
||||
expect(classifyRangeRejection("NOT_A_METRIC", 9_999_999)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("dropImplausibleMeasurements", () => {
|
||||
it("keeps the in-band rows and returns them untouched", () => {
|
||||
const rows = [
|
||||
{ type: "PULSE", value: 60 },
|
||||
{ type: "WEIGHT", value: 82.4 },
|
||||
];
|
||||
const kept = withEvent(() =>
|
||||
dropImplausibleMeasurements("withings", rows, (r) => r),
|
||||
);
|
||||
|
||||
expect(kept).toEqual(rows);
|
||||
expect(kept[0]).toBe(rows[0]);
|
||||
// Nothing was refused, so the event carries no tally at all — absence
|
||||
// reads as absence.
|
||||
expect(tally()).toBeUndefined();
|
||||
expect(event.toJSON().warnings).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops the out-of-band rows and tallies them on the wide event", () => {
|
||||
const kept = withEvent(() =>
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[
|
||||
{ type: "PULSE", value: 60 },
|
||||
{ type: "PULSE", value: 55643821.505 },
|
||||
{ type: "PULSE", value: 900 },
|
||||
{ type: "WEIGHT", value: 0.2 },
|
||||
],
|
||||
(r) => r,
|
||||
),
|
||||
);
|
||||
|
||||
expect(kept).toEqual([{ type: "PULSE", value: 60 }]);
|
||||
|
||||
const recorded = tally();
|
||||
expect(recorded).toEqual({
|
||||
total: 3,
|
||||
buckets: [
|
||||
{
|
||||
source: "withings",
|
||||
type: "PULSE",
|
||||
direction: "above_max",
|
||||
count: 2,
|
||||
},
|
||||
{
|
||||
source: "withings",
|
||||
type: "WEIGHT",
|
||||
direction: "below_min",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the refused readings out of the log line", () => {
|
||||
withEvent(() =>
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[{ type: "PULSE", value: 55643821.505 }],
|
||||
(r) => r,
|
||||
),
|
||||
);
|
||||
|
||||
// The value is a health reading. The tally says which metric, which
|
||||
// provider and which edge — never the number itself.
|
||||
const serialised = JSON.stringify(event.toJSON());
|
||||
expect(serialised).not.toContain("55643821");
|
||||
});
|
||||
|
||||
it("lifts the event to warn once per provider", () => {
|
||||
withEvent(() => {
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[{ type: "PULSE", value: 900 }],
|
||||
(r) => r,
|
||||
);
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[{ type: "PULSE", value: 901 }],
|
||||
(r) => r,
|
||||
);
|
||||
dropImplausibleMeasurements(
|
||||
"fitbit",
|
||||
[{ type: "PULSE", value: 902 }],
|
||||
(r) => r,
|
||||
);
|
||||
});
|
||||
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
expect(event.toJSON().warnings).toHaveLength(2);
|
||||
expect(event.toJSON().warnings?.[0]).toContain("withings");
|
||||
expect(event.toJSON().warnings?.[1]).toContain("fitbit");
|
||||
});
|
||||
|
||||
it("accumulates across calls instead of overwriting the previous tally", () => {
|
||||
// Fitbit gates once per resource leg, and the reconciler gates once per
|
||||
// row: a tally that replaced rather than merged would report only the
|
||||
// last batch, and a run that dropped hundreds would read as dropping one.
|
||||
withEvent(() => {
|
||||
dropImplausibleMeasurements(
|
||||
"fitbit",
|
||||
[{ type: "PULSE", value: 900 }],
|
||||
(r) => r,
|
||||
);
|
||||
dropImplausibleMeasurements(
|
||||
"fitbit",
|
||||
[
|
||||
{ type: "PULSE", value: 901 },
|
||||
{ type: "OXYGEN_SATURATION", value: 4 },
|
||||
],
|
||||
(r) => r,
|
||||
);
|
||||
});
|
||||
|
||||
expect(tally()).toEqual({
|
||||
total: 3,
|
||||
buckets: [
|
||||
{
|
||||
source: "fitbit",
|
||||
type: "OXYGEN_SATURATION",
|
||||
direction: "below_min",
|
||||
count: 1,
|
||||
},
|
||||
{ source: "fitbit", type: "PULSE", direction: "above_max", count: 2 },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("still drops outside a wide-event context", () => {
|
||||
// A sync running without an ambient event has nowhere to report. The row
|
||||
// is refused anyway: observability is what is lost, never the guarantee.
|
||||
const kept = dropImplausibleMeasurements(
|
||||
"nightscout",
|
||||
[
|
||||
{ type: "BLOOD_GLUCOSE", value: 110 },
|
||||
{ type: "BLOOD_GLUCOSE", value: 9_000 },
|
||||
],
|
||||
(r) => r,
|
||||
);
|
||||
expect(kept).toEqual([{ type: "BLOOD_GLUCOSE", value: 110 }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordRangeRejections", () => {
|
||||
it("writes nothing when there is nothing to report", () => {
|
||||
withEvent(() => recordRangeRejections("withings", []));
|
||||
expect(tally()).toBeUndefined();
|
||||
expect(event.getLevel()).toBe("info");
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,9 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "../plausibility-gate";
|
||||
import {
|
||||
reconcileExternalMeasurement,
|
||||
type ExternalMeasurementWrite,
|
||||
@@ -383,3 +386,104 @@ describe("reconcileExternalMeasurement", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The reconciler is the shared write seam for every provider that does not
|
||||
* hold Prisma itself — Polar, Oura, WHOOP, the Withings sleep leg, the Apple
|
||||
* export importer, the batch endpoint. Gating here rather than at each caller
|
||||
* is what makes a provider added tomorrow inherit the refusal instead of
|
||||
* having to remember a check it never knew about.
|
||||
*/
|
||||
describe("reconcileExternalMeasurement — plausibility gate", () => {
|
||||
it("reconciles a value inside the metric's declared band", async () => {
|
||||
const { tx, measurement } = transaction([]);
|
||||
measurement.create.mockResolvedValue(row({ id: "fresh" }));
|
||||
|
||||
// PULSE is declared 20–300.
|
||||
const verdict = await reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({ type: "PULSE", value: 112, unit: "bpm", sleepStage: null }),
|
||||
);
|
||||
|
||||
expect(verdict.status).toBe("inserted");
|
||||
expect(measurement.create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("refuses an out-of-band value without touching the transaction", async () => {
|
||||
const { tx, measurement } = transaction([]);
|
||||
|
||||
const verdict = await reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
unit: "bpm",
|
||||
sleepStage: null,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(verdict).toEqual({
|
||||
status: "rejected_range",
|
||||
rejection: { type: "PULSE", direction: "above_max" },
|
||||
});
|
||||
// No row written, no row updated, and no savepoint opened — the refusal
|
||||
// happens before the reconciler does any work at all.
|
||||
expect(measurement.create).not.toHaveBeenCalled();
|
||||
expect(measurement.update).not.toHaveBeenCalled();
|
||||
expect(tx.$executeRawUnsafe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses rather than overwriting a good stored row", async () => {
|
||||
// The dangerous shape: a live row already holds a real reading, and the
|
||||
// provider re-sends the same identity carrying an impossible value. The
|
||||
// update branch would have rewritten a good value with a bad one.
|
||||
const existing = row({
|
||||
id: "exact-row",
|
||||
type: "PULSE",
|
||||
value: 62,
|
||||
unit: "bpm",
|
||||
sleepStage: null,
|
||||
measuredAt: new Date("2026-07-20T06:30:00.000Z"),
|
||||
});
|
||||
const { tx, measurement } = transaction([existing]);
|
||||
|
||||
const verdict = await reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
unit: "bpm",
|
||||
sleepStage: null,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(verdict.status).toBe("rejected_range");
|
||||
expect(measurement.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("tallies the refusal on the ambient wide event under the provider", async () => {
|
||||
const { tx } = transaction([]);
|
||||
const event = new WideEventBuilder("background");
|
||||
|
||||
await eventStorage.run(event, () =>
|
||||
reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
unit: "bpm",
|
||||
source: "WHOOP",
|
||||
sleepStage: null,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
expect(event.toJSON().meta?.[RANGE_REJECTED_META_KEY]).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{ source: "whoop", type: "PULSE", direction: "above_max", count: 1 },
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
});
|
||||
@@ -720,6 +720,16 @@ export async function streamParseExportXml(
|
||||
),
|
||||
);
|
||||
|
||||
// The reconciler carries the same plausibility gate the block above
|
||||
// already applied, so this arm means the two disagreed. Tally it under
|
||||
// the same counter the earlier guard uses and count no row.
|
||||
if (verdict.status === "rejected_range") {
|
||||
unknown[`${type}::aggregate_out_of_range`] =
|
||||
(unknown[`${type}::aggregate_out_of_range`] ?? 0) + 1;
|
||||
stat.durationMs += Date.now() - rowStart;
|
||||
continue;
|
||||
}
|
||||
|
||||
cumulativeEstimatedDays.add(dayKey);
|
||||
cumulativeEstimatedRows += 1;
|
||||
if (verdict.status === "inserted") {
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* The ingest-side half of the app's plausibility domain.
|
||||
*
|
||||
* `VALUE_RANGES` in `@/lib/validations/measurement` is the band the
|
||||
* application declares a metric can occupy — PULSE 20–300, WEIGHT 1–500,
|
||||
* SLEEP_DURATION 0–1440 minutes. Every path a person can drive enforces it:
|
||||
* the measurement routes, the batch endpoint, the CSV and Apple-export
|
||||
* importers, MCP, Telegram. The provider sync writers did not. They took the
|
||||
* number the remote handed over and wrote it, so a decode slip or a glitched
|
||||
* device landed a row the application itself calls impossible, and every mean,
|
||||
* median and personal band downstream digested it as a reading.
|
||||
*
|
||||
* `isPlausibleMetricValue` (in `value-domain.ts`) already answers the question
|
||||
* for the READ side, which is where the damage surfaced first. This module is
|
||||
* the same question asked one layer earlier, at the point of writing, so the
|
||||
* row never lands. The two ends share the one predicate on purpose: a band the
|
||||
* reader hides but the writer accepts is a row that exists and cannot be seen,
|
||||
* which is the worse of the two failures.
|
||||
*
|
||||
* ## Disposition: drop, and say so
|
||||
*
|
||||
* An out-of-band incoming value is DROPPED. Not clamped (a clamp invents a
|
||||
* reading nobody took), not stored-and-flagged (a quarantined row is still a
|
||||
* row, and every consumer would need to learn the flag), not fatal to the
|
||||
* batch (one glitched sample must not strand the sync's other readings behind
|
||||
* a held watermark). Absence reads as absence: the sample is simply not part
|
||||
* of the record.
|
||||
*
|
||||
* A silent drop, though, is indistinguishable from a provider that sent
|
||||
* nothing — so every drop is tallied onto the ambient wide event under
|
||||
* {@link RANGE_REJECTED_META_KEY}, with the metric type, the provider, and the
|
||||
* direction it broke the band in. Counts only. The value itself is a health
|
||||
* reading and never enters the log line; the direction plus the metric's own
|
||||
* declared bound is what a diagnosis actually needs.
|
||||
*
|
||||
* ## Why meta and not `action`
|
||||
*
|
||||
* A wide event holds exactly ONE action, and `setAction` replaces it. A drop
|
||||
* happens in the middle of a sync whose action is decided at the end, so an
|
||||
* `action` set here is either clobbered a moment later (the drop vanishes) or,
|
||||
* on the paths that annotate no terminal action, relabels an otherwise
|
||||
* successful sync as a rejection. Meta accumulates, survives every later
|
||||
* `setAction`, and is the shape a dashboard can sum. One warning per provider
|
||||
* per event rides along so the drop also lifts the event's level to `warn`
|
||||
* rather than hiding inside a green line.
|
||||
*/
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { isPlausibleMetricValue } from "@/lib/measurements/value-domain";
|
||||
import { VALUE_RANGES } from "@/lib/validations/measurement";
|
||||
|
||||
/**
|
||||
* Stable meta key the tally lands under. Pinned so dashboards can count it
|
||||
* across providers without knowing which one wrote the event.
|
||||
*/
|
||||
export const RANGE_REJECTED_META_KEY = "measurement.range_rejected";
|
||||
|
||||
/** Which edge of the declared band the value broke, or that it was not a number. */
|
||||
export type RangeRejectionDirection = "below_min" | "above_max" | "not_finite";
|
||||
|
||||
/** One refused reading, reduced to the facts that carry no health value. */
|
||||
export interface MeasurementRangeRejection {
|
||||
/** `MeasurementType` of the refused reading. */
|
||||
type: string;
|
||||
direction: RangeRejectionDirection;
|
||||
}
|
||||
|
||||
/** One (provider, metric, direction) cell of the event's running tally. */
|
||||
export interface RangeRejectionBucket {
|
||||
source: string;
|
||||
type: string;
|
||||
direction: RangeRejectionDirection;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/** The pinned meta shape written under {@link RANGE_REJECTED_META_KEY}. */
|
||||
export interface RangeRejectionTally {
|
||||
total: number;
|
||||
buckets: RangeRejectionBucket[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a candidate write against the metric's declared band.
|
||||
*
|
||||
* Returns `null` when the value may be written. A type with no declared range
|
||||
* passes — an absent range is an absent fact, never a guessed one, and the
|
||||
* same asymmetry the read side uses (`isPlausibleMetricValue`). A non-finite
|
||||
* value never passes, whatever the type: NaN is not an input to anything.
|
||||
*/
|
||||
export function classifyRangeRejection(
|
||||
type: string,
|
||||
value: number,
|
||||
): MeasurementRangeRejection | null {
|
||||
if (isPlausibleMetricValue(type, value)) return null;
|
||||
if (!Number.isFinite(value)) return { type, direction: "not_finite" };
|
||||
const range = VALUE_RANGES[type];
|
||||
// `isPlausibleMetricValue` only answers false for a finite value when the
|
||||
// type HAS a range, so the lookup cannot miss here.
|
||||
return {
|
||||
type,
|
||||
direction: value < range!.min ? "below_min" : "above_max",
|
||||
};
|
||||
}
|
||||
|
||||
function readTally(): RangeRejectionTally {
|
||||
const existing = getEvent()?.toJSON().meta?.[RANGE_REJECTED_META_KEY];
|
||||
if (
|
||||
existing !== null &&
|
||||
typeof existing === "object" &&
|
||||
Array.isArray((existing as RangeRejectionTally).buckets)
|
||||
) {
|
||||
return existing as RangeRejectionTally;
|
||||
}
|
||||
return { total: 0, buckets: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold refusals onto the ambient wide event.
|
||||
*
|
||||
* Merges with whatever the event already carries, so a provider that gates
|
||||
* several batches (or reconciles row by row) accumulates instead of
|
||||
* overwriting. Outside a wide-event context this is a no-op — the caller still
|
||||
* drops the row, it simply has nowhere to report.
|
||||
*/
|
||||
export function recordRangeRejections(
|
||||
source: string,
|
||||
rejections: readonly MeasurementRangeRejection[],
|
||||
): void {
|
||||
if (rejections.length === 0) return;
|
||||
const evt = getEvent();
|
||||
if (!evt) return;
|
||||
|
||||
const tally = readTally();
|
||||
const firstForSource = !tally.buckets.some((b) => b.source === source);
|
||||
|
||||
for (const rejection of rejections) {
|
||||
const bucket = tally.buckets.find(
|
||||
(b) =>
|
||||
b.source === source &&
|
||||
b.type === rejection.type &&
|
||||
b.direction === rejection.direction,
|
||||
);
|
||||
if (bucket) bucket.count += 1;
|
||||
else
|
||||
tally.buckets.push({
|
||||
source,
|
||||
type: rejection.type,
|
||||
direction: rejection.direction,
|
||||
count: 1,
|
||||
});
|
||||
tally.total += 1;
|
||||
}
|
||||
|
||||
tally.buckets.sort(
|
||||
(a, b) =>
|
||||
a.source.localeCompare(b.source) ||
|
||||
a.type.localeCompare(b.type) ||
|
||||
a.direction.localeCompare(b.direction),
|
||||
);
|
||||
|
||||
annotate({ meta: { [RANGE_REJECTED_META_KEY]: tally } });
|
||||
if (firstForSource) {
|
||||
evt.addWarning(
|
||||
`${source}: refused reading(s) outside the declared plausibility range — ` +
|
||||
`dropped, see meta.${RANGE_REJECTED_META_KEY}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the rows whose value sits inside the metric's declared band; tally and
|
||||
* drop the rest.
|
||||
*
|
||||
* `select` exists because the sync writers carry their own row shapes and none
|
||||
* of them is worth reshaping at the call site: the gate reads the two fields
|
||||
* it needs and returns the caller's own objects untouched.
|
||||
*/
|
||||
export function dropImplausibleMeasurements<T>(
|
||||
source: string,
|
||||
rows: readonly T[],
|
||||
select: (row: T) => { type: string; value: number },
|
||||
): T[] {
|
||||
const kept: T[] = [];
|
||||
const rejections: MeasurementRangeRejection[] = [];
|
||||
for (const row of rows) {
|
||||
const { type, value } = select(row);
|
||||
const rejection = classifyRangeRejection(type, value);
|
||||
if (rejection) rejections.push(rejection);
|
||||
else kept.push(row);
|
||||
}
|
||||
recordRangeRejections(source, rejections);
|
||||
return kept;
|
||||
}
|
||||
@@ -4,6 +4,11 @@ import {
|
||||
type MeasurementAggregationProvenance,
|
||||
type Prisma as PrismaTypes,
|
||||
} from "@/generated/prisma/client";
|
||||
import {
|
||||
classifyRangeRejection,
|
||||
recordRangeRejections,
|
||||
type MeasurementRangeRejection,
|
||||
} from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
const SAVEPOINT = "measurement_identity_reconcile";
|
||||
|
||||
@@ -50,6 +55,18 @@ export type MeasurementReconciliationVerdict =
|
||||
| {
|
||||
status: "failed";
|
||||
error: { message: string; code?: string };
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* The incoming value sits outside the metric's declared plausibility
|
||||
* band, so nothing was written and no existing row was touched. Not a
|
||||
* failure: the write was refused on purpose and the caller must carry
|
||||
* on with the rest of its batch. Every caller narrows on this status
|
||||
* because it carries no `row` — the compiler is what makes a future
|
||||
* provider handle it rather than dereference a row that does not exist.
|
||||
*/
|
||||
status: "rejected_range";
|
||||
rejection: MeasurementRangeRejection;
|
||||
};
|
||||
|
||||
export type FailedMeasurementReconciliation = Extract<
|
||||
@@ -219,12 +236,26 @@ async function retireCollision(
|
||||
* advisory lock serializes cooperative writers sharing either identity, and a
|
||||
* savepoint turns a non-benign database error into a structured `failed`
|
||||
* verdict without poisoning the caller's surrounding transaction.
|
||||
*
|
||||
* This is also the plausibility gate for every provider that reconciles rather
|
||||
* than writing Prisma directly. Gating HERE rather than at each caller is the
|
||||
* point: a provider added tomorrow inherits the refusal by construction, and
|
||||
* cannot land an impossible reading by forgetting a check it never knew about.
|
||||
*/
|
||||
export async function reconcileExternalMeasurement(
|
||||
tx: PrismaTypes.TransactionClient,
|
||||
desired: ExternalMeasurementWrite,
|
||||
options: MeasurementReconciliationOptions = {},
|
||||
): Promise<MeasurementReconciliationVerdict> {
|
||||
const rejection = classifyRangeRejection(desired.type, desired.value);
|
||||
if (rejection) {
|
||||
recordRangeRejections(
|
||||
desired.source ? String(desired.source).toLowerCase() : "external",
|
||||
[rejection],
|
||||
);
|
||||
return { status: "rejected_range", rejection };
|
||||
}
|
||||
|
||||
await tx.$executeRawUnsafe(`SAVEPOINT ${SAVEPOINT}`);
|
||||
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* The Nightscout pull writes BLOOD_GLUCOSE rows directly, so it does not
|
||||
* inherit the shared reconciler's gate. It is also first-write-wins: a row
|
||||
* that lands here is immutable, so an impossible sample never corrects itself
|
||||
* on a later pass. These cases pin the refusal and its annotation.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const {
|
||||
upsertMock,
|
||||
findManyMock,
|
||||
emitArrivalsMock,
|
||||
recomputeMock,
|
||||
invalidateMock,
|
||||
} = vi.hoisted(() => ({
|
||||
upsertMock: vi.fn(),
|
||||
findManyMock: vi.fn(),
|
||||
emitArrivalsMock: vi.fn(),
|
||||
recomputeMock: vi.fn(),
|
||||
invalidateMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: { measurement: { upsert: upsertMock, findMany: findManyMock } },
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/arrivals/measurement-emit", () => ({
|
||||
emitInsertedMeasurementArrivals: emitArrivalsMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) =>
|
||||
rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })),
|
||||
recomputeBucketsForMeasurement: recomputeMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: invalidateMock,
|
||||
}));
|
||||
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { upsertNightscoutEntries } from "../sync";
|
||||
|
||||
/** BLOOD_GLUCOSE is declared 20–800 mg/dL. */
|
||||
const REAL_ENTRY = { id: "abc", sgv: 112, date: 1718000000000 };
|
||||
/** A sensor fault, or an mg/dL field carrying something else entirely. */
|
||||
const IMPOSSIBLE_ENTRY = { id: "def", sgv: 41_600, date: 1718000300000 };
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
findManyMock.mockResolvedValue([]);
|
||||
upsertMock.mockResolvedValue({ id: "row-1" });
|
||||
emitArrivalsMock.mockResolvedValue(undefined);
|
||||
recomputeMock.mockResolvedValue(undefined);
|
||||
invalidateMock.mockResolvedValue(undefined);
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("upsertNightscoutEntries — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertNightscoutEntries("user-1", [REAL_ENTRY]),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ inserted: 1, failedRows: 0 });
|
||||
expect(upsertMock).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertNightscoutEntries("user-1", [REAL_ENTRY, IMPOSSIBLE_ENTRY]),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ inserted: 1, failedRows: 0 });
|
||||
expect(upsertMock).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
(upsertMock.mock.calls[0]![0] as { create: { value: number } }).create
|
||||
.value,
|
||||
).toBe(112);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "nightscout",
|
||||
type: "BLOOD_GLUCOSE",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("writes nothing when the whole page is out of band", async () => {
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertNightscoutEntries("user-1", [IMPOSSIBLE_ENTRY]),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, failedRows: 0 });
|
||||
expect(upsertMock).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,7 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
markSyncFailureRecorded,
|
||||
recordSyncFailure,
|
||||
@@ -221,7 +222,16 @@ export async function upsertNightscoutEntries(
|
||||
if (entries.length === 0) return { inserted: 0, failedRows: 0 };
|
||||
|
||||
const type = "BLOOD_GLUCOSE" as MeasurementType;
|
||||
const mapped = entries.map((entry) => mapSgvEntryToMeasurement(entry));
|
||||
// A CGM that reports a glucose the app's own band calls impossible is
|
||||
// reporting a sensor fault or a unit slip, not a reading. Drop it: the row
|
||||
// would otherwise be immutable (first-write-wins), so an impossible sample
|
||||
// that lands here never corrects itself on a later pass.
|
||||
const mapped = dropImplausibleMeasurements(
|
||||
"nightscout",
|
||||
entries.map((entry) => mapSgvEntryToMeasurement(entry)),
|
||||
(m) => ({ type, value: m.value }),
|
||||
);
|
||||
if (mapped.length === 0) return { inserted: 0, failedRows: 0 };
|
||||
|
||||
// Probe which keys already exist so a re-confirmed reading (the upsert's
|
||||
// no-op update branch) is not miscounted as a fresh import and the arrival
|
||||
|
||||
@@ -575,6 +575,10 @@ export async function upsertOuraMeasurements(
|
||||
for (let index = 0; index < readings.length; index++) {
|
||||
const reading = readings[index]!;
|
||||
const verdict = verdicts[index]!;
|
||||
// Refused by the plausibility gate: nothing was written and nothing was
|
||||
// touched, so the reading counts as neither imported nor a rollup dirt
|
||||
// mark. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
|
||||
@@ -364,6 +364,10 @@ export async function upsertPolarMeasurements(
|
||||
for (let index = 0; index < readings.length; index++) {
|
||||
const reading = readings[index]!;
|
||||
const verdict = verdicts[index]!;
|
||||
// Refused by the plausibility gate: nothing was written and nothing was
|
||||
// touched, so the reading counts as neither imported nor a rollup dirt
|
||||
// mark. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
|
||||
@@ -255,11 +255,29 @@ const moodEntrySchema = z
|
||||
mood: z.string().min(1),
|
||||
score: z.number().int().min(0).max(10),
|
||||
tags: moodEntryTagsSchema,
|
||||
// The free text, in both storage shapes. A portable export carries the
|
||||
// decrypted `note`; a disaster-recovery file carries `noteEncrypted` plus
|
||||
// whatever legacy plaintext the row still holds. Same pair, and same
|
||||
// handling on restore, as `Measurement.notes` / `notesEncrypted`.
|
||||
note: z.string().nullable().optional(),
|
||||
noteEncrypted: base64BytesSchema.nullable().optional(),
|
||||
source: z.string().min(1).optional(),
|
||||
externalId: z.string().nullable().optional(),
|
||||
loggedAt: isoDateTime,
|
||||
// The IANA zone the `date` string is anchored to. Absent means the legacy
|
||||
// Europe/Berlin reading, which is what a row written before v1.4.25 means
|
||||
// by a NULL here — so an old file keeps its old meaning exactly.
|
||||
tz: z.string().nullable().optional(),
|
||||
syncedAt: isoDateTime.optional(),
|
||||
syncVersion: z.number().int().optional(),
|
||||
deletedAt: isoDateTime.nullable().optional(),
|
||||
createdAt: isoDateTime.optional(),
|
||||
updatedAt: isoDateTime.optional(),
|
||||
factors: z.array(moodFactorSchema).default([]),
|
||||
// BINARY structured-tag keys. Separate from `factors`, which keeps its
|
||||
// RATED-only meaning, so every file written before this one parses
|
||||
// unchanged and restores identically: absent reads as none.
|
||||
structuredTags: z.array(z.string().min(1)).default([]),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
|
||||
@@ -480,6 +480,10 @@ export async function upsertWhoopMeasurements(
|
||||
for (let index = chunkStart; index < chunkEnd; index++) {
|
||||
const reading = readings[index]!;
|
||||
const verdict = verdicts[index - chunkStart]!;
|
||||
// Refused by the plausibility gate: nothing was written and nothing was
|
||||
// touched, so the reading counts as neither imported nor a rollup dirt
|
||||
// mark. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
touchedTypes.add(dirty.type);
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Withings writes the measurement table directly — the measures leg row by
|
||||
* row, the activity leg as a planned batch — so neither inherits the shared
|
||||
* reconciler's gate. These cases pin that both legs now clear the app's own
|
||||
* declared plausibility band before anything reaches Prisma, and that a
|
||||
* refusal lands on the wide event rather than vanishing.
|
||||
*
|
||||
* The measures leg matters twice over: it does not only insert, it OVERWRITES
|
||||
* the stored value of an existing slot, so an ungated impossible reading could
|
||||
* replace a real one.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findFirst: vi.fn(),
|
||||
findMany: vi.fn(),
|
||||
create: vi.fn(),
|
||||
createMany: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
withingsConnection: {
|
||||
findUnique: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
$queryRaw: vi.fn(),
|
||||
$transaction: vi.fn((ops: unknown) =>
|
||||
Array.isArray(ops) ? Promise.all(ops) : Promise.resolve(),
|
||||
),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({
|
||||
decrypt: vi.fn((v: string) => v),
|
||||
encrypt: vi.fn((v: string) => v),
|
||||
}));
|
||||
|
||||
vi.mock("../client", async () => {
|
||||
const actual = await vi.importActual<typeof import("../client")>("../client");
|
||||
return {
|
||||
...actual,
|
||||
fetchMeasurements: vi.fn(),
|
||||
refreshAccessToken: vi.fn(),
|
||||
subscribeWebhook: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserWithingsCredentials: vi.fn(async () => ({
|
||||
clientId: "cid",
|
||||
clientSecret: "secret",
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn().mockResolvedValue(false),
|
||||
parkIntegrationAtReauth: vi.fn(),
|
||||
recordSyncFailure: vi.fn().mockResolvedValue(undefined),
|
||||
recordSyncSuccess: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@/lib/rollups/measurement-rollups")
|
||||
>("@/lib/rollups/measurement-rollups");
|
||||
return {
|
||||
...actual,
|
||||
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/cache/invalidate", () => ({
|
||||
invalidateUserDashboardSnapshot: vi.fn(),
|
||||
}));
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { fetchMeasurements } from "../client";
|
||||
import { syncUserActivity } from "../sync-activity";
|
||||
import { syncUserMeasurements } from "../sync";
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
function installActivityFetch(
|
||||
entries: Array<{ date: string; steps?: number; distance?: number }>,
|
||||
) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers(),
|
||||
json: async () => ({
|
||||
status: 0,
|
||||
body: { activities: entries, more: false, offset: 0 },
|
||||
}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
event = new WideEventBuilder("background");
|
||||
vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({
|
||||
id: "conn-1",
|
||||
withingsUserId: "wu-1",
|
||||
accessToken: "enc-access",
|
||||
refreshToken: "enc-refresh",
|
||||
scope: "user.metrics,user.activity",
|
||||
tokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000),
|
||||
lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000),
|
||||
} as never);
|
||||
vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never);
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.create).mockResolvedValue({
|
||||
id: "m-1",
|
||||
type: "PULSE",
|
||||
measuredAt: new Date(),
|
||||
} as never);
|
||||
vi.mocked(prisma.measurement.createMany).mockResolvedValue({
|
||||
count: 1,
|
||||
} as never);
|
||||
vi.mocked(prisma.measurement.update).mockResolvedValue({} as never);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("syncUserMeasurements — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
vi.mocked(fetchMeasurements).mockResolvedValue([
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 112,
|
||||
measuredAt: new Date("2026-05-16T08:00:00Z"),
|
||||
},
|
||||
] as never);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
syncUserMeasurements("user-1"),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ imported: 1, failed: false });
|
||||
expect(prisma.measurement.create).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
vi.mocked(fetchMeasurements).mockResolvedValue([
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 112,
|
||||
measuredAt: new Date("2026-05-16T08:00:00Z"),
|
||||
},
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
measuredAt: new Date("2026-05-16T08:05:00Z"),
|
||||
},
|
||||
] as never);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
syncUserMeasurements("user-1"),
|
||||
);
|
||||
|
||||
// Only the real reading landed, and the run is still a clean success: one
|
||||
// impossible sample must not strand the rest behind a held watermark.
|
||||
expect(result).toEqual({ imported: 1, failed: false });
|
||||
expect(prisma.measurement.create).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{ source: "withings", type: "PULSE", direction: "above_max", count: 1 },
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("does not overwrite a stored reading with an out-of-band one", async () => {
|
||||
// The slot already holds a real pulse; the provider re-sends the same
|
||||
// instant carrying a decode slip. Ungated, the update branch rewrote the
|
||||
// good value with the bad one.
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue({
|
||||
id: "existing-row",
|
||||
} as never);
|
||||
vi.mocked(fetchMeasurements).mockResolvedValue([
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
measuredAt: new Date("2026-05-16T08:00:00Z"),
|
||||
},
|
||||
] as never);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
syncUserMeasurements("user-1"),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ imported: 0, failed: false });
|
||||
expect(prisma.measurement.update).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncUserActivity — plausibility gate", () => {
|
||||
it("plans a daily total inside the metric's declared band", async () => {
|
||||
installActivityFetch([{ date: "2026-05-16", steps: 8_400 }]);
|
||||
|
||||
const imported = await eventStorage.run(event, () =>
|
||||
syncUserActivity("user-1"),
|
||||
);
|
||||
|
||||
expect(imported).toBe(1);
|
||||
expect(prisma.measurement.createMany).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band daily total and annotates the drop", async () => {
|
||||
// ACTIVITY_STEPS is declared 0–200 000.
|
||||
installActivityFetch([
|
||||
{ date: "2026-05-16", steps: 8_400 },
|
||||
{ date: "2026-05-17", steps: 9_900_000 },
|
||||
]);
|
||||
|
||||
const imported = await eventStorage.run(event, () =>
|
||||
syncUserActivity("user-1"),
|
||||
);
|
||||
|
||||
expect(imported).toBe(1);
|
||||
const created = vi.mocked(prisma.measurement.createMany).mock.calls[0]?.[0]
|
||||
?.data as Array<{ value: number }>;
|
||||
expect(created).toHaveLength(1);
|
||||
expect(created[0]!.value).toBe(8_400);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "withings",
|
||||
type: "ACTIVITY_STEPS",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
});
|
||||
@@ -39,6 +39,7 @@ import { prisma } from "@/lib/db";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { safeFetch } from "@/lib/safe-fetch";
|
||||
import { getUnitForType } from "@/lib/validations/measurement";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
collapseToTypeDayKeys,
|
||||
recomputeBucketsForMeasurement,
|
||||
@@ -342,7 +343,7 @@ export async function syncUserActivity(
|
||||
value: number;
|
||||
externalId: string;
|
||||
}
|
||||
const planned: PlannedRow[] = [];
|
||||
const collected: PlannedRow[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.date) continue;
|
||||
const measuredAt = activityMeasuredAt(entry.date);
|
||||
@@ -352,7 +353,7 @@ export async function syncUserActivity(
|
||||
// valid (a day of rest still records 0 steps). Only skip
|
||||
// undefined / null / NaN.
|
||||
if (raw == null || !Number.isFinite(raw)) continue;
|
||||
planned.push({
|
||||
collected.push({
|
||||
type,
|
||||
measuredAt,
|
||||
value: raw,
|
||||
@@ -361,6 +362,15 @@ export async function syncUserActivity(
|
||||
}
|
||||
}
|
||||
|
||||
// A daily-activity total the app's own band calls impossible (a step count
|
||||
// past 200 000, a negative distance) is a provider artefact, not a day. Drop
|
||||
// it before the plan is read against the table, so it can neither insert a
|
||||
// fresh impossible slot nor overwrite a good stored one.
|
||||
const planned = dropImplausibleMeasurements("withings", collected, (p) => ({
|
||||
type: p.type,
|
||||
value: p.value,
|
||||
}));
|
||||
|
||||
if (planned.length > 0) {
|
||||
const slotKey = (type: MeasurementType, measuredAt: Date) =>
|
||||
`${type}|${measuredAt.getTime()}`;
|
||||
|
||||
@@ -474,6 +474,9 @@ export async function syncUserSleep(
|
||||
if (verdict.status === "failed") {
|
||||
throw new MeasurementReconciliationError(verdict);
|
||||
}
|
||||
// Refused by the plausibility gate: nothing was written and nothing
|
||||
// was touched. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
@@ -542,6 +545,9 @@ export async function syncUserSleep(
|
||||
if (verdict.status === "failed") {
|
||||
throw new MeasurementReconciliationError(verdict);
|
||||
}
|
||||
// Refused by the plausibility gate: nothing was written and
|
||||
// nothing was touched. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import { getUserWithingsCredentials } from "./credentials";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { isP2002 } from "@/lib/prisma-errors";
|
||||
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
isReauthRequired,
|
||||
recordSyncFailure,
|
||||
@@ -229,14 +230,24 @@ export async function syncUserMeasurements(
|
||||
)
|
||||
: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days back
|
||||
|
||||
let measures: Awaited<ReturnType<typeof fetchMeasurements>>;
|
||||
let fetched: Awaited<ReturnType<typeof fetchMeasurements>>;
|
||||
try {
|
||||
measures = await fetchMeasurements(tokenInfo.accessToken, startDate);
|
||||
fetched = await fetchMeasurements(tokenInfo.accessToken, startDate);
|
||||
} catch (err) {
|
||||
await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES);
|
||||
throw err;
|
||||
}
|
||||
|
||||
// The app's own plausibility band applies to a provider reading exactly as
|
||||
// it applies to a typed one. A remote that reports a pulse in the millions
|
||||
// is reporting a decode slip, not a heartbeat, and the row would poison
|
||||
// every mean and personal band derived from it. Refuse it here, before the
|
||||
// update branch below can overwrite a good stored value with it.
|
||||
const measures = dropImplausibleMeasurements("withings", fetched, (m) => ({
|
||||
type: m.type,
|
||||
value: m.value,
|
||||
}));
|
||||
|
||||
let imported = 0;
|
||||
// v1.4.39.1 — track every (type, measuredAt) we touched so the
|
||||
// persistent rollup tier can be re-folded at the end of the sync.
|
||||
|
||||
Reference in new issue
Block a user