From 09fbac3a33b8d2991679911b9d7ae2e55d0cab2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 14 Aug 2026 10:08:43 +0200 Subject: [PATCH] Hygiene: drop dead export modifiers, delete the dead series module, wire the tokenizer version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The knip blind spot (ignoreExportsUsedInFile) left eight symbols carrying an export nothing outside their file consumes — the modifier goes, the symbols stay. The score series module goes whole: its only importer was its own guard test, whose composite- and config-guarding suites survive in place. DocumentContentIndex.tokenizerVersion gains the consumer that was the point of storing it: the content-index job now re-tokenises rows whose version trails CONTENT_TOKENIZER_VERSION, from the decrypted index text along the same path the rotation script uses, keyset-paginated and fault-isolated per row — a future tokenizer bump migrates the index instead of silently splitting it. ImportJob.uploadSha256 turned out to already have its dedupe consumer (both kick-off routes short-circuit on same-bytes non-failed jobs); its comment now names that instead of pretending to be write-only. The remaining write-only columns carry their honest annotations in the shape the audit sweep parses. --- prisma/schema.prisma | 49 +++++- src/lib/ai/coach/outbound-guard.ts | 2 +- src/lib/ai/coach/refusal.ts | 5 +- .../__tests__/config-delta-guard.test.ts | 147 +--------------- src/lib/analytics/score/breadth.ts | 2 +- src/lib/analytics/score/config.ts | 2 +- src/lib/analytics/score/glycaemia.ts | 2 +- src/lib/analytics/score/series.ts | 155 ---------------- src/lib/analytics/score/shared.ts | 2 +- .../document-content-index-backfill.test.ts | 166 +++++++++++++++++- .../jobs/document-content-index-backfill.ts | 104 ++++++++++- src/lib/jobs/reminder/register-maintenance.ts | 6 +- src/lib/rollups/tiered-context.ts | 2 +- src/lib/targets/consistency.ts | 2 +- 14 files changed, 315 insertions(+), 331 deletions(-) delete mode 100644 src/lib/analytics/score/series.ts diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b2c46a171..af1b34cd0 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -102,6 +102,10 @@ model User { // Codex OAuth (ChatGPT subscription) codexAccessTokenEncrypted String? @map("codex_access_token_encrypted") codexRefreshTokenEncrypted String? @map("codex_refresh_token_encrypted") + /// @internal: dead mirror of the expiry inside the encrypted Codex creds — + /// the refresh path reads the decrypted blob's own `expiresAt` + /// (`src/lib/ai/provider.ts`), never this column. Kept (and kept written) + /// for wire/backup compatibility rather than dropped. codexTokenExpiresAt DateTime? @map("codex_token_expires_at") codexConnectedAt DateTime? @map("codex_connected_at") codexConnectionStatus String @default("disconnected") @map("codex_connection_status") @@ -1048,6 +1052,9 @@ model EnvironmentContext { /// that won precedence). lat Float lon Float + /// @internal: human-readable label of the resolved location, stored as + /// operator/debug provenance beside the coordinates; readers select the + /// numeric and weather columns only, never this label. locationLabel String @map("location_label") source EnvironmentLocationSource @@ -2419,6 +2426,9 @@ model PersonalRecord { /// FK to the Measurement row that achieved the record. SET NULL /// on delete so the historical fact survives even if the /// underlying measurement is later removed. + /// @internal: audit-trail provenance written by the PR detection worker; + /// no read path selects it — the record row itself carries the surfaced + /// value/unit/achievedAt. sourceMeasurementId String? @map("source_measurement_id") source MeasurementSource @default(MANUAL) externalId String? @map("external_id") @@ -3584,10 +3594,16 @@ model McpOAuthConnection { model InviteToken { id String @id @default(cuid()) tokenHash String @unique @map("token_hash") + /// Consumed through the `creator` relation: the admin invites list + /// (`src/app/api/admin/invites/route.ts`) selects the creating admin's + /// id + username via it. createdBy String @map("created_by") createdAt DateTime @default(now()) @map("created_at") expiresAt DateTime @map("expires_at") usedAt DateTime? @map("used_at") + /// LAST consumer only (overwritten on multi-use invites; the full history + /// lives in `InviteRedemption`). Consumed through the `consumer` relation + /// by the admin invites list (`src/app/api/admin/invites/route.ts`). usedBy String? @map("used_by") maxUses Int @default(1) @map("max_uses") uses Int @default(0) @@ -3644,8 +3660,11 @@ model RefreshToken { /// When this token was used (one-time-use). Set during rotation; a second /// use of the same token is a hard fail and triggers reuse-detection. usedAt DateTime? @map("used_at") - /// Pointer to the row that replaced this one during rotation. Helps audit - /// trail and reuse-detection ("this token was already exchanged into X"). + /// Pointer to the row that replaced this one during rotation ("this token + /// was already exchanged into X"). + /// @internal: audit-trail bookkeeping written by the rotation path + /// (`src/lib/auth/refresh-token.ts`); reuse-detection keys on + /// `usedAt`/`tokenHash`, so no code path reads this pointer. replacedById String? @map("replaced_by_id") userAgent String? @map("user_agent") ipAddress String? @map("ip_address") @@ -4923,12 +4942,18 @@ model ImportJob { failureReason String? @map("failure_reason") /// Bytes uploaded — surfaced to the operator audit trail. uploadBytes Int @map("upload_bytes") - /// SHA-256 hex digest of the uploaded `export.zip`. Lets the - /// kick-off endpoint short-circuit a re-upload of the exact same - /// file back to the existing job id without re-queueing. + /// SHA-256 hex digest of the uploaded `export.zip`. Consumer: both kick-off + /// endpoints (`src/app/api/import/apple-health-export/route.ts` and the + /// admin variant) dedupe on (userId, uploadSha256, parserRevision) and + /// short-circuit a re-upload of the exact same file back to the existing + /// job id without re-queueing. A prior `failed` job is excluded from the + /// match (issue #486), so the same bytes can always be retried. uploadSha256 String? @map("upload_sha256") /// Parser semantics revision. Existing archives migrated at revision 1 can - /// be deliberately reprocessed when the current parser revision advances. + /// be deliberately reprocessed when the current parser revision advances: + /// the kick-off dedupe lookup (see `uploadSha256`) pins the CURRENT + /// `APPLE_HEALTH_IMPORT_PARSER_REVISION`, so a job parsed under an older + /// revision never short-circuits a fresh upload of the same bytes. parserRevision Int @default(1) @map("parser_revision") /// Apple's `ExportDate` attribute string-formatted, parsed during /// the unpack phase. NULL until the parser sees it. @@ -5769,6 +5794,8 @@ model CoachConversation { // conversation keeps memory of what came before without re-sending every // turn to the provider. AES-256-GCM (same codec as CoachMessage.encrypted). summaryEncrypted Bytes? @map("summary_encrypted") + /// @internal: diagnostic timestamp of the last summary rewrite — the elision + /// logic keys on `summaryTurnCount`, not on this instant; nothing reads it. summaryUpdatedAt DateTime? @map("summary_updated_at") summaryTurnCount Int @default(0) @map("summary_turn_count") @@ -7787,7 +7814,12 @@ model DocumentContentIndex { /// Provider type that produced the text (diagnostic only), or null on the /// text-ocr path. providerType String? @map("provider_type") - /// Tokeniser algorithm version — bump to force a re-index on a scheme change. + /// Tokeniser algorithm version — bump `CONTENT_TOKENIZER_VERSION` + /// (`src/lib/documents/content-index.ts`) to force a re-index on a scheme + /// change. Consumer: the content-index backfill job re-tokenises any row + /// whose stored version trails the current one from the decrypted + /// `textEncrypted` (`retokeniseStaleRows` in + /// `src/lib/jobs/document-content-index-backfill.ts`). tokenizerVersion String @map("tokenizer_version") createdAt DateTime @default(now()) @map("created_at") @@ -7824,7 +7856,8 @@ model DocumentThumbnail { height Int /// Plaintext byte length of the JPEG (pre-encryption), for `Content-Length`. byteSize Int @map("byte_size") - /// Source MIME the preview was rendered from (diagnostic only). + /// @internal: source MIME the preview was rendered from — operator-side + /// diagnostic provenance only; no query selects it and no client surfaces it. sourceMime String @map("source_mime") createdAt DateTime @default(now()) @map("created_at") diff --git a/src/lib/ai/coach/outbound-guard.ts b/src/lib/ai/coach/outbound-guard.ts index 9baede2a9..10d21bf9f 100644 --- a/src/lib/ai/coach/outbound-guard.ts +++ b/src/lib/ai/coach/outbound-guard.ts @@ -88,7 +88,7 @@ export const COACH_OUTBOUND_DOSE_BLOCK_DE = export const COACH_OUTBOUND_RISK_BLOCK_EN = "I can't put a number on a clinical risk like that — it isn't something your tracked data lets me compute, and an invented figure would be misleading. I can walk you through what your own measurements are doing instead; ask me about a specific metric."; -export const COACH_OUTBOUND_RISK_BLOCK_DE = +const COACH_OUTBOUND_RISK_BLOCK_DE = "Eine solche klinische Risikozahl kann ich nicht angeben — deine erfassten Daten lassen das nicht berechnen, und ein erfundener Wert wäre irreführend. Stattdessen kann ich dir zeigen, was deine eigenen Messwerte machen; frag mich nach einer konkreten Metrik."; /** Resolve the localised fallback copy for a tripped outbound screen. */ diff --git a/src/lib/ai/coach/refusal.ts b/src/lib/ai/coach/refusal.ts index e68330b23..2899a5b2c 100644 --- a/src/lib/ai/coach/refusal.ts +++ b/src/lib/ai/coach/refusal.ts @@ -40,10 +40,7 @@ import { getServerTranslator } from "@/lib/i18n/server-translator"; * exact wording without a translator round-trip and cannot drift from what the * user is actually shown. */ -export function coachRefusalCopy( - reason: CoachRefusalReason, - locale: Locale, -): string { +function coachRefusalCopy(reason: CoachRefusalReason, locale: Locale): string { return getServerTranslator(locale).t( reason === "prompt_injection" ? "coach.refusal.promptInjection" diff --git a/src/lib/analytics/score/__tests__/config-delta-guard.test.ts b/src/lib/analytics/score/__tests__/config-delta-guard.test.ts index 01d1419cc..fa0ca4253 100644 --- a/src/lib/analytics/score/__tests__/config-delta-guard.test.ts +++ b/src/lib/analytics/score/__tests__/config-delta-guard.test.ts @@ -1,11 +1,11 @@ /** - * The per-user recipe boundary, at the three places it exists. + * The per-user recipe boundary, at the two places it exists. * * The end-to-end proof that a settings change is never narrated as a * health event lives in `config-change-is-not-a-health-event.test.ts`, * over the real snapshot and digest. This file covers what that one * cannot reach without a fixture per case: the boundary arithmetic's - * edges, the notice key's two dimensions, and the seam's shape. + * edges and the notice key's two dimensions. * * Each guard here has a counter-case beside it. A boundary that * suppressed everything would pass every "no false drop" assertion ever @@ -27,12 +27,6 @@ import { scoreConfigBoundary, UNCONFIGURED_SCORE_BOUNDARY, } from "../config"; -import { - scoreCompositeSegments, - scorePillarSeries, - scoreSeriesSeams, - type ScoreSeriesRecord, -} from "../series"; import type { CompositeValue, PillarValue, @@ -337,140 +331,3 @@ describe("the notice key", () => { ); }); }); - -describe("the seam", () => { - function day( - dayKey: string, - configVersion: number | null, - ): ScoreSeriesRecord { - return { - dayKey, - composite: 70, - band: "yellow", - composition: ["ACTIVITY", "SLEEP", "ADIPOSITY"], - pillarScores: { ACTIVITY: 70, SLEEP: 70, ADIPOSITY: 70 }, - configVersion, - }; - } - - it("marks the first day after a change and nothing else", () => { - const points = scoreSeriesSeams([ - day("2026-08-17", 1), - day("2026-08-18", 1), - day("2026-08-19", 2), - day("2026-08-20", 2), - ]); - - expect(points.map((point) => point.seamBreak)).toEqual([ - false, - false, - true, - false, - ]); - }); - - it("never marks the first day of the series", () => { - // A beginning is not a break. Flagging it would put a "you changed - // what counts here" marker on the day someone started. - expect(scoreSeriesSeams([day("2026-08-20", 3)])[0].seamBreak).toBe(false); - expect(scoreSeriesSeams([])).toEqual([]); - }); - - it("stays quiet across days that share a recipe", () => { - const points = scoreSeriesSeams([ - day("2026-08-18", 0), - day("2026-08-19", 0), - day("2026-08-20", 0), - ]); - expect(points.some((point) => point.seamBreak)).toBe(false); - expect(scoreCompositeSegments(points)).toHaveLength(1); - }); - - it("orders by day before comparing, so an unordered read cannot invent a seam", () => { - const points = scoreSeriesSeams([ - day("2026-08-20", 2), - day("2026-08-18", 1), - day("2026-08-19", 2), - ]); - expect(points.map((point) => point.dayKey)).toEqual([ - "2026-08-18", - "2026-08-19", - "2026-08-20", - ]); - expect(points.map((point) => point.seamBreak)).toEqual([ - false, - true, - false, - ]); - }); - - it("treats a day that names no recipe as a break rather than a link", () => { - // Both directions, including two unknowns in a row. Without the - // second case the arm would be doing nothing that plain inequality - // does not already do: `null !== 1` breaks on its own, while - // `null !== null` would quietly claim two rows share a basis that - // neither of them names. - expect( - scoreSeriesSeams([day("2026-08-19", null), day("2026-08-20", 1)])[1] - .seamBreak, - ).toBe(true); - expect( - scoreSeriesSeams([day("2026-08-19", 1), day("2026-08-20", null)])[1] - .seamBreak, - ).toBe(true); - expect( - scoreSeriesSeams([day("2026-08-19", null), day("2026-08-20", null)])[1] - .seamBreak, - ).toBe(true); - }); - - it("breaks the composite line and never rejoins it", () => { - const points = scoreSeriesSeams([ - day("2026-08-17", 1), - day("2026-08-18", 1), - day("2026-08-19", 2), - day("2026-08-20", 2), - ]); - const segments = scoreCompositeSegments(points); - - expect(segments).toHaveLength(2); - expect(segments[0].map((point) => point.dayKey)).toEqual([ - "2026-08-17", - "2026-08-18", - ]); - expect(segments[1].map((point) => point.dayKey)).toEqual([ - "2026-08-19", - "2026-08-20", - ]); - // Every day is in exactly one segment: no day is repeated to make - // the ends meet, and none is dropped to hide the gap. - expect(segments.flat().map((point) => point.dayKey)).toEqual( - points.map((point) => point.dayKey), - ); - }); - - it("keeps a pillar's own row running straight through the seam", () => { - const points = scoreSeriesSeams([ - { ...day("2026-08-18", 1), pillarScores: { SLEEP: 61 } }, - { ...day("2026-08-19", 2), pillarScores: { SLEEP: 62 } }, - { ...day("2026-08-20", 2), pillarScores: { SLEEP: 63 } }, - ]); - - expect(points[1].seamBreak).toBe(true); - expect(scorePillarSeries(points, "SLEEP")).toEqual([ - { dayKey: "2026-08-18", score: 61 }, - { dayKey: "2026-08-19", score: 62 }, - { dayKey: "2026-08-20", score: 63 }, - ]); - }); - - it("leaves out a day the pillar did not count on, rather than scoring it zero", () => { - const points = scoreSeriesSeams([ - { ...day("2026-08-19", 1), pillarScores: { SLEEP: 61 } }, - { ...day("2026-08-20", 1), pillarScores: { ACTIVITY: 70 } }, - ]); - expect(scorePillarSeries(points, "SLEEP")).toEqual([ - { dayKey: "2026-08-19", score: 61 }, - ]); - }); -}); diff --git a/src/lib/analytics/score/breadth.ts b/src/lib/analytics/score/breadth.ts index 7f1aae52c..302c27514 100644 --- a/src/lib/analytics/score/breadth.ts +++ b/src/lib/analytics/score/breadth.ts @@ -23,7 +23,7 @@ export const SCORE_MIN_ELIGIBLE_DOMAINS = 3; * `WELLBEING` are the only two that do not, which is why a selection of * those two alone yields no score however many domains it spans. */ -export const PHYSIOLOGICAL_PILLARS: ReadonlySet = new Set([ +const PHYSIOLOGICAL_PILLARS: ReadonlySet = new Set([ "BLOOD_PRESSURE", "GLYCAEMIA", "SLEEP", diff --git a/src/lib/analytics/score/config.ts b/src/lib/analytics/score/config.ts index 5db900a11..32242119f 100644 --- a/src/lib/analytics/score/config.ts +++ b/src/lib/analytics/score/config.ts @@ -88,7 +88,7 @@ export const scorePillarIdSchema = z.enum(SCORE_PILLAR_IDS); * REQUEST body is the strict end: an unknown id there is a 422, because * a person's client has no business sending one. */ -export const healthScoreConfigSchema = z.object({ +const healthScoreConfigSchema = z.object({ excludedPillars: z.array(z.string().max(64)).max(64).optional(), /** Per-user recipe version. Increments on every accepted write. */ version: z.number().int().min(1).optional(), diff --git a/src/lib/analytics/score/glycaemia.ts b/src/lib/analytics/score/glycaemia.ts index 7dfdbfbec..08cb64555 100644 --- a/src/lib/analytics/score/glycaemia.ts +++ b/src/lib/analytics/score/glycaemia.ts @@ -17,7 +17,7 @@ import { export const HBA1C_WINDOW_DAYS = 180; export const FASTING_GLUCOSE_WINDOW_DAYS = 90; export const FASTING_GLUCOSE_MIN_READINGS = 8; -export const HBA1C_REFERENCE = { low: 5, high: 5.6 } as const; +const HBA1C_REFERENCE = { low: 5, high: 5.6 } as const; export const FASTING_GLUCOSE_REFERENCE = { low: 70, high: 99 } as const; export function computeGlycaemiaPillar( diff --git a/src/lib/analytics/score/series.ts b/src/lib/analytics/score/series.ts deleted file mode 100644 index cd05f857e..000000000 --- a/src/lib/analytics/score/series.ts +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Reading a run of stored score days as a series, and refusing to draw - * a line through the place where the recipe changed. - * - * Official statistics has one convention for this and it is small: a - * single machine-readable flag on the FIRST observation after the break - * (Eurostat's `b`, "break occurring when there is a change in the - * standards for defining and observing a variable over time"). This - * module is that flag, plus the two consequences that follow from it. - * - * **The composite line breaks.** Two segments, visibly two. A recipe - * change fails the equal-construct requirement for a legitimate - * equating by construction: the numbers on either side are averages of - * different things. The stored day under the old recipe is an overlap - * point of exactly one, which is enough to date the seam and enough to - * refuse a comparison across it, and not nearly enough to compute an - * adjustment factor that would rescale one segment onto the other. So - * no adjustment is offered. There is no "adjusted" variant of - * {@link scoreCompositeSegments} and there should not be one: splicing - * two methods into one continuous line is the failure that produced - * "years of sleep tracking might be no more useful than common sense", - * and it always starts as a helpful-looking option. - * - * **The per-pillar rows do not break.** A pillar's own 0-100 score is - * graded against its own published reference band and means the same - * thing on both sides of the seam. Only the aggregate changes basis. - * That is why the record stores the per-pillar scores beside the - * composite rather than folding them away, and it is what lets someone - * look at their sleep across a period that spans a recipe change - * without a gap. - * - * The seam is drawn on the recipe's VERSION, not on whether the - * composition visibly moved. A person who deselects a pillar that had - * no data that week changes what counts even though this week's number - * did not budge, and the delta guard suppresses that week from less - * information than this module has. Two surfaces disagreeing about - * whether the same act was an event is worse than either answer, so - * both key on the recipe's identity and describe one event. - * - * Pure. No database, no clock. The trend view that renders the segments - * is deliberately not in this release — the records start empty, so - * there is nothing yet to show — but the flag and the refusal ship with - * the machinery that creates the seam rather than after it. - */ -import type { ScoreBand, ScorePillarId } from "./types"; - -/** - * One stored score day, in the shape `HealthScoreRecord` rows already - * have, so a Prisma `select` feeds this without a mapping layer in - * between to get wrong. - */ -export interface ScoreSeriesRecord { - /** Local calendar day, `YYYY-MM-DD`. */ - dayKey: string; - composite: number; - band: ScoreBand; - composition: string[]; - pillarScores: Record; - /** The recipe version the day was scored under; null on a row written - * before the writer supplied one. */ - configVersion: number | null; -} - -export interface ScoreSeriesPoint extends ScoreSeriesRecord { - /** - * Eurostat `b`. True on the first observation computed under a - * different recipe than the observation before it. Never true on the - * first observation of the series: a beginning is not a break. - */ - seamBreak: boolean; -} - -/** Ascending by local day. Lexicographic order IS chronological here. */ -function byDay(left: ScoreSeriesRecord, right: ScoreSeriesRecord): number { - return left.dayKey < right.dayKey ? -1 : left.dayKey > right.dayKey ? 1 : 0; -} - -/** - * Whether two consecutive days sit on different bases. - * - * An unknown version on either side counts as a break. The seam's job - * is to refuse a link it cannot justify, and "these two rows were - * scored under the same recipe" is a claim: a row that does not say - * which recipe produced it cannot support it. Only rows predating the - * writer carry null, so in practice this arm is the answer to a - * question production never asks — which is the point of answering it - * deliberately rather than letting `null !== 0` decide by accident. - */ -function breaksFrom( - previous: ScoreSeriesRecord, - current: ScoreSeriesRecord, -): boolean { - if (previous.configVersion === null || current.configVersion === null) { - return true; - } - return previous.configVersion !== current.configVersion; -} - -/** - * Mark every stored day with whether it opens a new basis. - * - * Input order does not matter; the result is ascending by day. - */ -export function scoreSeriesSeams( - records: readonly ScoreSeriesRecord[], -): ScoreSeriesPoint[] { - return [...records].sort(byDay).map((record, index, ordered) => ({ - ...record, - seamBreak: index > 0 && breaksFrom(ordered[index - 1], record), - })); -} - -/** - * The composite series, cut into the segments that may honestly be - * drawn as one line each. - * - * Every seam starts a new segment, and nothing here joins two of them: - * no interpolation across the gap, no offset applied to make the ends - * meet, no single array with a hole in it that a chart library would - * happily bridge. A caller that wants one line gets the segments and - * has to decide, visibly, what to do about the fact that there are two. - */ -export function scoreCompositeSegments( - points: readonly ScoreSeriesPoint[], -): ScoreSeriesPoint[][] { - const segments: ScoreSeriesPoint[][] = []; - for (const point of points) { - if (segments.length === 0 || point.seamBreak) { - segments.push([point]); - continue; - } - segments[segments.length - 1].push(point); - } - return segments; -} - -/** - * One pillar's own scores across the whole run, seams and all. - * - * Continuous on purpose. The pillar is graded against its own published - * reference band on both sides of a recipe change, so it stays - * comparable when the average of the pillars does not. Days on which - * the pillar did not count are absent rather than zero. - */ -export function scorePillarSeries( - points: readonly ScoreSeriesPoint[], - pillar: ScorePillarId, -): Array<{ dayKey: string; score: number }> { - return points.flatMap((point) => { - const score = point.pillarScores[pillar]; - return typeof score === "number" && Number.isFinite(score) - ? [{ dayKey: point.dayKey, score }] - : []; - }); -} diff --git a/src/lib/analytics/score/shared.ts b/src/lib/analytics/score/shared.ts index fde7fffd7..e5dfbbdfa 100644 --- a/src/lib/analytics/score/shared.ts +++ b/src/lib/analytics/score/shared.ts @@ -23,7 +23,7 @@ export function clampScore(value: number): number { export const SCORE_GREEN_FLOOR = 70; /** Below this is red. */ -export const SCORE_YELLOW_FLOOR = 50; +const SCORE_YELLOW_FLOOR = 50; /** * The band a 0-100 score falls in, from the number alone. diff --git a/src/lib/jobs/__tests__/document-content-index-backfill.test.ts b/src/lib/jobs/__tests__/document-content-index-backfill.test.ts index b0817c6ff..33049dd2e 100644 --- a/src/lib/jobs/__tests__/document-content-index-backfill.test.ts +++ b/src/lib/jobs/__tests__/document-content-index-backfill.test.ts @@ -3,13 +3,19 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; /** * Content-index backfill (Document vault P2). Pins: no provider / no consent → * clean no-op; indexes not-yet-indexed docs; stops when the budget is reached - * (resumable); skips are bounded by the MIME-filtered candidate set. + * (resumable); skips are bounded by the MIME-filtered candidate set; rows on a + * stale `tokenizerVersion` are re-tokenised locally (the column's consumer) + * while current-version rows are left alone. */ vi.mock("@/lib/db", () => ({ prisma: { inboundDocument: { findMany: vi.fn(), findFirst: vi.fn() }, - documentContentIndex: { upsert: vi.fn() }, + documentContentIndex: { + upsert: vi.fn(), + findMany: vi.fn(), + update: vi.fn(), + }, }, })); vi.mock("@/lib/documents/store", () => ({ @@ -27,6 +33,9 @@ vi.mock("@/lib/documents/describe", () => ({ })); vi.mock("@/lib/documents/content-index", () => ({ upsertContentIndex: vi.fn().mockResolvedValue({ tokenCount: 3 }), + CONTENT_TOKENIZER_VERSION: "1", + decryptIndexText: vi.fn(() => "decrypted index text"), + tokeniseAndHash: vi.fn(() => ["hash-a", "hash-b"]), })); vi.mock("@/lib/documents/provider-order", () => ({ resolveDocumentVisionProvider: vi.fn(), @@ -54,7 +63,10 @@ import { ConsentRequiredError, } from "@/lib/ai/consent-guard"; import { reserveBudget } from "@/lib/ai/coach/budget"; -import { upsertContentIndex } from "@/lib/documents/content-index"; +import { + decryptIndexText, + upsertContentIndex, +} from "@/lib/documents/content-index"; const PICK = { chain: [{ providerType: "anthropic", instance: {} }], @@ -83,8 +95,41 @@ beforeEach(() => { vi.mocked(prisma.inboundDocument.findFirst).mockImplementation(((args: { where: { id: string }; }) => Promise.resolve(doc(args.where.id))) as never); + // Default: no stale-tokenizer rows — the local re-tokenise sweep no-ops. + vi.mocked(prisma.documentContentIndex.findMany).mockResolvedValue( + [] as never, + ); }); +/** + * Seed the stale-row sweep with index rows and make the findMany mock APPLY + * the query's own `tokenizerVersion: { not: … }` filter, so a sweep that + * forgot the filter would receive (and wrongly update) the current-version + * row — the watched-red arm of both tests below. + */ +function seedIndexRows( + rows: { id: string; tokenizerVersion: string; textEncrypted: Uint8Array }[], +) { + vi.mocked(prisma.documentContentIndex.findMany).mockImplementation(((args: { + where?: { + tokenizerVersion?: { not?: string }; + id?: { gt?: string }; + }; + take?: number; + }) => { + const not = args?.where?.tokenizerVersion?.not; + const after = args?.where?.id?.gt; + const matched = rows + .filter((row) => + not === undefined ? true : row.tokenizerVersion !== not, + ) + .filter((row) => (after === undefined ? true : row.id > after)) + .slice(0, args?.take ?? rows.length) + .map(({ id, textEncrypted }) => ({ id, textEncrypted })); + return Promise.resolve(matched); + }) as never); +} + describe("runContentIndexBackfillForUser", () => { it("no-ops with no provider", async () => { vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ @@ -94,6 +139,7 @@ describe("runContentIndexBackfillForUser", () => { const result = await runContentIndexBackfillForUser("user-1"); expect(result).toEqual({ indexed: 0, + retokenised: 0, skipped: 0, failed: 0, reason: "no-provider", @@ -109,6 +155,7 @@ describe("runContentIndexBackfillForUser", () => { const result = await runContentIndexBackfillForUser("user-1"); expect(result).toEqual({ indexed: 0, + retokenised: 0, skipped: 0, failed: 0, reason: "no-consent", @@ -128,6 +175,7 @@ describe("runContentIndexBackfillForUser", () => { const result = await runContentIndexBackfillForUser("user-1"); expect(result).toEqual({ indexed: 2, + retokenised: 0, skipped: 0, failed: 0, reason: "ok", @@ -174,6 +222,7 @@ describe("runContentIndexBackfillForUser", () => { ); expect(result).toEqual({ indexed: 1, + retokenised: 0, skipped: 0, failed: 0, reason: "ok", @@ -203,6 +252,7 @@ describe("runContentIndexBackfillForUser", () => { expect(transcribeDocument).not.toHaveBeenCalled(); expect(result).toEqual({ indexed: 0, + retokenised: 0, skipped: 1, failed: 0, reason: "ok", @@ -225,10 +275,120 @@ describe("runContentIndexBackfillForUser", () => { const result = await runContentIndexBackfillForUser("user-1"); expect(result).toEqual({ indexed: 1, + retokenised: 0, skipped: 0, failed: 0, reason: "budget-reached", }); expect(upsertContentIndex).toHaveBeenCalledTimes(1); }); + + it("re-tokenises a row on a stale tokenizerVersion and leaves the current-version row alone", async () => { + // No provider at all: the sweep is local work and must run anyway. + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + seedIndexRows([ + { + id: "idx-stale", + tokenizerVersion: "0", + textEncrypted: new Uint8Array([1]), + }, + { + id: "idx-current", + tokenizerVersion: "1", + textEncrypted: new Uint8Array([2]), + }, + ]); + + const result = await runContentIndexBackfillForUser("user-1"); + + // The sweep's query names the version filter (the column's consumer)… + expect(prisma.documentContentIndex.findMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + userId: "user-1", + tokenizerVersion: { not: "1" }, + }), + }), + ); + // …selects ONLY the stale row, and stamps it onto the current version. + expect(prisma.documentContentIndex.update).toHaveBeenCalledTimes(1); + expect(prisma.documentContentIndex.update).toHaveBeenCalledWith({ + where: { id: "idx-stale" }, + data: { searchTokens: ["hash-a", "hash-b"], tokenizerVersion: "1" }, + }); + expect(result).toEqual({ + indexed: 0, + retokenised: 1, + skipped: 0, + failed: 0, + reason: "no-provider", + }); + }); + + it("touches nothing when every row already carries the current tokenizerVersion", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + seedIndexRows([ + { + id: "idx-current", + tokenizerVersion: "1", + textEncrypted: new Uint8Array([2]), + }, + ]); + + const result = await runContentIndexBackfillForUser("user-1"); + + expect(prisma.documentContentIndex.update).not.toHaveBeenCalled(); + expect(result).toEqual({ + indexed: 0, + retokenised: 0, + skipped: 0, + failed: 0, + reason: "no-provider", + }); + }); + + it("counts an undecryptable stale row as failed and keeps walking", async () => { + vi.mocked(resolveDocumentVisionProvider).mockResolvedValue({ + chain: [], + pick: null, + } as never); + seedIndexRows([ + { + id: "idx-bad", + tokenizerVersion: "0", + textEncrypted: new Uint8Array([9]), + }, + { + id: "idx-stale", + tokenizerVersion: "0", + textEncrypted: new Uint8Array([1]), + }, + ]); + vi.mocked(decryptIndexText) + .mockImplementationOnce(() => { + throw new Error("bad key"); + }) + .mockImplementation(() => "decrypted index text"); + + const result = await runContentIndexBackfillForUser("user-1"); + + expect(prisma.documentContentIndex.update).toHaveBeenCalledTimes(1); + expect(prisma.documentContentIndex.update).toHaveBeenCalledWith({ + where: { id: "idx-stale" }, + data: { searchTokens: ["hash-a", "hash-b"], tokenizerVersion: "1" }, + }); + expect(result).toEqual({ + indexed: 0, + retokenised: 1, + skipped: 0, + failed: 1, + reason: "no-provider", + }); + }); }); diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index 8ddd81594..d2b8a8051 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -3,7 +3,11 @@ * * Indexes a user's EXISTING stored documents for content search: for each live * document that has no `DocumentContentIndex` yet, run one provider vision - * transcription and upsert the encrypted-text + blind-token index. On-demand + * transcription and upsert the encrypted-text + blind-token index. The run + * also sweeps rows whose stored `tokenizerVersion` trails + * `CONTENT_TOKENIZER_VERSION` and re-tokenises them locally from the + * decrypted text (see `retokeniseStaleRows`) — the consumer that makes a + * tokeniser bump actually re-index the corpus. On-demand * only (fired by the "index all documents" action) — NOT a boot/cron pass: * indexing egresses to a provider under the user's key + budget, so it must be * a deliberate, consented act, exactly like the per-document index route. @@ -41,7 +45,12 @@ import { loadOwnedDocument, prepareVisionInput, } from "@/lib/documents/ai-route-support"; -import { upsertContentIndex } from "@/lib/documents/content-index"; +import { + CONTENT_TOKENIZER_VERSION, + decryptIndexText, + tokeniseAndHash, + upsertContentIndex, +} from "@/lib/documents/content-index"; import { transcribeDocument } from "@/lib/documents/describe"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; @@ -68,17 +77,79 @@ export interface ContentIndexBackfillPayload { export interface ContentIndexBackfillSummary { indexed: number; + /** + * Rows whose stored `tokenizerVersion` trailed `CONTENT_TOKENIZER_VERSION` + * and were re-tokenised locally from the decrypted `textEncrypted` — no + * provider call, no consent, no budget involved. + */ + retokenised: number; /** * Documents visited but not indexable this run (gone by load time, * undecryptable, or a PDF no rasteriser could render). Refunded, never * billed, and left un-indexed for a later run. */ skipped: number; - /** Documents whose provider call failed. Refunded; a later run retries. */ + /** + * Documents whose provider call failed, plus stale-tokenizer rows whose + * local re-tokenise failed (undecryptable text). Refunded where a + * reservation existed; a later run retries. + */ failed: number; reason: "ok" | "no-provider" | "no-consent" | "budget-reached"; } +/** + * The consumer of `DocumentContentIndex.tokenizerVersion`: sweep this user's + * rows written under an older tokeniser and re-tokenise them from the + * decrypted `textEncrypted` — the same local path the key-rotation script + * takes (`scripts/rotate-encryption-key.ts`), so bumping + * `CONTENT_TOKENIZER_VERSION` actually re-indexes the corpus instead of + * stranding every stored row on the old rules. Purely local (decrypt + HMAC), + * so it runs before — and regardless of — the provider/consent/budget gates. + * Keyset pagination on `id` because a re-tokenised row drops out of the + * filtered set mid-walk, which would break cursor-based paging. + */ +async function retokeniseStaleRows( + userId: string, +): Promise<{ retokenised: number; failed: number }> { + let retokenised = 0; + let failed = 0; + let afterId: string | null = null; + for (;;) { + const rows: { id: string; textEncrypted: Uint8Array }[] = + await prisma.documentContentIndex.findMany({ + where: { + userId, + tokenizerVersion: { not: CONTENT_TOKENIZER_VERSION }, + ...(afterId ? { id: { gt: afterId } } : {}), + }, + select: { id: true, textEncrypted: true }, + orderBy: { id: "asc" }, + take: PAGE_SIZE, + }); + if (rows.length === 0) break; + for (const row of rows) { + try { + const plaintext = decryptIndexText(row.textEncrypted); + const searchTokens = tokeniseAndHash(plaintext); + await prisma.documentContentIndex.update({ + where: { id: row.id }, + data: { searchTokens, tokenizerVersion: CONTENT_TOKENIZER_VERSION }, + }); + retokenised += 1; + } catch { + // Fail-closed decrypt or a write miss — leave the row on its old + // version (it stays in the candidate set for the next run) and + // keep walking; one bad row must not abort the sweep. + failed += 1; + } + } + afterId = rows[rows.length - 1]!.id; + if (rows.length < PAGE_SIZE) break; + } + return { retokenised, failed }; +} + /** * Index one user's not-yet-indexed documents. Idempotent + resumable: an * already-indexed document drops out of the candidate set, and the run stops at @@ -87,9 +158,20 @@ export interface ContentIndexBackfillSummary { export async function runContentIndexBackfillForUser( userId: string, ): Promise { + // Stale-tokeniser rows first: local work that owes nothing to the + // provider, consent, or budget gates below, so a user without a usable + // provider still gets their existing index moved to the current rules. + const stale = await retokeniseStaleRows(userId); + const { pick } = await resolveDocumentVisionProvider(userId); if (!pick) - return { indexed: 0, skipped: 0, failed: 0, reason: "no-provider" }; + return { + indexed: 0, + retokenised: stale.retokenised, + skipped: 0, + failed: stale.failed, + reason: "no-provider", + }; try { await assertDocumentEgressConsent({ userId, @@ -98,7 +180,13 @@ export async function runContentIndexBackfillForUser( }); } catch (err) { if (err instanceof ConsentRequiredError) { - return { indexed: 0, skipped: 0, failed: 0, reason: "no-consent" }; + return { + indexed: 0, + retokenised: stale.retokenised, + skipped: 0, + failed: stale.failed, + reason: "no-consent", + }; } throw err; } @@ -114,7 +202,7 @@ export async function runContentIndexBackfillForUser( const dailyCap = resolveDailyCap([{ providerType: pick.entry.providerType }]); let indexed = 0; let skipped = 0; - let failed = 0; + let failed = stale.failed; let reason: ContentIndexBackfillSummary["reason"] = "ok"; let cursor: string | null = null; @@ -201,9 +289,9 @@ export async function runContentIndexBackfillForUser( annotate({ action: { name: "documents.contentIndex.backfill" }, - meta: { indexed, skipped, failed, reason }, + meta: { indexed, retokenised: stale.retokenised, skipped, failed, reason }, }); - return { indexed, skipped, failed, reason }; + return { indexed, retokenised: stale.retokenised, skipped, failed, reason }; } /** diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 2f29ccc4c..b3faf909d 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -1171,6 +1171,7 @@ export async function registerMaintenanceQueues( let users = 0; let skipped = 0; let documentsIndexed = 0; + let documentsRetokenised = 0; let documentsSkipped = 0; let documentsFailed = 0; for (const job of jobs) { @@ -1182,17 +1183,19 @@ export async function registerMaintenanceQueues( try { const { indexed, + retokenised, skipped: docsSkipped, failed: docsFailed, reason, } = await runContentIndexBackfillForUser(userId); users++; documentsIndexed += indexed; + documentsRetokenised += retokenised; documentsSkipped += docsSkipped; documentsFailed += docsFailed; workerLog( "info", - `[document-content-index-backfill] user=${userId} indexed=${indexed} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`, + `[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`, ); } catch (err) { recordError(); @@ -1209,6 +1212,7 @@ export async function registerMaintenanceQueues( users, skipped, documents_indexed: documentsIndexed, + documents_retokenised: documentsRetokenised, documents_skipped: documentsSkipped, documents_failed: documentsFailed, }); diff --git a/src/lib/rollups/tiered-context.ts b/src/lib/rollups/tiered-context.ts index ac798e546..1546f3460 100644 --- a/src/lib/rollups/tiered-context.ts +++ b/src/lib/rollups/tiered-context.ts @@ -43,7 +43,7 @@ import { probeRollupCoverage } from "./measurement-coverage"; const DAY_MS = 24 * 60 * 60 * 1000; /** Band cut points in days-ago. */ -export const TIERED_BANDS = { +const TIERED_BANDS = { rawDays: 14, dayUntil: 30, weekUntil: 90, diff --git a/src/lib/targets/consistency.ts b/src/lib/targets/consistency.ts index fcb634e36..66de155d2 100644 --- a/src/lib/targets/consistency.ts +++ b/src/lib/targets/consistency.ts @@ -4,7 +4,7 @@ import type { DayBand, TargetConsistency } from "./types"; const DAY_MS = 24 * 60 * 60 * 1000; const EMPTY_STRIP = [null, null, null, null, null, null, null] as const; -export const EMPTY_CONSISTENCY: TargetConsistency = { +const EMPTY_CONSISTENCY: TargetConsistency = { daysInRange7d: 0, daysLogged7d: 0, daysInRange30d: 0,