From 0278e22ed116bf0dc28906aca3bedc6f46831956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 19 Jun 2026 20:54:56 +0200 Subject: [PATCH] feat(jobs): add retention prune for stored Coach history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit coach_messages is append-only and grows forever — a daily Coach user accrues encrypted conversation rows with no lifecycle, while every sibling ledger (push_attempts, audit_logs, host_metrics) already prunes. Add a daily pg-boss cleanup that hard-deletes rows past a configurable window (COACH_MESSAGE_RETENTION_DAYS, default 365, 30-day floor so a misconfig cannot drop live in-context history). Wire it into the maintenance registrar with the dead-queue contract guard and a focused helper test. --- .../coach-message-cleanup-queue.test.ts | 56 ++++++++++++ .../__tests__/coach-message-cleanup.test.ts | 90 +++++++++++++++++++ src/lib/jobs/coach-message-cleanup.ts | 50 +++++++++++ src/lib/jobs/reminder/cleanup-handlers.ts | 20 +++++ src/lib/jobs/reminder/register-maintenance.ts | 26 ++++++ 5 files changed, 242 insertions(+) create mode 100644 src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts create mode 100644 src/lib/jobs/__tests__/coach-message-cleanup.test.ts create mode 100644 src/lib/jobs/coach-message-cleanup.ts diff --git a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts new file mode 100644 index 000000000..42ba476e2 --- /dev/null +++ b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts @@ -0,0 +1,56 @@ +/** + * v1.18.7 — coach-message-cleanup queue registration guard. + * + * Same source-text-grep approach as the other queue-wiring guards: assert the + * queue is registered in `allQueues`, scheduled, and wired to a `boss.work` + * handler — without booting pg-boss + Prisma. An unregistered queue silently + * never drains (the recurring v1.4.37 dead-queue bug), which would leave the + * encrypted Coach history growing unbounded with the retention job dark. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REGISTRAR_PATH = join( + __dirname, + "..", + "reminder", + "register-maintenance.ts", +); +const workerSource = + readFileSync(REGISTRAR_PATH, "utf8") + + readFileSync(join(__dirname, "..", "reminder", "cleanup-handlers.ts"), "utf8"); + +describe("reminder-worker — coach-message-cleanup wiring", () => { + it("imports the cleanup handler from the cleanup-handlers module", () => { + expect(workerSource).toMatch(/\bhandleCoachMessageCleanup\b/); + expect(workerSource).toMatch(/\bCoachMessageCleanupPayload\b/); + }); + + it("registers the coach-message-cleanup queue in the allQueues loop", () => { + const allQueuesMatch = workerSource.match( + /const allQueues\s*=\s*\[([\s\S]*?)\];/, + ); + expect(allQueuesMatch).not.toBeNull(); + expect(allQueuesMatch![1]).toMatch(/\bCOACH_MESSAGE_CLEANUP_QUEUE\b/); + }); + + it("schedules the coach-message-cleanup cron", () => { + expect(workerSource).toMatch( + /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON\]/, + ); + }); + + it("registers a boss.work handler for the queue", () => { + expect(workerSource).toMatch( + /boss\.work[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, + ); + }); + + it("prunes coach messages inside the handler", () => { + expect(workerSource).toMatch( + /handleCoachMessageCleanup[\s\S]{0,400}cleanupOldCoachMessages/, + ); + }); +}); diff --git a/src/lib/jobs/__tests__/coach-message-cleanup.test.ts b/src/lib/jobs/__tests__/coach-message-cleanup.test.ts new file mode 100644 index 000000000..dfe02e057 --- /dev/null +++ b/src/lib/jobs/__tests__/coach-message-cleanup.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { PrismaClient } from "@/generated/prisma/client"; +import { + cleanupOldCoachMessages, + getCoachMessageRetentionDays, + DEFAULT_COACH_MESSAGE_RETENTION_DAYS, +} from "../coach-message-cleanup"; + +function makePrismaMock(deletedCount: number) { + return { + coachMessage: { + deleteMany: vi.fn().mockResolvedValue({ count: deletedCount }), + }, + } as unknown as PrismaClient; +} + +const ORIGINAL_ENV = { ...process.env }; + +beforeEach(() => { + process.env = { ...ORIGINAL_ENV }; + delete process.env.COACH_MESSAGE_RETENTION_DAYS; +}); + +afterEach(() => { + process.env = ORIGINAL_ENV; + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe("getCoachMessageRetentionDays", () => { + it("returns 365 when env is unset", () => { + expect(getCoachMessageRetentionDays()).toBe( + DEFAULT_COACH_MESSAGE_RETENTION_DAYS, + ); + }); + + it("respects a valid override", () => { + process.env.COACH_MESSAGE_RETENTION_DAYS = "180"; + expect(getCoachMessageRetentionDays()).toBe(180); + }); + + it("ignores nonsensical values (NaN, 0, negative)", () => { + process.env.COACH_MESSAGE_RETENTION_DAYS = "0"; + expect(getCoachMessageRetentionDays()).toBe( + DEFAULT_COACH_MESSAGE_RETENTION_DAYS, + ); + process.env.COACH_MESSAGE_RETENTION_DAYS = "-100"; + expect(getCoachMessageRetentionDays()).toBe( + DEFAULT_COACH_MESSAGE_RETENTION_DAYS, + ); + process.env.COACH_MESSAGE_RETENTION_DAYS = "not a number"; + expect(getCoachMessageRetentionDays()).toBe( + DEFAULT_COACH_MESSAGE_RETENTION_DAYS, + ); + }); + + it("rejects too-short retention (< 30 days) to protect live history", () => { + process.env.COACH_MESSAGE_RETENTION_DAYS = "7"; + expect(getCoachMessageRetentionDays()).toBe( + DEFAULT_COACH_MESSAGE_RETENTION_DAYS, + ); + }); +}); + +describe("cleanupOldCoachMessages", () => { + it("deletes coach messages older than 365 days by default", async () => { + const prisma = makePrismaMock(42); + const now = new Date("2026-05-04T00:00:00Z"); + const deleted = await cleanupOldCoachMessages(prisma, now); + + expect(deleted).toBe(42); + expect(prisma.coachMessage.deleteMany).toHaveBeenCalledTimes(1); + const cutoff = new Date(now.getTime() - 365 * 86_400_000); + expect(prisma.coachMessage.deleteMany).toHaveBeenCalledWith({ + where: { createdAt: { lt: cutoff } }, + }); + }); + + it("uses the configured retention", async () => { + process.env.COACH_MESSAGE_RETENTION_DAYS = "90"; + const prisma = makePrismaMock(7); + const now = new Date("2026-05-04T00:00:00Z"); + await cleanupOldCoachMessages(prisma, now); + + const cutoff = new Date(now.getTime() - 90 * 86_400_000); + expect(prisma.coachMessage.deleteMany).toHaveBeenCalledWith({ + where: { createdAt: { lt: cutoff } }, + }); + }); +}); diff --git a/src/lib/jobs/coach-message-cleanup.ts b/src/lib/jobs/coach-message-cleanup.ts new file mode 100644 index 000000000..78768d9c1 --- /dev/null +++ b/src/lib/jobs/coach-message-cleanup.ts @@ -0,0 +1,50 @@ +/** + * Daily cleanup for the `coach_messages` table. + * + * v1.18.7 retention (GDPR Art. 5(1)(e) "storage limitation"): the Coach + * persists every turn's prose as an encrypted `encryptedContent` Bytes + * column. The table is append-only and grows forever — a daily Coach user + * accrues rows indefinitely with no lifecycle. The conversation history is + * useful context, but stale beyond a year it is dead weight inflating the + * table and the `(conversation_id, created_at)` indexes. + * + * Default retention is 365 days (configurable via COACH_MESSAGE_RETENTION_DAYS + * env). Rows older than the cutoff are deleted in a single bulk `deleteMany`; + * runs daily via pg-boss. The DELETE is keyed on `created_at` directly — the + * pre-existing `(conversation_id, created_at)` index is not a perfect cover + * for an unqualified `created_at` scan, but the trailing-edge delete touches + * only the oldest sliver of the table. + * + * The retention floor is 30 days: a misconfigured tiny value never nukes + * recent, in-context conversation history. Content stays encrypted at rest + * until the moment it is hard-deleted; there is no plaintext path. + */ +import type { PrismaClient } from "@/generated/prisma/client"; + +export const DEFAULT_COACH_MESSAGE_RETENTION_DAYS = 365; + +export function getCoachMessageRetentionDays(): number { + const raw = process.env.COACH_MESSAGE_RETENTION_DAYS; + if (raw === undefined) return DEFAULT_COACH_MESSAGE_RETENTION_DAYS; + const parsed = parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return DEFAULT_COACH_MESSAGE_RETENTION_DAYS; + } + // Refuse a too-short retention window. Coach history is in-context for the + // proactive nudge frequency gate and the conversation rail; 30 days is the + // safe floor so a misconfig measured in seconds cannot prune live history. + if (parsed < 30) return DEFAULT_COACH_MESSAGE_RETENTION_DAYS; + return parsed; +} + +export async function cleanupOldCoachMessages( + prisma: PrismaClient, + now: Date = new Date(), +): Promise { + const days = getCoachMessageRetentionDays(); + const cutoff = new Date(now.getTime() - days * 86_400_000); + const { count } = await prisma.coachMessage.deleteMany({ + where: { createdAt: { lt: cutoff } }, + }); + return count; +} diff --git a/src/lib/jobs/reminder/cleanup-handlers.ts b/src/lib/jobs/reminder/cleanup-handlers.ts index 8ae6c8415..9ae4568b3 100644 --- a/src/lib/jobs/reminder/cleanup-handlers.ts +++ b/src/lib/jobs/reminder/cleanup-handlers.ts @@ -12,6 +12,7 @@ import { } from "@/lib/jobs/whoop-oauth-state-cleanup"; import { cleanupExpiredIdempotencyKeys } from "@/lib/jobs/idempotency-cleanup"; import { cleanupOldAuditLogs } from "@/lib/jobs/audit-log-cleanup"; +import { cleanupOldCoachMessages } from "@/lib/jobs/coach-message-cleanup"; import { cleanupExpiredWithingsOAuthStates } from "@/lib/jobs/withings-oauth-state-cleanup"; import { cleanupExpiredMeasurementTombstones, @@ -49,6 +50,10 @@ export interface AuditLogCleanupPayload { triggeredAt: string; } +export interface CoachMessageCleanupPayload { + triggeredAt: string; +} + export interface WithingsOAuthStateCleanupPayload { triggeredAt: string; } @@ -201,6 +206,21 @@ export async function handleAuditLogCleanup( }); } +export async function handleCoachMessageCleanup( + jobs: Job[], +) { + void jobs; + await withBackgroundEvent("job.coach_message_cleanup", async (evt) => { + const p = getWorkerPrisma(); + try { + const deleted = await cleanupOldCoachMessages(p); + evt.addMeta("coach_message_cleanup_deleted", deleted); + } catch (err) { + evt.addWarning(`coach-message-cleanup failed: ${err}`); + } + }); +} + export async function handleWithingsOAuthStateCleanup( jobs: Job[], ) { diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index e601c0f67..e003d1d2a 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -77,6 +77,8 @@ import { handleRateLimitCleanup, handleIdempotencyCleanup, handleAuditLogCleanup, + CoachMessageCleanupPayload, + handleCoachMessageCleanup, } from "./cleanup-handlers"; import { HostMetricSamplePayload, @@ -156,6 +158,16 @@ const PUSH_ATTEMPT_CLEANUP_CRON = "35 3 * * *"; const MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE = "measurement-tombstone-cleanup"; const MEASUREMENT_TOMBSTONE_CLEANUP_CRON = "40 3 * * *"; +// v1.18.7 — daily prune for the encrypted Coach conversation history. +// coach_messages is append-only and grows forever; rows older than the +// configurable retention window (default 365 days) are hard-deleted. +// Slots at 03:50 Europe/Berlin after the measurement-tombstone cleanup +// (03:40) and the cumulative drain (03:45), staying inside the existing +// 03:xx maintenance window. + +const COACH_MESSAGE_CLEANUP_QUEUE = "coach-message-cleanup"; + +const COACH_MESSAGE_CLEANUP_CRON = "50 3 * * *"; const allQueues = [ DATA_BACKUP_QUEUE, @@ -193,6 +205,10 @@ const allQueues = [ // the daily schedule silently no-ops and pruned-past-retention tombstones // pile up forever. MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, + // v1.18.7 — Coach conversation-history retention prune. Without this entry + // the daily schedule silently no-ops and the encrypted coach_messages + // table grows unbounded. + COACH_MESSAGE_CLEANUP_QUEUE, ]; const schedules: ScheduleEntry[] = [ @@ -242,6 +258,8 @@ const schedules: ScheduleEntry[] = [ // v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement // tombstones. [MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON], + // v1.18.7 — daily 03:50 Europe/Berlin prune for stale Coach history. + [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON], ]; /** @@ -323,6 +341,14 @@ export async function registerMaintenanceQueues( { localConcurrency: 1 }, handleMeasurementTombstoneCleanup, ); + // v1.18.7 — daily prune of the encrypted Coach conversation history. + // Single-flight like every other cleanup queue; two ticks racing on the + // same DELETE is wasted work and the second is a no-op. + await boss.work( + COACH_MESSAGE_CLEANUP_QUEUE, + { localConcurrency: 1 }, + handleCoachMessageCleanup, + ); await boss.work( PR_DETECTION_QUEUE, { localConcurrency: PR_DETECTION_CONCURRENCY },