mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 13:52:20 +00:00
Add a reminder-satisfy pg-boss queue (mirrors pr-detection) that every measurement and lab ingest path enqueues — manual create, batch, and the Withings / Polar / Oura / Nightscout / Fitbit / WHOOP syncs. A reminder now auto-resolves the moment a matching reading lands instead of waiting up to 15 minutes. The 15-minute cron stays as the idempotent safety net; the forward-only guard lets the eventful hook and the cron converge without double-stamping. The worker honours the module toggle (a reminder for a disabled module produces no engine activity) and runs the same satisfaction primitives the cron uses.
706 lines
27 KiB
TypeScript
706 lines
27 KiB
TypeScript
/**
|
|
* Fitbit / Google Health sync service (v1.12.0).
|
|
*
|
|
* This wave (OAuth + credentials) lands the token-management half:
|
|
* - `getValidToken` decrypts the stored token, refreshes at
|
|
* `tokenExpiresAt - 5 min`, persists the new access token + expiry, and —
|
|
* UNLIKE WHOOP — only overwrites the stored refresh token when the response
|
|
* carries a fresh one (Google does not rotate refresh tokens).
|
|
* - `recordFitbitSyncFailure` / `classificationToFailureKind` map a
|
|
* classified API error onto the shared integration-status ledger.
|
|
*
|
|
* The per-resource data sync (`upsertFitbitMeasurements`, `syncUserFitbit`, the
|
|
* collection walkers, and the 403 soft-skip orchestration) lands in a later
|
|
* wave and extends this file.
|
|
*/
|
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
import pLimit from "p-limit";
|
|
import { prisma } from "@/lib/db";
|
|
import type { MeasurementType } from "@/generated/prisma/client";
|
|
import { encrypt, decrypt } from "@/lib/crypto";
|
|
import { annotate, getEvent } from "@/lib/logging/context";
|
|
import {
|
|
isReauthRequired,
|
|
recordSyncFailure,
|
|
recordSyncSuccess,
|
|
type FailureKind,
|
|
} from "@/lib/integrations/status";
|
|
import {
|
|
collapseToTypeDayKeys,
|
|
recomputeBucketsForMeasurement,
|
|
recomputeUserRollups,
|
|
} from "@/lib/rollups/measurement-rollups";
|
|
import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate";
|
|
import { refreshAccessToken } from "./client";
|
|
import { getUserFitbitCredentials } from "./credentials";
|
|
import {
|
|
FitbitApiError,
|
|
classifyFitbitError,
|
|
type FitbitClassification,
|
|
} from "./response-classifier";
|
|
|
|
/** Refresh the access token this many ms before `tokenExpiresAt`. */
|
|
const TOKEN_REFRESH_BUFFER_MS = 5 * 60 * 1000;
|
|
|
|
/**
|
|
* Overlap window for the incremental sync, in ms. Google re-rolls daily
|
|
* summaries after the fact (a daily SpO2 / HRV / RHR can finalise hours after
|
|
* the night), so the default overlap is a full 24 h to make sure the re-rolled
|
|
* row is re-fetched on the next tick. Mirrors WHOOP's recovery/sleep overlap.
|
|
*/
|
|
export const FITBIT_DEFAULT_OVERLAP_MS = 24 * 60 * 60 * 1000;
|
|
|
|
export interface FitbitTokenInfo {
|
|
accessToken: string;
|
|
connection: { id: string; fitbitUserId: string };
|
|
}
|
|
|
|
/**
|
|
* Resolve a valid Fitbit access token for a user, refreshing if it is within
|
|
* the 5-minute expiry buffer. Returns null when there is no connection, no
|
|
* credentials, or the refresh fails (the failure is recorded so scheduled syncs
|
|
* back off).
|
|
*
|
|
* KEY DELTA vs WHOOP: Google does not rotate refresh tokens. On refresh, persist
|
|
* the new access token + expiry and overwrite the stored `refreshToken` ONLY
|
|
* when the response carries a fresh one — otherwise keep the existing refresh
|
|
* token so the next refresh still authenticates.
|
|
*/
|
|
export async function getValidToken(
|
|
userId: string,
|
|
): Promise<FitbitTokenInfo | null> {
|
|
const connection = await prisma.fitbitConnection.findUnique({
|
|
where: { userId },
|
|
});
|
|
if (!connection) return null;
|
|
|
|
const accessToken = decrypt(connection.accessToken);
|
|
const refreshToken = decrypt(connection.refreshToken);
|
|
|
|
if (
|
|
connection.tokenExpiresAt.getTime() - TOKEN_REFRESH_BUFFER_MS <
|
|
Date.now()
|
|
) {
|
|
try {
|
|
const creds = await getUserFitbitCredentials(userId);
|
|
if (!creds) {
|
|
getEvent()?.addWarning(
|
|
`No Fitbit credentials found for user ${userId} during token refresh`,
|
|
);
|
|
await recordSyncFailure({
|
|
userId,
|
|
integration: "fitbit",
|
|
kind: "reauth_required",
|
|
message: "Fitbit credentials missing — token refresh skipped",
|
|
errorCode: "credentials_missing",
|
|
});
|
|
return null;
|
|
}
|
|
|
|
const newTokens = await refreshAccessToken(refreshToken, creds);
|
|
const expiresAt = new Date(Date.now() + newTokens.expires_in * 1000);
|
|
|
|
// Google does NOT rotate refresh tokens: a routine refresh returns a new
|
|
// access token but usually omits `refresh_token`. Persist the new access
|
|
// token + expiry and only overwrite the stored refresh token when the
|
|
// response actually carries a fresh one — otherwise keep the existing one.
|
|
await prisma.fitbitConnection.update({
|
|
where: { id: connection.id },
|
|
data: {
|
|
accessToken: encrypt(newTokens.access_token),
|
|
tokenExpiresAt: expiresAt,
|
|
...(newTokens.refresh_token
|
|
? { refreshToken: encrypt(newTokens.refresh_token) }
|
|
: {}),
|
|
},
|
|
});
|
|
|
|
return {
|
|
accessToken: newTokens.access_token,
|
|
connection: {
|
|
id: connection.id,
|
|
fitbitUserId: connection.fitbitUserId,
|
|
},
|
|
};
|
|
} catch (err) {
|
|
getEvent()?.addWarning(
|
|
`Fitbit token refresh failed for user ${userId}: ${err}`,
|
|
);
|
|
await recordFitbitSyncFailure(userId, err);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
return {
|
|
accessToken,
|
|
connection: {
|
|
id: connection.id,
|
|
fitbitUserId: connection.fitbitUserId,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* True when a caught error is a per-data-class 403 (forbidden). The six Google
|
|
* Health Restricted bundles are granted independently in the consent flow, so a
|
|
* 403 on ONE data class is a scope gate on THAT class — soft-skip it and keep
|
|
* the connection connected rather than parking the whole integration at
|
|
* `error_reauth`. Connection-wide reauth is reserved for a 401 (token rejected)
|
|
* and for a 403 on the token-refresh / profile path (a genuine grant revoke),
|
|
* which run outside the per-resource catch blocks. Matters more here than for
|
|
* WHOOP because partial grants are likely with six independent bundles.
|
|
*/
|
|
export function isCollectionForbidden(err: unknown): boolean {
|
|
return err instanceof FitbitApiError && err.httpStatus === 403;
|
|
}
|
|
|
|
/**
|
|
* Per-`syncUserFitbit`-cycle counter for collection-403 soft-skips. Scoped by
|
|
* AsyncLocalStorage so the count never bleeds across concurrent per-user
|
|
* pg-boss jobs. Mirrors the WHOOP `softSkipStorage` tracker.
|
|
*/
|
|
interface SoftSkipTracker {
|
|
count: number;
|
|
}
|
|
const softSkipStorage = new AsyncLocalStorage<SoftSkipTracker>();
|
|
|
|
/**
|
|
* Per-`syncUserFitbit`-cycle accumulator of the `(type, day)` keys every
|
|
* resource's writes touched. Only populated on a `fullSync` backfill (when the
|
|
* inline per-day rollup hook is deferred); the orchestrator drains it into ONE
|
|
* `recomputeUserRollups(from, to)` pass at the end of the cycle. Scoped by
|
|
* AsyncLocalStorage so concurrent per-user jobs never cross-pollinate, mirroring
|
|
* the soft-skip tracker.
|
|
*/
|
|
interface RollupDeferTracker {
|
|
keys: Array<{ type: MeasurementType; measuredAt: Date }>;
|
|
}
|
|
const rollupDeferStorage = new AsyncLocalStorage<RollupDeferTracker>();
|
|
|
|
/**
|
|
* Single-source the per-resource collection-fetch error handling. A 403 on one
|
|
* data class soft-skips it (warn + return 0) so sibling resources still sync;
|
|
* anything else records a classified sync failure and rethrows. A soft-skip
|
|
* increments the ambient tracker so `syncUserFitbit` can refuse to stamp success
|
|
* on an all-403 grant-revoke cycle that imported nothing.
|
|
*/
|
|
export async function handleCollectionFetchError(
|
|
resource: string,
|
|
userId: string,
|
|
err: unknown,
|
|
): Promise<number> {
|
|
if (isCollectionForbidden(err)) {
|
|
getEvent()?.addWarning(
|
|
`fitbit ${resource} sync skipped for ${userId}: collection 403 (soft-skip)`,
|
|
);
|
|
const tracker = softSkipStorage.getStore();
|
|
if (tracker) tracker.count += 1;
|
|
return 0;
|
|
}
|
|
await recordFitbitSyncFailure(userId, err);
|
|
throw err;
|
|
}
|
|
|
|
/**
|
|
* Options threaded from `syncUserFitbit` to each per-resource sync. The `start`
|
|
* watermark is snapshotted ONCE by the orchestrator (a single `lastSyncedAt`
|
|
* read) so every resource fetches from the same lower bound; no resource re-reads
|
|
* `lastSyncedAt` or stamps `markSynced` itself. On a full/backfill run `start`
|
|
* is undefined (no lower bound). The orchestrator owns the single end-of-cycle
|
|
* `markSynced`.
|
|
*/
|
|
export interface FitbitResourceSyncOptions {
|
|
fullSync?: boolean;
|
|
/** The incremental lower bound, snapshotted once by the orchestrator. */
|
|
start?: Date;
|
|
/**
|
|
* When true, `upsertFitbitMeasurements` writes the rows but SKIPS the inline
|
|
* per-(type,day) DAY-rollup recompute + status-insight invalidate, returning
|
|
* the touched type-days to the caller instead. The orchestrator collapses
|
|
* them into ONE `recomputeUserRollups(from, to)` pass at the end of a
|
|
* `fullSync` cycle, so a multi-year backfill pays a single range-recompute
|
|
* rather than thousands of per-day round-trips. The incremental path leaves
|
|
* this unset and keeps the inline per-day hook (small touched set, warm read
|
|
* on the next tick).
|
|
*/
|
|
deferRollup?: boolean;
|
|
}
|
|
|
|
/**
|
|
* Compute the incremental `start` for a resource sync. `fullSync` returns
|
|
* undefined (the backfill walks deep history without a lower bound). Otherwise
|
|
* start from `lastSyncedAt - overlap`, or 30 days back on the very first
|
|
* incremental tick. Mirrors WHOOP's `incrementalStart`.
|
|
*/
|
|
export function incrementalStart(
|
|
lastSyncedAt: Date | null,
|
|
opts: { fullSync?: boolean; overlapMs?: number } = {},
|
|
): Date | undefined {
|
|
if (opts.fullSync) return undefined;
|
|
const overlap = opts.overlapMs ?? FITBIT_DEFAULT_OVERLAP_MS;
|
|
if (lastSyncedAt) return new Date(lastSyncedAt.getTime() - overlap);
|
|
return new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
|
|
}
|
|
|
|
/**
|
|
* One mapped reading destined for a `Measurement` row, with the per-point anchor
|
|
* already resolved into a full `externalId`. The per-resource syncs build these
|
|
* from the client mappers (`<anchor>:<fieldTag>`). Mirrors WHOOP's
|
|
* `WhoopMeasurementUpsert` (no `sleepStage` — sleep lands in W5).
|
|
*/
|
|
export interface FitbitMeasurementUpsert {
|
|
type: string;
|
|
value: number;
|
|
unit: string;
|
|
measuredAt: Date;
|
|
externalId: string;
|
|
/**
|
|
* Per-stage sleep rows carry the `SleepStage` axis so the (up to six) stage
|
|
* rows for one night stay distinct under the dedup key; every other row omits
|
|
* it (null). W5 (sleep) is the only producer.
|
|
*/
|
|
sleepStage?: "IN_BED" | "AWAKE" | "ASLEEP" | "REM" | "CORE" | "DEEP" | null;
|
|
}
|
|
|
|
/** Chunk size for the batched `createMany` insert of fresh readings. */
|
|
const FITBIT_CREATE_CHUNK = 500;
|
|
|
|
/**
|
|
* Write a batch of mapped Fitbit readings for one user and (unless deferred)
|
|
* fold the rollup tier + invalidate status-insight caches once at the end
|
|
* (mirrors the WHOOP / Withings sync tail). Returns the count of rows written
|
|
* plus the distinct `(type, day)` keys the write touched.
|
|
*
|
|
* TOMBSTONE-SAFE: the live-row probe filters `deletedAt: null`, so a row the
|
|
* user soft-deleted (iOS LWW reconciler) is NOT matched — a fresh insert is
|
|
* created instead of resurrecting the tombstone on the next hourly sync. The
|
|
* partial unique index (`Measurement … WHERE deleted_at IS NULL`) keeps the
|
|
* fresh insert from colliding with the live set; the tombstone sits outside it.
|
|
*
|
|
* OVERWRITE CONTRACT preserved: a re-fetched daily summary (`stats:`-keyed and
|
|
* every other stable per-point anchor) maps to the SAME live `externalId`, so
|
|
* the probe finds the existing live row and takes the in-place update branch
|
|
* (bumping `syncVersion` for the iOS LWW reconciler) — never a duplicate.
|
|
*
|
|
* BATCHING: the existence probe is a single `findMany` over the batch's
|
|
* externalIds; fresh rows go through chunked `createMany` (collapsing the N+1
|
|
* insert round-trips a deep backfill used to pay), and only the (bounded)
|
|
* already-live rows take a per-row `update` for their differing values.
|
|
*
|
|
* Best-effort on the rollup fold + insight invalidate — a populator hiccup never
|
|
* fails the user's sync.
|
|
*/
|
|
export async function upsertFitbitMeasurements(
|
|
userId: string,
|
|
readings: FitbitMeasurementUpsert[],
|
|
opts: { deferRollup?: boolean } = {},
|
|
): Promise<{
|
|
imported: number;
|
|
touched: Array<{ type: MeasurementType; measuredAt: Date }>;
|
|
}> {
|
|
if (readings.length === 0) return { imported: 0, touched: [] };
|
|
|
|
// Probe the LIVE rows (deletedAt: null) for every externalId in the batch in
|
|
// a single query. A tombstoned row deliberately does NOT appear here, so it is
|
|
// treated as absent → a fresh insert, not a resurrecting update.
|
|
const externalIds = readings.map((r) => r.externalId);
|
|
let liveByKey = new Map<string, { id: string }>();
|
|
try {
|
|
const existing = await prisma.measurement.findMany({
|
|
where: {
|
|
userId,
|
|
source: "FITBIT",
|
|
deletedAt: null,
|
|
externalId: { in: externalIds },
|
|
},
|
|
select: { id: true, type: true, externalId: true },
|
|
});
|
|
liveByKey = new Map(
|
|
existing
|
|
.filter((e) => e.externalId !== null)
|
|
.map((e) => [`${e.type} |