feat(import): read Health Connect exports

Upload the ZIP the Android Health Connect app writes and import its
`health_connect_export.db` in the background, the same way an Apple
Health export is imported: streamed to disk with a SHA-256, capped at
1 GiB, one import per account, a job row of kind `health_connect` that
the shared reconcile cron and staging sweep now look up on its own
queue.

The worker extracts the database (4 GiB ceiling, zip-bomb checks) and
opens it read-only with `immutable=1`, `trusted_schema=OFF` and
`query_only`. A view in place of a record table refuses the file,
columns are found by name, and schema versions below 9 are refused.
Records are read in batches of 500 and aggregated in SQLite: steps,
energy and distance as one total per day from the app the person's
Health Connect priority list ranks first, heart rate per minute for 90
days and as hourly means before that, sleep stages with the unknown
stage counted only in sessions without real stages, workouts, cycle
days, water and micronutrients. Apps whose integration the account has
connected are left out, and a reading already stored under another
source is skipped. Rows carry source HEALTH_CONNECT, `hc:<uuid>` or the
`stats:` ids, so a second import of the same export writes nothing.

Both staged files are removed on every way out; the job result holds
counts per type and per app only. Settings gain a Health Connect card
with the export steps, progress and the outcome, in all seven
languages, and the import is documented in
docs/integrations/health-connect.md.
This commit is contained in:
Marc-André Bombeck committed 2026-10-08 09:54:42 +02:00
1 parent 2622f63571
commit 996d2519d1
45 files changed
+6289 -100

No files matched your search

@@ -107,6 +107,8 @@ const ALLOWED: Record<string, string> = {
"Cross-source merge probe for one posted sleep segment: SLEEP_DURATION in a ±tolerance window around its end.",
"lib/targets/build-response.ts::{ in: recentTypes":
"Thirty days of weight, blood pressure, resting heart rate, body fat and steps (drained to one row per day). PULSE is filtered out of `recentTypes` and folds in SQL.",
"lib/import/health-connect/import.ts::{ in: spotTypes":
"The Health Connect import's same-reading check: the spot types of one 500-row batch (weight, blood pressure, vitals) in that batch's time window, ±2 s. Heart rate and sleep never take the check.",
"lib/doctor-report/collect.ts::{ notIn: rawExcluded":
"The report window's sparse types. Pulse, heart-rate variability and blood oxygen over 10 000 readings or any window over 90 days, and glucose over 90 days, are in `rawExcluded` and read as day buckets; glucose inside 90 days stays raw for the clinical panel (per-reading metrics), below 26 000 rows at a sensor's rate.",
};
+2 -2
View File
@@ -83,8 +83,8 @@ const NO_FLOOR_NEEDED: Readonly<Record<string, string>> = {
"unit test of the ownership helper",
// List a temporary directory the test itself filled.
"lib/import/__tests__/apple-health-staging.test.ts":
"lists its own temp staging directory",
"lib/import/__tests__/unzip-health-connect.test.ts":
"lists the temp directory to show a refused extraction left no file behind",
"lib/multipart/__tests__/stream-to-disk.test.ts":
"lists its own temp directory",
};
@@ -81,8 +81,6 @@ const RELEASE_PENDING_PREFIXES: Record<string, string> = {
"auth.claim": "v1.42 managed-profile claim page (#959)",
"recordSharing.managed.handover":
"v1.42 managed-profile handover dialog (#959)",
"settings.sections.export.import.healthConnect":
"v1.42 Health Connect import card (#972)",
"settings.sections.environment.airQuality":
"v1.42 environment settings, air quality (#615)",
"environment.fields.apparentMax": "v1.42 air-quality field label (#615)",
@@ -76,6 +76,10 @@ const WRITERS: Record<string, Disposition> = {
"lib/mcp/writes.ts": { kind: "validator" },
"lib/measurements/create-from-telegram.ts": { kind: "validator" },
"lib/measurements/import-apple-health-export.ts": { kind: "validator" },
// v1.42 — the Health Connect export import. Spot rows go through the bulk
// insert after `validateMeasurementRange`; day totals through the gated
// reconcile seam.
"lib/import/health-connect/import.ts": { kind: "validator" },
"app/api/import/csv/route.ts": {
kind: "upstream",
validatedIn: "lib/import/csv-measurements.ts",
@@ -72,6 +72,8 @@ const WRITE_SITES: Record<string, string> = {
"a primitive; every caller runs the tail over the verdicts' dirty identities",
"lib/measurements/import-apple-health-export.ts":
"the import worker refolds the import's span and re-warms every imported type",
"lib/import/health-connect/import.ts":
"the Health Connect import worker refolds the import's span and re-warms every imported type",
"lib/export/restore-backup.ts":
"a restore replaces the whole record: it refolds the whole window and drops every per-user cache",
// Value-preserving reshapes: the same readings folded into daily or hourly
@@ -157,8 +159,12 @@ describe("measurement write tail", () => {
const handRolled = Object.entries(WRITE_SITES)
.filter(([, reason]) => reason.includes("re-warms"))
.map(([file]) => file)
// The two export importers leave both legs to their workers, which
// run them once over the whole import.
.filter(
(file) => file !== "lib/measurements/import-apple-health-export.ts",
(file) =>
file !== "lib/measurements/import-apple-health-export.ts" &&
file !== "lib/import/health-connect/import.ts",
);
expect(handRolled.length).toBeGreaterThan(0);
const missing = handRolled.filter((file) => {
@@ -107,6 +107,15 @@ const PROVIDER_INGESTS: ProviderIngest[] = [
writeFile: "lib/measurements/import-apple-health-export.ts",
metadataKey: "activityType",
},
{
provider: "Health Connect export",
mapper: "mapHealthConnectSportType",
mapperFile: "lib/import/health-connect/mapper.ts",
captureFile: "lib/import/health-connect/import.ts",
captureField: "exercise_type",
writeFile: "lib/import/health-connect/import.ts",
metadataKey: "healthConnectExerciseType",
},
{
provider: "Fitbit",
mapper: "mapFitbitSportType",
@@ -22,6 +22,7 @@ import { checkRateLimit } from "@/lib/rate-limit";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import {
APPLE_HEALTH_IMPORT_V2_QUEUE,
APPLE_HEALTH_IMPORT_KIND,
APPLE_HEALTH_IMPORT_PARSER_REVISION,
APPLE_HEALTH_IMPORT_SEND_OPTIONS,
type AppleHealthImportPayload,
@@ -31,6 +32,7 @@ import {
createImportJobUnlessBusy,
discardStagedUpload,
IMPORT_BUSY_CODE,
IMPORT_BUSY_MESSAGE,
} from "@/lib/import/apple-health-staging";
import { unlink } from "node:fs/promises";
@@ -128,6 +130,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
const existing = await prisma.importJob.findFirst({
where: {
userId: targetUser.id,
kind: APPLE_HEALTH_IMPORT_KIND,
uploadSha256: uploaded.sha256,
parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
status: { not: "failed" },
@@ -170,11 +173,10 @@ export const POST = apiHandler(async (request: NextRequest) => {
});
if ("busy" in staged) {
await discardStagedUpload(uploaded.filePath);
return apiError(
"An Apple Health import is already running for this account. Wait for it to finish, then upload again.",
409,
{ errorCode: IMPORT_BUSY_CODE, jobId: staged.busy.id },
);
return apiError(IMPORT_BUSY_MESSAGE, 409, {
errorCode: IMPORT_BUSY_CODE,
jobId: staged.busy.id,
});
}
const importJob = staged.created;
@@ -30,6 +30,7 @@ import { checkRateLimit } from "@/lib/rate-limit";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import {
APPLE_HEALTH_IMPORT_V2_QUEUE,
APPLE_HEALTH_IMPORT_KIND,
APPLE_HEALTH_IMPORT_PARSER_REVISION,
APPLE_HEALTH_IMPORT_SEND_OPTIONS,
type AppleHealthImportPayload,
@@ -39,6 +40,7 @@ import {
createImportJobUnlessBusy,
discardStagedUpload,
IMPORT_BUSY_CODE,
IMPORT_BUSY_MESSAGE,
} from "@/lib/import/apple-health-staging";
import { unlink } from "node:fs/promises";
@@ -123,6 +125,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
const existing = await prisma.importJob.findFirst({
where: {
userId: user.id,
kind: APPLE_HEALTH_IMPORT_KIND,
uploadSha256: uploaded.sha256,
parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
status: { not: "failed" },
@@ -167,11 +170,10 @@ export const POST = apiHandler(async (request: NextRequest) => {
});
if ("busy" in staged) {
await discardStagedUpload(uploaded.filePath);
return apiError(
"An Apple Health import is already running for this account. Wait for it to finish, then upload again.",
409,
{ errorCode: IMPORT_BUSY_CODE, jobId: staged.busy.id },
);
return apiError(IMPORT_BUSY_MESSAGE, 409, {
errorCode: IMPORT_BUSY_CODE,
jobId: staged.busy.id,
});
}
const importJob = staged.created;
@@ -0,0 +1,252 @@
/**
* The Health Connect upload and status routes: the size cap, the staging
* name the sweep knows, the job's `kind`, the content-hash dedupe, the one
* import per account rule and the status of the account's latest job.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
vi.mock("@/lib/auth/audit", () => ({
auditLog: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() }));
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("next/headers", () => ({
headers: vi.fn(async () => ({ get: () => null })),
cookies: vi.fn(async () => ({
get: () => undefined,
set: () => {},
delete: () => {},
})),
}));
const mocks = vi.hoisted(() => ({
runningImportFindFirst: vi.fn(async () => null),
checkRateLimit: vi.fn().mockResolvedValue({ allowed: true }),
bossSend: vi.fn().mockResolvedValue("boss-job-1"),
getGlobalBoss: vi.fn(),
importJobCreate: vi.fn(),
importJobFindFirst: vi.fn().mockResolvedValue(null),
importJobUpdate: vi.fn(),
streamToDisk: vi.fn(),
unlink: vi.fn().mockResolvedValue(undefined),
rm: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/rate-limit", () => ({
checkRateLimit: mocks.checkRateLimit,
}));
vi.mock("@/lib/jobs/boss-instance", () => ({
getGlobalBoss: mocks.getGlobalBoss,
}));
vi.mock("@/lib/multipart/stream-to-disk", () => ({
streamMultipartToDisk: mocks.streamToDisk,
}));
vi.mock("node:fs/promises", async (importOriginal) => {
const actual = await importOriginal<typeof import("node:fs/promises")>();
return { ...actual, unlink: mocks.unlink, rm: mocks.rm };
});
vi.mock("@/lib/db", () => ({
prisma: {
importJob: {
create: mocks.importJobCreate,
findFirst: mocks.importJobFindFirst,
update: mocks.importJobUpdate,
},
// The one-running-import check and the new row, in one transaction.
// The check finds nothing running unless a test says otherwise.
$transaction: async (fn: (tx: unknown) => unknown) =>
fn({
$executeRaw: async () => 0,
importJob: {
findFirst: mocks.runningImportFindFirst,
create: mocks.importJobCreate,
},
}),
},
}));
import { POST } from "../route";
import { GET as STATUS } from "../status/route";
import { getSession } from "@/lib/auth/session";
const SESSION_OK = {
session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
user: { id: "user-1", username: "testuser", role: "USER" as const },
};
const STAGED_PATH =
"/tmp/healthlog-health-connect-import-0b7f3c2a-1d4e-4f5a-9b8c-7d6e5f4a3b2c.bin";
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
mocks.checkRateLimit.mockResolvedValue({ allowed: true });
mocks.bossSend.mockResolvedValue("boss-job-1");
mocks.getGlobalBoss.mockReturnValue({ send: mocks.bossSend });
mocks.importJobCreate.mockResolvedValue({ id: "ij-1" });
mocks.importJobFindFirst.mockResolvedValue(null);
mocks.runningImportFindFirst.mockResolvedValue(null);
mocks.importJobUpdate.mockResolvedValue({ id: "ij-1" });
mocks.streamToDisk.mockResolvedValue({
filePath: STAGED_PATH,
bytes: 100,
sha256: "deadbeef",
originalFilename: "Health Connect.zip",
textFields: {},
});
mocks.unlink.mockResolvedValue(undefined);
mocks.rm.mockResolvedValue(undefined);
});
function upload(headers: Record<string, string> = {}): NextRequest {
return new NextRequest("http://localhost/api/import/health-connect-export", {
method: "POST",
headers: {
"content-type": "multipart/form-data; boundary=---x",
...headers,
},
body: "stub",
});
}
describe("POST /api/import/health-connect-export", () => {
it("returns 401 when unauthenticated", async () => {
vi.mocked(getSession).mockResolvedValue(null);
expect((await POST(upload())).status).toBe(401);
});
it("refuses a declared body above 1 GiB before reading it", async () => {
const res = await POST(
upload({ "content-length": String(1024 * 1024 * 1024 + 1) }),
);
expect(res.status).toBe(413);
expect(mocks.streamToDisk).not.toHaveBeenCalled();
});
it("caps the streamed body at 1 GiB and stages it under its own name", async () => {
await POST(upload());
expect(mocks.streamToDisk).toHaveBeenCalledWith(
expect.anything(),
expect.any(String),
expect.objectContaining({
maxBytes: 1024 * 1024 * 1024,
tmpPrefix: "healthlog-health-connect-import",
}),
);
});
it("creates a health_connect job and queues it on its own queue", async () => {
const res = await POST(upload());
expect(res.status).toBe(202);
expect((await res.json()).data).toEqual({
jobId: "ij-1",
status: "queued",
});
expect(mocks.importJobCreate).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({
userId: "user-1",
kind: "health_connect",
uploadSha256: "deadbeef",
}),
}),
);
expect(mocks.bossSend).toHaveBeenCalledWith(
"health-connect-import",
{ userId: "user-1", importJobId: "ij-1", uploadPath: STAGED_PATH },
{ retryLimit: 0, expireInSeconds: 21600 },
);
});
it("resolves the same bytes to the earlier job and drops the new copy", async () => {
mocks.importJobFindFirst.mockResolvedValue({ id: "ij-0", status: "done" });
const res = await POST(upload());
expect(res.status).toBe(202);
expect((await res.json()).data).toEqual({
jobId: "ij-0",
status: "done",
idempotent: true,
});
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({
kind: "health_connect",
status: { not: "failed" },
}),
}),
);
expect(mocks.unlink).toHaveBeenCalledWith(STAGED_PATH);
expect(mocks.bossSend).not.toHaveBeenCalled();
});
it("refuses a second import while one runs and discards the upload", async () => {
mocks.runningImportFindFirst.mockResolvedValue({
id: "ij-running",
status: "parsing",
} as never);
const res = await POST(upload());
expect(res.status).toBe(409);
const body = await res.json();
expect(body.meta).toMatchObject({
errorCode: "import.apple_health.busy",
jobId: "ij-running",
});
expect(mocks.rm).toHaveBeenCalledWith(STAGED_PATH, { force: true });
expect(mocks.bossSend).not.toHaveBeenCalled();
});
it("answers 503 and removes the upload when no worker runs", async () => {
mocks.getGlobalBoss.mockReturnValue(null);
const res = await POST(upload());
expect(res.status).toBe(503);
expect(mocks.rm).toHaveBeenCalledWith(STAGED_PATH, { force: true });
});
});
describe("GET /api/import/health-connect-export/status", () => {
it("answers null when the account never imported", async () => {
const res = await STATUS(
new NextRequest(
"http://localhost/api/import/health-connect-export/status",
),
);
expect(res.status).toBe(200);
expect((await res.json()).data).toEqual({ job: null });
});
it("answers the account's latest Health Connect job", async () => {
mocks.importJobFindFirst.mockResolvedValue({
id: "ij-9",
status: "done",
startedAt: new Date("2026-10-01T10:00:00Z"),
completedAt: new Date("2026-10-01T10:01:00Z"),
uploadBytes: 1234,
progress: { rowsUpserted: 5 },
result: { kind: "health_connect", totals: { rowsUpserted: 5 } },
failureReason: null,
});
const res = await STATUS(
new NextRequest(
"http://localhost/api/import/health-connect-export/status",
),
);
expect((await res.json()).data.job).toMatchObject({
jobId: "ij-9",
status: "done",
completedAt: "2026-10-01T10:01:00.000Z",
});
expect(mocks.importJobFindFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: { userId: "user-1", kind: "health_connect" },
}),
);
});
});
@@ -1,15 +1,191 @@
/**
* `POST /api/import/health-connect-export` — upload an Android Health Connect
* export ZIP for background import (v1.42, #972). Outside the proxy matcher
* like the Apple Health upload (`src/lib/http/proxy-bypass-routes.ts`), so the
* body is streamed to disk after authentication instead of being buffered.
* export ZIP for background import (v1.42, #972).
*
* Contract stub: authenticates as the finished route will, then answers 501.
* Mirrors the Apple Health upload (`../apple-health-export/route.ts`): the
* route sits outside the proxy matcher (`src/lib/http/proxy-bypass-routes.ts`)
* so the body is streamed to disk after authentication instead of buffered,
* hashed with SHA-256 on the way, refused past the size cap, and handed to
* the `health-connect-import` queue. One import runs per account at a time,
* whatever its kind (`createImportJobUnlessBusy`). Re-uploading the same
* bytes resolves to the still-viable job instead of importing twice; a failed
* job never blocks a retry of the same file.
*
* The audit row carries the upload's size and hash and nothing from inside
* the file.
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { notImplemented } from "@/lib/http/not-implemented";
import type { NextRequest } from "next/server";
import { unlink } from "node:fs/promises";
export const POST = apiHandler(async () => {
await requireAuth();
return notImplemented();
import { prisma } from "@/lib/db";
import { apiHandler, requireAuth, HttpError } from "@/lib/api-handler";
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { auditLog } from "@/lib/auth/audit";
import { checkRateLimit } from "@/lib/rate-limit";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import {
HEALTH_CONNECT_IMPORT_KIND,
HEALTH_CONNECT_IMPORT_PARSER_REVISION,
HEALTH_CONNECT_IMPORT_QUEUE,
HEALTH_CONNECT_IMPORT_SEND_OPTIONS,
type HealthConnectImportPayload,
} from "@/lib/jobs/health-connect-import-worker";
import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk";
import {
createImportJobUnlessBusy,
discardStagedUpload,
IMPORT_BUSY_CODE,
IMPORT_BUSY_MESSAGE,
} from "@/lib/import/apple-health-staging";
export const dynamic = "force-dynamic";
/**
* Hard cap on the upload. The ZIP holds one SQLite file that compresses to
* about half; a multi-year export with minute-by-minute heart rate sits in
* the low hundreds of megabytes, and 1 GiB leaves room for the long tail.
*/
const HEALTH_CONNECT_MAX_UPLOAD_BYTES = 1024 * 1024 * 1024;
/** Three uploads per minute per account, like the Apple Health upload. */
const RATE_LIMIT_MAX = 3;
const RATE_LIMIT_WINDOW_MS = 60_000;
export const POST = apiHandler(async (request: NextRequest) => {
const { user } = await requireAuth();
annotate({ action: { name: "import.health-connect.kickoff" } });
const rl = await checkRateLimit(
`import:health-connect:${user.id}`,
RATE_LIMIT_MAX,
RATE_LIMIT_WINDOW_MS,
);
if (!rl.allowed) {
throw new HttpError(429, "Too many import uploads, try again later");
}
const declaredBytes = Number(request.headers.get("content-length") ?? 0);
if (declaredBytes > HEALTH_CONNECT_MAX_UPLOAD_BYTES) {
await auditLog("import.health-connect.kickoff.denied", {
userId: user.id,
ipAddress: getClientIp(request),
details: {
reason: "content_length_exceeded",
contentLength: declaredBytes,
},
});
return apiError("Upload exceeds the 1 GB limit", 413);
}
const body = request.body;
if (!body) return apiError("Request body is required", 400);
let uploaded;
try {
uploaded = await streamMultipartToDisk(
body,
request.headers.get("content-type"),
{
maxBytes: HEALTH_CONNECT_MAX_UPLOAD_BYTES,
fieldName: "file",
tmpPrefix: "healthlog-health-connect-import",
},
);
} catch (err) {
const message = err instanceof Error ? err.message : "upload_failed";
await auditLog("import.health-connect.kickoff.denied", {
userId: user.id,
ipAddress: getClientIp(request),
details: { reason: "stream_to_disk_failed" },
});
return apiError(`Multipart upload failed: ${message}`, 422);
}
const existing = await prisma.importJob.findFirst({
where: {
userId: user.id,
kind: HEALTH_CONNECT_IMPORT_KIND,
uploadSha256: uploaded.sha256,
parserRevision: HEALTH_CONNECT_IMPORT_PARSER_REVISION,
status: { not: "failed" },
},
orderBy: { startedAt: "desc" },
});
if (existing) {
await unlink(uploaded.filePath).catch(() => {});
annotate({ meta: { idempotent_hit: true, job_id: existing.id } });
return apiSuccess(
{ jobId: existing.id, status: existing.status, idempotent: true },
202,
);
}
const boss = getGlobalBoss();
if (!boss) {
await auditLog("import.health-connect.kickoff.denied", {
userId: user.id,
ipAddress: getClientIp(request),
details: { reason: "worker_not_running" },
});
await discardStagedUpload(uploaded.filePath);
throw new HttpError(503, "Background worker is not running");
}
const staged = await createImportJobUnlessBusy(prisma, {
userId: user.id,
kind: HEALTH_CONNECT_IMPORT_KIND,
status: "queued",
uploadBytes: uploaded.bytes,
uploadSha256: uploaded.sha256,
parserRevision: HEALTH_CONNECT_IMPORT_PARSER_REVISION,
});
if ("busy" in staged) {
await discardStagedUpload(uploaded.filePath);
return apiError(IMPORT_BUSY_MESSAGE, 409, {
errorCode: IMPORT_BUSY_CODE,
jobId: staged.busy.id,
});
}
const importJob = staged.created;
const payload: HealthConnectImportPayload = {
userId: user.id,
importJobId: importJob.id,
uploadPath: uploaded.filePath,
};
const bossJobId = await boss.send(
HEALTH_CONNECT_IMPORT_QUEUE,
payload,
HEALTH_CONNECT_IMPORT_SEND_OPTIONS,
);
if (!bossJobId) {
await prisma.importJob.update({
where: { id: importJob.id },
data: {
status: "failed",
failureReason:
"The import could not be queued. Upload the export again.",
completedAt: new Date(),
},
});
await discardStagedUpload(uploaded.filePath);
throw new HttpError(503, "The import could not be queued");
}
await prisma.importJob.update({
where: { id: importJob.id },
data: { pgBossJobId: bossJobId },
});
await auditLog("import.health-connect.kickoff", {
userId: user.id,
ipAddress: getClientIp(request),
details: {
jobId: importJob.id,
uploadBytes: uploaded.bytes,
uploadSha256: uploaded.sha256,
},
});
return apiSuccess({ jobId: importJob.id, status: "queued" as const }, 202);
});
@@ -1,13 +1,53 @@
/**
* `GET /api/import/health-connect-export/status` — the caller's most recent
* Health Connect import job (v1.42, #972).
* Health Connect import job (v1.42, #972), or `null` when there never was
* one. The settings card polls it while a job runs and shows the last
* outcome after a reload.
*
* Contract stub: authenticates as the finished route will, then answers 501.
* The result is counts only (per type, per app); the import never stores a
* value or a timestamp from the file on the job row.
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { notImplemented } from "@/lib/http/not-implemented";
import type { NextRequest } from "next/server";
export const GET = apiHandler(async () => {
await requireAuth();
return notImplemented();
import { prisma } from "@/lib/db";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { HEALTH_CONNECT_IMPORT_KIND } from "@/lib/jobs/health-connect-import-worker";
export const dynamic = "force-dynamic";
export const GET = apiHandler(async (_request: NextRequest) => {
const { user } = await requireAuth();
annotate({ action: { name: "import.health-connect.status" } });
const row = await prisma.importJob.findFirst({
where: { userId: user.id, kind: HEALTH_CONNECT_IMPORT_KIND },
orderBy: { startedAt: "desc" },
select: {
id: true,
status: true,
startedAt: true,
completedAt: true,
uploadBytes: true,
progress: true,
result: true,
failureReason: true,
},
});
return apiSuccess({
job: row
? {
jobId: row.id,
status: row.status,
startedAt: row.startedAt.toISOString(),
completedAt: row.completedAt?.toISOString() ?? null,
uploadBytes: row.uploadBytes,
progress: (row.progress as Record<string, unknown>) ?? {},
result: (row.result as Record<string, unknown> | null) ?? null,
failureReason: row.failureReason,
}
: null,
});
});
@@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import {
healthConnectFailureKind,
summarizeHealthConnectResult,
} from "../health-connect-import-card";
describe("healthConnectFailureKind", () => {
it("reads the code at the front of a worker reason", () => {
expect(
healthConnectFailureKind("unsupported_version: database version 7"),
).toBe("unsupportedVersion");
expect(healthConnectFailureKind("not_health_connect: no member")).toBe(
"notHealthConnect",
);
expect(healthConnectFailureKind("unsafe_schema: x is a view")).toBe(
"unsafeSchema",
);
expect(healthConnectFailureKind("too_large: 5 GB")).toBe("tooLarge");
expect(healthConnectFailureKind("staging_missing: gone")).toBe(
"stagingMissing",
);
expect(healthConnectFailureKind("interrupted_by_restart")).toBe(
"interrupted",
);
expect(healthConnectFailureKind("connection reset")).toBe("raw");
expect(healthConnectFailureKind(null)).toBeNull();
});
});
describe("summarizeHealthConnectResult", () => {
it("lists the written types, largest first, and the records left out", () => {
const summary = summarizeHealthConnectResult({
perType: {
WEIGHT: { inserted: 3, updated: 1 },
PULSE: { inserted: 1440 },
BODY_FAT: { inserted: 0, skipped: 2 },
},
perApp: {
"com.withings.wiscale2": { records: 12, leftOut: true },
"com.fitbit.FitbitMobile": { records: 1500, leftOut: false },
},
totals: { rowsUpserted: 1444 },
});
expect(summary.types).toEqual([
{ type: "PULSE", written: 1440 },
{ type: "WEIGHT", written: 4 },
]);
expect(summary.written).toBe(1444);
expect(summary.leftOutRecords).toBe(12);
});
it("counts genuine refusals, not deliberate skips", () => {
const summary = summarizeHealthConnectResult({
totals: { rowsUpserted: 1 },
skipped: {
connected_integration: 40,
"SLEEP_DURATION::out_of_bed": 4,
"WEIGHT::out_of_range": 2,
},
});
expect(summary.refused).toBe(2);
});
});
@@ -0,0 +1,401 @@
"use client";
/**
* v1.42 (#972) — Settings → Export & Import → Health Connect export.
*
* Uploads the ZIP the Android Health Connect app writes and follows the
* background import through `GET /api/import/health-connect-export/status`,
* which answers the account's latest job, so the outcome is still there
* after a reload. Health Connect only exports to a cloud storage app, never
* to the phone's own storage; the card says so up front, because that is
* where people get stuck.
*/
import { useCallback, useId, useRef, useState } from "react";
import { useQuery, useQueryClient } from "@tanstack/react-query";
import { AlertCircle, Loader2, Smartphone, Upload } from "lucide-react";
import { Button } from "@/components/ui/button";
import { SettingsCardActions } from "@/components/settings/_card-actions";
import { WrittenOutcomeLine } from "@/components/outcome/written-outcome-line";
import { apiFetchRaw, apiGet } from "@/lib/api/api-fetch";
import { useTranslations } from "@/lib/i18n/context";
import { MEASUREMENT_TYPE_LABEL_KEYS } from "@/lib/measurements/type-label-keys";
import { classifyWrittenOutcome } from "@/lib/outcome/written-outcome";
import { queryKeys } from "@/lib/query-keys";
import { cn } from "@/lib/utils";
import type { MeasurementType } from "@/generated/prisma/client";
import { ImportCardShell } from "./import-card-shell";
const TERMINAL_STATES: readonly string[] = ["done", "failed"];
interface TypeStat {
inserted?: number;
updated?: number;
skipped?: number;
}
/** The job as the status route answers it. Counts only. */
export interface HealthConnectJob {
jobId: string;
status: string;
progress: { rowsUpserted?: number; recordsRead?: number } | null;
result: {
perType?: Record<string, TypeStat>;
perApp?: Record<string, { records?: number; leftOut?: boolean }>;
workouts?: { inserted?: number };
cycle?: { written?: number; moduleOff?: boolean };
nutrients?: { written?: number; moduleOff?: boolean };
totals?: { rowsUpserted?: number; recordsRead?: number };
warnings?: string[];
skipped?: Record<string, number>;
} | null;
failureReason: string | null;
}
/** The failure code at the front of a worker reason, when there is one. */
export function healthConnectFailureKind(
reason: string | null,
):
| "unsupportedVersion"
| "notHealthConnect"
| "unsafeSchema"
| "tooLarge"
| "stagingMissing"
| "interrupted"
| "raw"
| null {
if (!reason) return null;
if (reason === "interrupted_by_restart") return "interrupted";
if (reason.startsWith("unsupported_version")) return "unsupportedVersion";
if (reason.startsWith("not_health_connect")) return "notHealthConnect";
if (reason.startsWith("unsafe_schema")) return "unsafeSchema";
if (reason.startsWith("too_large")) return "tooLarge";
if (reason.startsWith("staging_missing")) return "stagingMissing";
return "raw";
}
/** What the result line and the per-type list show. */
export function summarizeHealthConnectResult(
result: HealthConnectJob["result"],
): {
written: number;
refused: number;
types: Array<{ type: string; written: number }>;
leftOutRecords: number;
} {
const types: Array<{ type: string; written: number }> = [];
let refused = 0;
for (const [type, stat] of Object.entries(result?.perType ?? {})) {
const written = (stat.inserted ?? 0) + (stat.updated ?? 0);
if (written > 0) types.push({ type, written });
}
types.sort((a, b) => b.written - a.written);
let leftOutRecords = 0;
for (const app of Object.values(result?.perApp ?? {})) {
if (app.leftOut) leftOutRecords += app.records ?? 0;
}
for (const [reason, count] of Object.entries(result?.skipped ?? {})) {
if (/::(out_of_range|write_failed|unreadable_record)$/.test(reason)) {
refused += count;
}
}
return {
written: result?.totals?.rowsUpserted ?? 0,
refused,
types,
leftOutRecords,
};
}
export function HealthConnectImportCard() {
const { t } = useTranslations();
const queryClient = useQueryClient();
const fileInputRef = useRef<HTMLInputElement>(null);
const [uploading, setUploading] = useState(false);
const [uploadError, setUploadError] = useState<string | null>(null);
const [dragActive, setDragActive] = useState(false);
const dropDescId = useId();
const statusQuery = useQuery({
queryKey: queryKeys.healthConnectImportStatus(),
refetchInterval: (query) => {
const job = (
query.state.data as { job: HealthConnectJob | null } | undefined
)?.job;
return job && !TERMINAL_STATES.includes(job.status) ? 2000 : false;
},
queryFn: () =>
apiGet<{ job: HealthConnectJob | null }>(
"/api/import/health-connect-export/status",
{ credentials: "include" },
),
});
const upload = useCallback(
async (file: File) => {
setUploadError(null);
setUploading(true);
try {
const form = new FormData();
form.append("file", file);
const res = await apiFetchRaw("/api/import/health-connect-export", {
method: "POST",
credentials: "include",
body: form,
});
if (res.status === 429)
return setUploadError(
t("settings.sections.export.import.healthConnect.rateLimited"),
);
if (res.status === 413)
return setUploadError(
t("settings.sections.export.import.healthConnect.tooLarge"),
);
if (res.status === 409)
return setUploadError(
t("settings.sections.export.import.healthConnect.busy"),
);
if (!res.ok)
return setUploadError(
t("settings.sections.export.import.healthConnect.uploadFailed"),
);
await queryClient.invalidateQueries({
queryKey: queryKeys.healthConnectImportStatus(),
});
} catch {
setUploadError(
t("settings.sections.export.import.healthConnect.uploadFailed"),
);
} finally {
setUploading(false);
}
},
[t, queryClient],
);
const job = statusQuery.data?.job ?? null;
const isRunning = job !== null && !TERMINAL_STATES.includes(job.status);
const isDone = job?.status === "done";
const isFailed = job?.status === "failed";
const busy = uploading || isRunning;
const summary = summarizeHealthConnectResult(job?.result ?? null);
const failureText = (() => {
const reason = job?.failureReason ?? null;
const kind = healthConnectFailureKind(reason);
if (kind === null) return null;
if (kind === "raw")
return t("settings.sections.export.import.healthConnect.failed", {
reason: reason ?? "",
});
return t(`settings.sections.export.import.healthConnect.failure.${kind}`);
})();
return (
<ImportCardShell
testId="import-card-health-connect"
icon={Smartphone}
title={t("settings.sections.export.import.healthConnect.title")}
description={t(
"settings.sections.export.import.healthConnect.description",
)}
>
<ol className="text-foreground list-decimal space-y-1 pl-4 text-xs">
<li>
{t("settings.sections.export.import.healthConnect.steps.export")}
</li>
<li>
{t("settings.sections.export.import.healthConnect.steps.cloud")}
</li>
<li>
{t("settings.sections.export.import.healthConnect.steps.upload")}
</li>
</ol>
<div
role="button"
tabIndex={0}
aria-describedby={dropDescId}
aria-disabled={busy}
onClick={() => !busy && fileInputRef.current?.click()}
onKeyDown={(e) => {
if ((e.key === "Enter" || e.key === " ") && !busy) {
e.preventDefault();
fileInputRef.current?.click();
}
}}
onDragOver={(e) => {
e.preventDefault();
if (!busy) setDragActive(true);
}}
onDragLeave={() => setDragActive(false)}
onDrop={(e) => {
e.preventDefault();
setDragActive(false);
const file = e.dataTransfer.files?.[0];
if (file && !busy) void upload(file);
}}
className={cn(
"border-border bg-muted/20 hover:bg-muted/40 focus-visible:ring-ring/50 flex min-h-24 cursor-pointer flex-col items-center justify-center gap-1 rounded-lg border border-dashed p-4 text-center transition-colors focus-visible:ring-2 focus-visible:outline-none",
dragActive && "border-primary bg-primary/5",
busy && "pointer-events-none opacity-60",
)}
>
<Upload className="text-muted-foreground h-5 w-5" aria-hidden="true" />
<span className="text-foreground text-sm font-medium">
{t("settings.sections.export.import.healthConnect.dropLabel")}
</span>
<span id={dropDescId} className="text-muted-foreground text-xs">
{t("settings.sections.export.import.healthConnect.dropHint")}
</span>
</div>
<input
ref={fileInputRef}
type="file"
accept=".zip,application/zip"
className="sr-only"
aria-label={t(
"settings.sections.export.import.healthConnect.fileInputLabel",
)}
onChange={(e) => {
const file = e.target.files?.[0];
if (file) void upload(file);
e.target.value = "";
}}
/>
<p className="text-muted-foreground text-xs">
{t("settings.sections.export.import.healthConnect.privacyNote")}
</p>
<div aria-live="polite" className="space-y-2">
{uploading && (
<p className="text-muted-foreground flex items-center gap-2 text-xs">
<Loader2 className="h-3.5 w-3.5 animate-spin motion-reduce:animate-none" />
{t("settings.sections.export.import.healthConnect.uploading")}
</p>
)}
{isRunning && (
<div
data-testid="import-health-connect-progress"
className="space-y-1"
>
<p className="text-muted-foreground flex items-center gap-2 text-xs">
<Loader2 className="h-3.5 w-3.5 animate-spin motion-reduce:animate-none" />
{t(
`settings.sections.export.import.healthConnect.phase.${job?.status ?? "queued"}`,
)}
</p>
{typeof job?.progress?.rowsUpserted === "number" && (
<p className="text-muted-foreground text-xs">
{t("settings.sections.export.import.healthConnect.rowsSoFar", {
count: job.progress.rowsUpserted,
})}
</p>
)}
</div>
)}
{isDone && (
<WrittenOutcomeLine
outcome={classifyWrittenOutcome({
written: summary.written,
skipped: summary.refused,
})}
message={
summary.written > 0
? t(
"settings.sections.export.import.healthConnect.doneSummary",
{ count: summary.written },
)
: t("settings.sections.export.import.healthConnect.doneNothing")
}
testId="import-health-connect-result"
/>
)}
{isDone && summary.types.length > 0 && (
<ul
data-testid="import-health-connect-types"
className="text-muted-foreground space-y-0.5 text-xs"
>
{summary.types.map(({ type, written }) => (
<li key={type} className="flex justify-between gap-2">
<span className="text-foreground">
{MEASUREMENT_TYPE_LABEL_KEYS[type as MeasurementType]
? t(MEASUREMENT_TYPE_LABEL_KEYS[type as MeasurementType])
: type}
</span>
<span className="tabular-nums">{written}</span>
</li>
))}
{(job?.result?.workouts?.inserted ?? 0) > 0 && (
<li className="flex justify-between gap-2">
<span className="text-foreground">
{t("settings.sections.export.import.healthConnect.workouts")}
</span>
<span className="tabular-nums">
{job?.result?.workouts?.inserted}
</span>
</li>
)}
</ul>
)}
{isDone && summary.leftOutRecords > 0 && (
<p className="text-muted-foreground text-xs">
{t(
"settings.sections.export.import.healthConnect.leftOutConnected",
{ count: summary.leftOutRecords },
)}
</p>
)}
{isDone && job?.result?.nutrients?.moduleOff && (
<p className="text-muted-foreground text-xs">
{t(
"settings.sections.export.import.healthConnect.nutrientsModuleOff",
)}
</p>
)}
{isDone && job?.result?.cycle?.moduleOff && (
<p className="text-muted-foreground text-xs">
{t("settings.sections.export.import.healthConnect.cycleModuleOff")}
</p>
)}
{(isFailed || uploadError) && (
<p
data-testid="import-health-connect-error"
role="alert"
className="text-destructive flex items-start gap-2 text-sm"
>
<AlertCircle
className="mt-0.5 h-3.5 w-3.5 shrink-0"
aria-hidden="true"
/>
<span>
{uploadError ??
failureText ??
t("settings.sections.export.import.healthConnect.uploadFailed")}
</span>
</p>
)}
</div>
<SettingsCardActions className="mt-auto" align="start">
<Button
type="button"
variant="outline"
size="sm"
className="min-h-11 sm:min-h-9"
disabled={busy}
onClick={() => fileInputRef.current?.click()}
data-testid="import-action-health-connect"
>
{busy ? (
<Loader2 className="h-3.5 w-3.5 animate-spin motion-reduce:animate-none" />
) : (
<Upload className="h-3.5 w-3.5" />
)}
{t("settings.sections.export.import.healthConnect.choose")}
</Button>
</SettingsCardActions>
</ImportCardShell>
);
}
@@ -8,6 +8,8 @@
*
* 1. Apple Health `export.zip` — `apple-health-import-card.tsx`
* (multipart POST + status poll).
* 1b. Health Connect export ZIP — `health-connect-import-card.tsx`
* (v1.42; multipart POST + poll of the account's latest job).
* 2. Generic JSON import — `json-import-card.tsx`
* (file/paste, client-side parseability guard).
* 3. CSV import — `csv-import-card.tsx`
@@ -22,6 +24,7 @@ import { Upload } from "lucide-react";
import { useTranslations } from "@/lib/i18n/context";
import { SectionHeading } from "@/components/ui/section-heading";
import { AppleHealthImportCard } from "./apple-health-import-card";
import { HealthConnectImportCard } from "./health-connect-import-card";
import { JsonImportCard } from "./json-import-card";
import { CsvImportCard } from "./csv-import-card";
import { DoseHistoryImportCard } from "./dose-history-import-card";
@@ -48,6 +51,7 @@ export function ImportPanel() {
/>
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-3">
<AppleHealthImportCard />
<HealthConnectImportCard />
<JsonImportCard />
<CsvImportCard />
<DoseHistoryImportCard />
+13 -3
View File
@@ -45,6 +45,16 @@ import type {
Prisma,
} from "@/generated/prisma/client";
/**
* A day-log as the write helper takes it. The client surfaces parse
* `CycleDayLogInput`, whose `source` is the closed client set (manual or
* Apple Health); the Health Connect export import is server-side and writes
* its own provenance, which no client may claim.
*/
export type CycleDayLogWriteEntry = Omit<CycleDayLogInput, "source"> & {
source: CycleDayLogInput["source"] | "HEALTH_CONNECT";
};
export interface DayLogWriteResult {
id: string;
existed: boolean;
@@ -210,7 +220,7 @@ const EXISTING_SELECT = {
*/
export async function upsertCycleDayLog(
userId: string,
entry: CycleDayLogInput,
entry: CycleDayLogWriteEntry,
tz: string | null,
cycleId: string | null = null,
encryptSensitiveFlag?: boolean,
@@ -256,10 +266,10 @@ export async function upsertCycleDayLog(
interface WriteArgs {
userId: string;
entry: CycleDayLogInput;
entry: CycleDayLogWriteEntry;
tz: string | null;
cycleId: string | null;
source: CycleDayLogInput["source"];
source: CycleDayLogWriteEntry["source"];
where: Prisma.CycleDayLogWhereUniqueInput;
existing: ExistingRow | null;
encryptSensitive: boolean;
@@ -62,12 +62,43 @@ describe("sweepStaleImportStaging", () => {
expect(
await sweepStaleImportStaging(dir, now, {
paths: new Set([join(dir, queued)]),
xmlInUse: true,
extractedInUse: true,
}),
).toBe(1);
expect(readdirSync(dir).sort()).toEqual([queued, xml].sort());
});
it("removes a Health Connect upload and its extracted database once they are old", async () => {
const dir = mkdtempSync(join(tmpdir(), "healthlog-sweep-"));
const uuid = "0b7f3c2a-1d4e-4f5a-9b8c-7d6e5f4a3b2c";
const upload = `healthlog-health-connect-import-${uuid}.bin`;
const db = `healthlog-hc-import-${"ef".repeat(12)}.db`;
const foreign = "health_connect_export.db";
const now = Date.now();
const old = new Date(now - STAGING_MAX_AGE_MS - 60_000);
for (const name of [upload, db, foreign]) {
writeFileSync(join(dir, name), "x");
utimesSync(join(dir, name), old, old);
}
expect(await sweepStaleImportStaging(dir, now)).toBe(2);
expect(readdirSync(dir)).toEqual([foreign]);
});
it("keeps an extracted database while an import is reading one", async () => {
const dir = mkdtempSync(join(tmpdir(), "healthlog-sweep-"));
const db = `healthlog-hc-import-${"ab".repeat(12)}.db`;
const now = Date.now();
const old = new Date(now - STAGING_MAX_AGE_MS - 60_000);
writeFileSync(join(dir, db), "x");
utimesSync(join(dir, db), old, old);
expect(
await sweepStaleImportStaging(dir, now, {
paths: new Set(),
extractedInUse: true,
}),
).toBe(0);
});
it("answers zero for a directory that is not there", async () => {
expect(await sweepStaleImportStaging("/nonexistent/healthlog")).toBe(0);
});
@@ -0,0 +1,94 @@
/**
* Pulling `health_connect_export.db` out of an uploaded ZIP: the member is
* found by name, a forged header (a size above the cap, a bomb ratio) is
* refused before anything is written, and the streamed output is held to the
* cap whatever the header claims.
*/
import {
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
statSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
buildHealthConnectFixture,
zipHealthConnectDb,
} from "../../../../tests/fixtures/health-connect/build-fixture";
import {
extractHealthConnectDb,
selectHealthConnectDbEntry,
} from "../unzip-export-xml";
function archive(options: Parameters<typeof zipHealthConnectDb>[2] = {}) {
const dir = mkdtempSync(join(tmpdir(), "hc-unzip-"));
const db = join(dir, "source.db");
buildHealthConnectFixture(db, { days: 2 });
const zip = join(dir, "Health Connect.zip");
zipHealthConnectDb(db, zip, options);
return { db, zip };
}
const extracted = () =>
readdirSync(tmpdir()).filter((n) =>
/^healthlog-hc-import-[0-9a-f]{24}\.db$/.test(n),
);
describe("extractHealthConnectDb", () => {
it("writes the database member byte for byte", async () => {
const { db, zip } = archive();
const out = await extractHealthConnectDb(zip);
try {
expect(out.dbBytes).toBe(statSync(db).size);
expect(readFileSync(out.dbPath).equals(readFileSync(db))).toBe(true);
} finally {
rmSync(out.dbPath, { force: true });
}
});
it("refuses an archive without the database member", async () => {
const { zip } = archive({ memberName: "export.xml" });
await expect(extractHealthConnectDb(zip)).rejects.toThrow(
/^not_health_connect:/,
);
});
it("refuses a member that declares more than the cap", async () => {
const { zip } = archive();
await expect(
extractHealthConnectDb(zip, { maxBytes: 1024 }),
).rejects.toThrow(/^too_large:/);
});
it("refuses a header that claims a zip-bomb ratio", async () => {
const { zip } = archive({ declaredSize: 0xfffffff0 });
await expect(
extractHealthConnectDb(zip, { maxBytes: 0xffffffff }),
).rejects.toThrow(/compression ratio/);
});
it("holds the real output to the cap when the header understates it, and leaves nothing behind", async () => {
const before = new Set(extracted());
const { zip } = archive({ declaredSize: 1000 });
await expect(
extractHealthConnectDb(zip, { maxBytes: 2048 }),
).rejects.toThrow();
expect(extracted().filter((n) => !before.has(n))).toEqual([]);
});
});
describe("selectHealthConnectDbEntry", () => {
it("finds the member in a folder and ignores macOS metadata", () => {
const entries = [
{ fileName: "__MACOSX/health_connect_export.db" },
{ fileName: "Health Connect/health_connect_export.db" },
];
expect(selectHealthConnectDbEntry(entries)).toEqual(entries[1]);
expect(selectHealthConnectDbEntry([{ fileName: "export.xml" }])).toBeNull();
});
});
+27 -10
View File
@@ -1,6 +1,7 @@
/**
* The staged Apple Health upload, from the request that writes it to the job
* that consumes it, and what happens to it on every way out.
* The staged import upload (an Apple Health `export.zip` or, since v1.42, a
* Health Connect export ZIP), from the request that writes it to the job that
* consumes it, and what happens to it on every way out.
*
* The upload is the person's whole health export in plain text, written to
* the shared temp directory by the kick-off request and read by the import
@@ -40,13 +41,28 @@ const ACTIVE_IMPORT_MAX_AGE_MS = 7 * 60 * 60 * 1000;
/** Staged files older than this belong to no running import. */
export const STAGING_MAX_AGE_MS = ACTIVE_IMPORT_MAX_AGE_MS;
/** The names the kick-off and the extractor give their temp files. */
/**
* The names the kick-offs and the extractors give their temp files: the
* uploads, the extracted Apple Health XML and the extracted Health Connect
* database.
*/
const STAGING_NAME =
/^healthlog-(?:apple-health-import|admin-apple-health-import|upload)-[0-9a-f-]{36}\.bin$|^healthlog-import-[0-9a-f]{24}\.xml$/;
/^healthlog-(?:apple-health-import|admin-apple-health-import|health-connect-import|upload)-[0-9a-f-]{36}\.bin$|^healthlog-import-[0-9a-f]{24}\.xml$|^healthlog-hc-import-[0-9a-f]{24}\.db$/;
/** The error code a second concurrent import is refused with. */
/** An extracted file (as opposed to an upload) a running import reads. */
const EXTRACTED_NAME = /\.(?:xml|db)$/;
/**
* The error code a second concurrent import is refused with. The name keeps
* its first importer's spelling because clients key on it; it covers every
* kind of import.
*/
export const IMPORT_BUSY_CODE = "import.apple_health.busy";
/** The message that goes with {@link IMPORT_BUSY_CODE}, whichever import runs. */
export const IMPORT_BUSY_MESSAGE =
"An import is already running for this account. Wait for it to finish, then upload again.";
export type CreateImportJobResult =
{ created: { id: string } } | { busy: { id: string; status: string } };
@@ -83,15 +99,16 @@ export async function discardStagedUpload(path: string): Promise<void> {
/**
* What a queued or running import still owns: the staged uploads its queued
* jobs name, and whether an import is extracting or reading its XML now.
* jobs name, and whether an import is extracting or reading an extracted file
* (the Apple Health XML or the Health Connect database) now.
*/
export interface StagingInUse {
paths: ReadonlySet<string>;
xmlInUse: boolean;
extractedInUse: boolean;
}
/**
* Remove staged uploads and extracted XML files older than
* Remove staged uploads and extracted files older than
* `STAGING_MAX_AGE_MS` from `dir`, except what `inUse` names. Age alone is
* not enough: an upload can wait in the queue behind another account's
* import for longer than that, and removing it failed the import when its
@@ -101,7 +118,7 @@ export interface StagingInUse {
export async function sweepStaleImportStaging(
dir: string = tmpdir(),
now: number = Date.now(),
inUse: StagingInUse = { paths: new Set(), xmlInUse: false },
inUse: StagingInUse = { paths: new Set(), extractedInUse: false },
): Promise<number> {
let names: string[];
try {
@@ -114,7 +131,7 @@ export async function sweepStaleImportStaging(
if (!STAGING_NAME.test(name)) continue;
const path = join(dir, name);
if (inUse.paths.has(path)) continue;
if (inUse.xmlInUse && name.endsWith(".xml")) continue;
if (inUse.extractedInUse && EXTRACTED_NAME.test(name)) continue;
try {
const info = await stat(path);
if (!info.isFile()) continue;
@@ -0,0 +1,191 @@
/**
* Health Connect records onto HealthLog rows: units (grams, small calories,
* mmol/L), blood pressure as two rows, the TEXT enum columns, sport types,
* device classes, identities and the per-category app ranking.
*/
import { describe, expect, it } from "vitest";
import {
AppRanking,
DAILY_SUM_SPECS,
INSTANT_SPECS,
NUTRITION_COLUMNS,
dayKeyFromEpochDay,
deviceClassFromHc,
glucoseContextFromRelation,
hcDailyStatsExternalId,
hcExternalId,
hcHourlyPulseExternalId,
mapHealthConnectSportType,
} from "../mapper";
const spec = (table: string) => {
const found = INSTANT_SPECS.find((s) => s.table === table);
if (!found) throw new Error(table);
return found;
};
describe("instant readings", () => {
it("weight and lean mass arrive in grams and are stored in kilograms", () => {
expect(spec("weight_record_table").map({ weight: 80_250 })).toEqual([
{ type: "WEIGHT", value: 80.25 },
]);
expect(spec("lean_body_mass_record_table").map({ mass: 61_000 })).toEqual([
{ type: "LEAN_BODY_MASS", value: 61 },
]);
});
it("blood pressure becomes a systolic and a diastolic row, never half of one", () => {
// The location and body position are TEXT columns holding '3' and '2';
// they must not get in the way of the reading.
expect(
spec("blood_pressure_record_table").map({
systolic: 128,
diastolic: 84,
measurement_location: "3",
body_position: "2",
}),
).toEqual([
{ type: "BLOOD_PRESSURE_SYS", value: 128 },
{ type: "BLOOD_PRESSURE_DIA", value: 84 },
]);
expect(
spec("blood_pressure_record_table").map({
systolic: 128,
diastolic: null,
}),
).toEqual([]);
});
it("glucose arrives in mmol/L and is stored in mg/dL with its meal relation", () => {
expect(
spec("blood_glucose_record_table").map({
level: 5.5,
relation_to_meal: "2",
}),
).toEqual([
{ type: "BLOOD_GLUCOSE", value: 99, glucoseContext: "FASTING" },
]);
expect(glucoseContextFromRelation(4)).toBe("POSTPRANDIAL");
expect(glucoseContextFromRelation(3)).toBeNull();
expect(glucoseContextFromRelation(null)).toBeNull();
});
it("vital signs pass through in their own units", () => {
expect(
spec("heart_rate_variability_rmssd_record_table").map({
heart_rate_variability_millis: 42.5,
}),
).toEqual([{ type: "HRV_RMSSD", value: 42.5 }]);
expect(
spec("oxygen_saturation_record_table").map({ percentage: 96 }),
).toEqual([{ type: "OXYGEN_SATURATION", value: 96 }]);
expect(spec("respiratory_rate_record_table").map({ rate: 14.5 })).toEqual([
{ type: "RESPIRATORY_RATE", value: 14.5 },
]);
expect(
spec("body_temperature_record_table").map({ temperature: 36.6 }),
).toEqual([{ type: "BODY_TEMPERATURE", value: 36.6 }]);
expect(
spec("vo2_max_record_table").map({
vo2_milliliters_per_minute_kilogram: 44.2,
}),
).toEqual([{ type: "VO2_MAX", value: 44.2 }]);
expect(
spec("resting_heart_rate_record_table").map({ beats_per_minute: 58 }),
).toEqual([{ type: "RESTING_HEART_RATE", value: 58 }]);
expect(spec("body_fat_record_table").map({ percentage: 21.5 })).toEqual([
{ type: "BODY_FAT", value: 21.5 },
]);
});
it("drops a record whose value column is not a number", () => {
expect(spec("weight_record_table").map({ weight: null })).toEqual([]);
expect(spec("weight_record_table").map({ weight: "heavy" })).toEqual([]);
});
});
describe("day totals and nutrients", () => {
it("active energy is stored in kcal, from small calories", () => {
const energy = DAILY_SUM_SPECS.find(
(s) => s.type === "ACTIVE_ENERGY_BURNED",
);
expect(320_000 * energy!.factor).toBe(320);
});
it("micronutrients arrive in grams and land in the catalog's unit", () => {
expect(0.06 * NUTRITION_COLUMNS.vitamin_c!.factor).toBeCloseTo(60);
expect(0.00001 * NUTRITION_COLUMNS.vitamin_d!.factor).toBeCloseTo(10);
// Energy and macros are not part of the catalog.
expect(Object.keys(NUTRITION_COLUMNS)).not.toContain("protein");
expect(Object.keys(NUTRITION_COLUMNS)).not.toContain("energy");
});
});
describe("identities", () => {
it("keys records by their UUID and derived rows by UUID and instant", () => {
const uuid = "01234567-89ab-cdef-0123-456789abcdef";
expect(hcExternalId(uuid)).toBe(`hc:${uuid}`);
expect(hcExternalId(uuid, 1_700_000_000_000)).toBe(
`hc:${uuid}:1700000000000`,
);
});
it("uses the stats: family for day totals and hourly means", () => {
expect(hcDailyStatsExternalId("ACTIVITY_STEPS", "2026-09-01")).toBe(
"stats:HKQuantityTypeIdentifierStepCount:2026-09-01",
);
expect(hcHourlyPulseExternalId("2026-09-01", 7)).toBe(
"stats:HKQuantityTypeIdentifierHeartRate:2026-09-01T07",
);
});
it("turns local_date into a calendar day", () => {
expect(dayKeyFromEpochDay(0)).toBe("1970-01-01");
expect(dayKeyFromEpochDay(20_697)).toBe("2026-09-01");
});
});
describe("sports and devices", () => {
it("maps exercise types and falls back to other", () => {
expect(mapHealthConnectSportType(33)).toBe("running");
expect(mapHealthConnectSportType(79)).toBe("other");
expect(mapHealthConnectSportType(null)).toBe("other");
});
it("maps device types onto the picker's classes", () => {
expect(deviceClassFromHc(1)).toBe("watch");
expect(deviceClassFromHc(3)).toBe("scale");
expect(deviceClassFromHc(7)).toBe("other");
expect(deviceClassFromHc(null)).toBeNull();
});
});
describe("AppRanking", () => {
const ranking = new AppRanking([{ category: 1, order: "3,2" }]);
it("ranks listed apps in order and unlisted ones after them", () => {
expect(ranking.rank(1, 3)).toBeLessThan(ranking.rank(1, 2));
expect(ranking.rank(1, 2)).toBeLessThan(ranking.rank(1, 9));
expect(ranking.better(1, 2, 3)).toBe(3);
expect(ranking.better(5, 7, 4)).toBe(4);
});
it("takes a day's total from the listed app even when another counted more", () => {
expect(
ranking.pickDaily(1, [
{ appId: 2, total: 9000 },
{ appId: 3, total: 7000 },
]),
).toEqual({ appId: 3, total: 7000 });
});
it("takes the largest total when no app is listed", () => {
expect(
ranking.pickDaily(4, [
{ appId: 5, total: 300 },
{ appId: 6, total: 900 },
]),
).toEqual({ appId: 6, total: 900 });
});
});
@@ -0,0 +1,194 @@
/**
* Opening an uploaded Health Connect database. The file is untrusted input:
* a version the importer cannot read, a view standing in for a record table
* and a file that is not a Health Connect database at all are refused before
* a row is read; a schema that drifted (a missing column, an extra table) is
* read as far as it goes, and a database last written in WAL mode opens in a
* directory the importer cannot write to.
*/
import {
chmodSync,
copyFileSync,
mkdirSync,
mkdtempSync,
readdirSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { afterEach, describe, expect, it } from "vitest";
import { buildHealthConnectFixture } from "../../../../../tests/fixtures/health-connect/build-fixture";
import {
HealthConnectImportError,
hcInt,
hcUuid,
openHealthConnectDb,
} from "../reader";
const dirs: string[] = [];
function fixture(
options: Parameters<typeof buildHealthConnectFixture>[1] = {},
) {
const dir = mkdtempSync(join(tmpdir(), "hc-reader-"));
dirs.push(dir);
const path = join(dir, "health_connect_export.db");
buildHealthConnectFixture(path, options);
return { dir, path };
}
afterEach(() => {
for (const dir of dirs.splice(0)) {
try {
chmodSync(dir, 0o755);
} catch {
// gone already
}
}
});
function refusal(fn: () => unknown): HealthConnectImportError {
try {
fn();
} catch (err) {
if (err instanceof HealthConnectImportError) return err;
throw err;
}
throw new Error("expected a refusal");
}
describe("openHealthConnectDb", () => {
it("opens the fixture read-only and finds its tables by name", () => {
const { path } = fixture();
const hc = openHealthConnectDb(path);
try {
expect(hc.userVersion).toBe(20);
expect(hc.warnings).toEqual([]);
expect(hc.usable("weight_record_table", ["uuid", "time", "weight"])).toBe(
true,
);
expect(hc.columns("weight_record_table")?.has("weight")).toBe(true);
// query_only: the importer cannot write to the upload even by mistake.
expect(() => hc.all("DELETE FROM weight_record_table")).toThrow();
} finally {
hc.close();
}
});
it("refuses a database older than schema version 9", () => {
const { path } = fixture({ userVersion: 8 });
expect(refusal(() => openHealthConnectDb(path)).code).toBe(
"unsupported_version",
);
});
it("reads a newer schema version than it knows, with a warning", () => {
const { path } = fixture({ userVersion: 31 });
const hc = openHealthConnectDb(path);
expect(hc.warnings).toContain("user_version_newer:31");
hc.close();
});
it("refuses a view standing in for a record table", () => {
const { path } = fixture();
const db = new DatabaseSync(path);
db.exec(
"DROP TABLE weight_record_table; CREATE VIEW weight_record_table AS SELECT 1 AS uuid, 2 AS time, 3 AS weight, 4 AS app_info_id",
);
db.close();
const err = refusal(() => openHealthConnectDb(path));
expect(err.code).toBe("unsafe_schema");
expect(err.message).toContain("weight_record_table is a view");
});
it("skips a table whose required column is missing and keeps the rest", () => {
const { path } = fixture();
const db = new DatabaseSync(path);
db.exec("ALTER TABLE body_fat_record_table DROP COLUMN percentage");
db.exec(
"CREATE TABLE some_future_record_table (row_id INTEGER, uuid BLOB)",
);
db.close();
const hc = openHealthConnectDb(path);
try {
expect(hc.usable("body_fat_record_table", ["uuid", "percentage"])).toBe(
false,
);
expect(hc.usable("weight_record_table", ["uuid", "weight"])).toBe(true);
expect(hc.warnings).toEqual([
"column_missing:body_fat_record_table.percentage",
]);
} finally {
hc.close();
}
});
it("refuses a database without the application table", () => {
const { path } = fixture();
const db = new DatabaseSync(path);
db.exec("PRAGMA foreign_keys = OFF; DROP TABLE application_info_table");
db.close();
expect(refusal(() => openHealthConnectDb(path)).code).toBe(
"not_health_connect",
);
});
it("refuses a file that is not a SQLite database", () => {
const dir = mkdtempSync(join(tmpdir(), "hc-reader-"));
const path = join(dir, "health_connect_export.db");
writeFileSync(path, "PK this is not a database");
expect(refusal(() => openHealthConnectDb(path)).code).toBe(
"not_health_connect",
);
});
it("opens a WAL-mode database in a read-only directory and leaves no companion files", () => {
const { dir, path } = fixture({ wal: true });
const ro = join(dir, "ro");
mkdirSync(ro);
copyFileSync(path, join(ro, "health_connect_export.db"));
chmodSync(ro, 0o555);
const hc = openHealthConnectDb(join(ro, "health_connect_export.db"));
try {
const [row] = hc.all("SELECT COUNT(*) AS n FROM weight_record_table");
expect(Number(row.n)).toBeGreaterThan(0);
} finally {
hc.close();
}
expect(readdirSync(ro)).toEqual(["health_connect_export.db"]);
});
it("hands rows out in batches of at most the batch size", () => {
const { path } = fixture({ days: 3 });
const hc = openHealthConnectDb(path);
try {
const sizes = [
...hc.batches("SELECT * FROM steps_record_table", [], 40),
].map((b) => b.length);
expect(sizes.every((n) => n <= 40)).toBe(true);
expect(sizes.reduce((a, b) => a + b, 0)).toBe(3 * 16 * 2);
} finally {
hc.close();
}
});
});
describe("column helpers", () => {
it("reads an enum stored in a TEXT column as its number", () => {
expect(hcInt("3")).toBe(3);
expect(hcInt(3)).toBe(3);
expect(hcInt("left arm")).toBeNull();
expect(hcInt(2.5)).toBeNull();
});
it("spells a 16-byte blob as an 8-4-4-4-12 UUID", () => {
const blob = Buffer.from("0123456789abcdef0123456789abcdef", "hex");
expect(hcUuid(new Uint8Array(blob))).toBe(
"01234567-89ab-cdef-0123-456789abcdef",
);
expect(hcUuid(new Uint8Array(15))).toBeNull();
expect(hcUuid("01234567")).toBeNull();
});
});
@@ -0,0 +1,56 @@
import { describe, expect, it } from "vitest";
import { keepNonOverlapping } from "../import";
import { mapHealthConnectSleepStage } from "../sleep-stage-map";
describe("mapHealthConnectSleepStage", () => {
it("maps the classified stages", () => {
expect(mapHealthConnectSleepStage(1, true)).toEqual({ stage: "AWAKE" });
expect(mapHealthConnectSleepStage(2, true)).toEqual({ stage: "ASLEEP" });
expect(mapHealthConnectSleepStage(4, true)).toEqual({ stage: "CORE" });
expect(mapHealthConnectSleepStage(5, true)).toEqual({ stage: "DEEP" });
expect(mapHealthConnectSleepStage(6, true)).toEqual({ stage: "REM" });
expect(mapHealthConnectSleepStage(7, true)).toEqual({ stage: "AWAKE" });
});
it("leaves out time out of bed", () => {
expect(mapHealthConnectSleepStage(3, true)).toEqual({ skip: "out_of_bed" });
expect(mapHealthConnectSleepStage(3, false)).toEqual({
skip: "out_of_bed",
});
});
it("counts an unknown stage as sleep only in a session without real stages", () => {
expect(mapHealthConnectSleepStage(0, false)).toEqual({ stage: "ASLEEP" });
expect(mapHealthConnectSleepStage(0, true)).toEqual({
skip: "unknown_stage",
});
});
it("leaves out a stage number it does not know", () => {
expect(mapHealthConnectSleepStage(42, true)).toEqual({
skip: "unmapped_stage",
});
});
});
describe("keepNonOverlapping", () => {
it("keeps the better-ranked of two overlapping sessions and every lone one", () => {
const sessions = [
{ id: "a", start: 0, end: 100, rank: 1 },
{ id: "b", start: 50, end: 150, rank: 0 },
{ id: "c", start: 200, end: 300, rank: 5 },
];
expect(
keepNonOverlapping(sessions, (s) => s.rank).map((s) => s.id),
).toEqual(["b", "c"]);
});
it("treats sessions that only touch as separate", () => {
const sessions = [
{ start: 0, end: 100, rank: 0 },
{ start: 100, end: 200, rank: 1 },
];
expect(keepNonOverlapping(sessions, (s) => s.rank)).toHaveLength(2);
});
});
File diff suppressed because it is too large. Load diff
+484
View File
@@ -0,0 +1,484 @@
/**
* Health Connect records onto HealthLog rows: units, identities, sport
* types, device classes and the per-category app ranking.
*
* Units as Health Connect stores them (the `*RecordInternal` classes in the
* AOSP HealthFitness module): mass in grams, energy in small calories (the
* app's own display divides by 1000), volume in litres, length in metres,
* blood glucose in mmol/L, temperature in °C, percentages as 0–100, times as
* epoch milliseconds in UTC with the zone offset in seconds beside them.
*
* Identities. Every record carries a 16-byte UUID that is fixed in the
* database and, for apps that set a client record id, derived from it; it is
* the re-import key, written as `hc:<8-4-4-4-12>`. Rows that come from one
* record but are not the record itself extend it: a heart-rate sample is
* `hc:<series uuid>:<epoch ms>`, a sleep stage `hc:<session uuid>:<stage
* start ms>`. Day totals and hourly means use the `stats:` family every other
* aggregate in the table uses, under source `HEALTH_CONNECT`, so they never
* meet an Apple Health row of the same name.
*/
import type {
GlucoseContext,
MeasurementType,
} from "@/generated/prisma/client";
import { mmolToMgdl } from "@/lib/glucose";
import { dailyStatsExternalId } from "@/lib/measurements/apple-health-mapping";
import { hourlyStatsExternalId } from "@/lib/measurements/dense-intraday-retention";
import { getUnitForType } from "@/lib/validations/measurement";
import { dateOnlyKey } from "@/lib/tz/date-only";
import { isStableExternalId } from "@/lib/validations/external-id";
import type { NutrientCode } from "@/lib/nutrients/catalog";
/** Health Connect `HealthDataCategory` values the importer ranks apps by. */
export const HC_CATEGORY = {
ACTIVITY: 1,
BODY_MEASUREMENTS: 2,
CYCLE_TRACKING: 3,
NUTRITION: 4,
SLEEP: 5,
VITALS: 6,
} as const;
export type HcCategory = (typeof HC_CATEGORY)[keyof typeof HC_CATEGORY];
/**
* `hc:<uuid>`, or `hc:<uuid>:<n>` for a row derived from one record. The
* UUID is formatted from the record's 16-byte blob, so it cannot be one of
* the shapes the stability floor refuses; the floor is applied anyway, so a
* future caller that passes something else fails here instead of writing a
* rotating identity.
*/
export function hcExternalId(uuid: string, suffix?: number): string {
const id = suffix === undefined ? `hc:${uuid}` : `hc:${uuid}:${suffix}`;
if (!isStableExternalId(uuid)) {
throw new Error("Health Connect record id is not a stable identity");
}
return id;
}
/** The HealthKit identifier the `stats:` ids of a type are spelled with. */
const STATS_IDENTIFIER: Partial<Record<MeasurementType, string>> = {
ACTIVITY_STEPS: "HKQuantityTypeIdentifierStepCount",
ACTIVE_ENERGY_BURNED: "HKQuantityTypeIdentifierActiveEnergyBurned",
WALKING_RUNNING_DISTANCE: "HKQuantityTypeIdentifierDistanceWalkingRunning",
PULSE: "HKQuantityTypeIdentifierHeartRate",
};
/** `stats:<identifier>:<YYYY-MM-DD>` for a day total. */
export function hcDailyStatsExternalId(
type: MeasurementType,
dayKey: string,
): string {
const identifier = STATS_IDENTIFIER[type];
if (!identifier) throw new Error(`no stats identifier for ${type}`);
return dailyStatsExternalId(identifier, dayKey);
}
/** `stats:<identifier>:<YYYY-MM-DD>T<HH>` for an hourly heart-rate mean. */
export function hcHourlyPulseExternalId(dayKey: string, hour: number): string {
return hourlyStatsExternalId(STATS_IDENTIFIER.PULSE!, dayKey, hour);
}
/**
* `YYYY-MM-DD` of a Health Connect `local_date`: the record's calendar day in
* its own zone, as a count of days since the epoch. A date-only value, so it
* is read as the label of UTC midnight.
*/
export function dayKeyFromEpochDay(epochDay: number): string {
return dateOnlyKey(new Date(epochDay * 86_400_000));
}
/**
* An instant reading table and the HealthLog type(s) one row becomes. The
* value column is in the table's own unit; `toDb` converts it.
*/
export interface InstantSpec {
table:
| "weight_record_table"
| "body_fat_record_table"
| "lean_body_mass_record_table"
| "blood_pressure_record_table"
| "resting_heart_rate_record_table"
| "heart_rate_variability_rmssd_record_table"
| "oxygen_saturation_record_table"
| "respiratory_rate_record_table"
| "blood_glucose_record_table"
| "body_temperature_record_table"
| "vo2_max_record_table";
category: HcCategory;
/** Columns read besides the common ones. */
columns: readonly string[];
/** The rows one record becomes; empty when its values are unusable. */
map(row: Record<string, unknown>): Array<{
type: MeasurementType;
value: number;
glucoseContext?: GlucoseContext | null;
}>;
}
const num = (v: unknown): number | null =>
typeof v === "number" && Number.isFinite(v) ? v : null;
/** One value column, scaled by `factor`. */
function single(
type: MeasurementType,
column: string,
factor = 1,
): Pick<InstantSpec, "columns" | "map"> {
return {
columns: [column],
map(row) {
const v = num(row[column]);
return v === null ? [] : [{ type, value: v * factor }];
},
};
}
/**
* `relation_to_meal` (a TEXT column holding an int) onto the context HealthLog
* records: fasting and after a meal have a counterpart, the rest do not.
*/
export function glucoseContextFromRelation(
relation: number | null,
): GlucoseContext | null {
if (relation === 2) return "FASTING";
if (relation === 4) return "POSTPRANDIAL";
return null;
}
function textInt(v: unknown): number | null {
if (typeof v === "number" && Number.isInteger(v)) return v;
if (typeof v === "string" && /^\d+$/.test(v.trim())) return Number(v.trim());
return null;
}
export const INSTANT_SPECS: readonly InstantSpec[] = [
{
table: "weight_record_table",
category: HC_CATEGORY.BODY_MEASUREMENTS,
...single("WEIGHT", "weight", 1 / 1000),
},
{
table: "body_fat_record_table",
category: HC_CATEGORY.BODY_MEASUREMENTS,
...single("BODY_FAT", "percentage"),
},
{
table: "lean_body_mass_record_table",
category: HC_CATEGORY.BODY_MEASUREMENTS,
...single("LEAN_BODY_MASS", "mass", 1 / 1000),
},
{
table: "blood_pressure_record_table",
category: HC_CATEGORY.VITALS,
columns: ["systolic", "diastolic"],
map(row) {
const sys = num(row.systolic);
const dia = num(row.diastolic);
// A reading is the pair; half of one is not stored.
if (sys === null || dia === null) return [];
return [
{ type: "BLOOD_PRESSURE_SYS", value: sys },
{ type: "BLOOD_PRESSURE_DIA", value: dia },
];
},
},
{
table: "resting_heart_rate_record_table",
category: HC_CATEGORY.VITALS,
...single("RESTING_HEART_RATE", "beats_per_minute"),
},
{
table: "heart_rate_variability_rmssd_record_table",
category: HC_CATEGORY.VITALS,
...single("HRV_RMSSD", "heart_rate_variability_millis"),
},
{
table: "oxygen_saturation_record_table",
category: HC_CATEGORY.VITALS,
...single("OXYGEN_SATURATION", "percentage"),
},
{
table: "respiratory_rate_record_table",
category: HC_CATEGORY.VITALS,
...single("RESPIRATORY_RATE", "rate"),
},
{
table: "blood_glucose_record_table",
category: HC_CATEGORY.VITALS,
columns: ["level"],
map(row) {
const level = num(row.level);
if (level === null) return [];
return [
{
type: "BLOOD_GLUCOSE",
// mmol/L to the canonical mg/dL, the same conversion every other
// writer of a mmol/L reading uses.
value: mmolToMgdl(level),
glucoseContext: glucoseContextFromRelation(
textInt(row.relation_to_meal),
),
},
];
},
},
{
table: "body_temperature_record_table",
category: HC_CATEGORY.VITALS,
...single("BODY_TEMPERATURE", "temperature"),
},
{
table: "vo2_max_record_table",
category: HC_CATEGORY.ACTIVITY,
...single("VO2_MAX", "vo2_milliliters_per_minute_kilogram"),
},
];
/** Optional columns an instant spec reads when the table has them. */
export const INSTANT_OPTIONAL_COLUMNS: Readonly<
Partial<Record<InstantSpec["table"], readonly string[]>>
> = {
blood_glucose_record_table: ["relation_to_meal"],
};
/** An interval table summed per day. */
export interface DailySumSpec {
table:
| "steps_record_table"
| "active_calories_burned_record_table"
| "distance_record_table";
column: string;
type: MeasurementType;
/** Multiplies the summed column into the type's database unit. */
factor: number;
}
export const DAILY_SUM_SPECS: readonly DailySumSpec[] = [
{
table: "steps_record_table",
column: "count",
type: "ACTIVITY_STEPS",
factor: 1,
},
{
// Small calories to kcal.
table: "active_calories_burned_record_table",
column: "energy",
type: "ACTIVE_ENERGY_BURNED",
factor: 1 / 1000,
},
{
table: "distance_record_table",
column: "distance",
type: "WALKING_RUNNING_DISTANCE",
factor: 1,
},
];
/**
* Nutrition columns (grams) onto the nutrient catalog. Energy and the
* macronutrients are left out: the catalog is supplements and water, never a
* food diary.
*/
export const NUTRITION_COLUMNS: Readonly<
Partial<Record<NutrientCode, { column: string; factor: number }>>
> = (() => {
const mg = 1000;
const ug = 1_000_000;
const codes: Array<[NutrientCode, number]> = [
["vitamin_a", ug],
["thiamin", mg],
["riboflavin", mg],
["niacin", mg],
["pantothenic_acid", mg],
["vitamin_b6", mg],
["biotin", ug],
["folate", ug],
["vitamin_b12", ug],
["vitamin_c", mg],
["vitamin_d", ug],
["vitamin_e", mg],
["vitamin_k", ug],
["calcium", mg],
["iron", mg],
["magnesium", mg],
["phosphorus", mg],
["zinc", mg],
["copper", mg],
["manganese", mg],
["selenium", ug],
["chromium", ug],
["molybdenum", ug],
["iodine", ug],
["caffeine", mg],
];
return Object.fromEntries(
codes.map(([code, factor]) => [code, { column: code, factor }]),
);
})();
/** Litres of `hydration_record_table.volume` to the catalog's millilitres. */
export const HYDRATION_ML_PER_LITRE = 1000;
/** The database unit of `type`. */
export function hcUnit(type: MeasurementType): string {
return getUnitForType(type);
}
/**
* Health Connect `Device.type` onto the device classes the source picker
* knows (`watch | band | ring | phone | scale | other | unknown`).
*/
export function deviceClassFromHc(deviceType: number | null): string | null {
switch (deviceType) {
case null:
return null;
case 0:
return "unknown";
case 1:
return "watch";
case 2:
return "phone";
case 3:
return "scale";
case 4:
return "ring";
case 6:
return "band";
default:
return "other";
}
}
/**
* `ExerciseSessionType` onto the canonical sport set. Types with no
* counterpart land on `other`; the raw number is kept on the workout's
* metadata (`healthConnectExerciseType`) so the mapping stays reversible.
*/
const EXERCISE_TYPE_MAP: Readonly<Record<number, string>> = {
1: "badminton",
3: "basketball",
4: "cycling",
5: "cycling",
6: "crossTraining",
8: "strength",
10: "dance",
11: "mixedCardio",
16: "golf",
17: "mindAndBody",
20: "hiit",
21: "hiking",
27: "mindAndBody",
31: "rowing",
33: "running",
34: "running",
41: "soccer",
44: "stairClimber",
45: "strength",
46: "mindAndBody",
48: "swimming",
49: "swimming",
51: "tennis",
53: "walking",
55: "strength",
57: "yoga",
58: "other",
59: "stairClimber",
60: "elliptical",
61: "rowing",
};
/** Resolve a Health Connect exercise type to a canonical sport label. */
export function mapHealthConnectSportType(exerciseType: unknown): string {
if (typeof exerciseType !== "number" || !Number.isInteger(exerciseType)) {
return "other";
}
return EXERCISE_TYPE_MAP[exerciseType] ?? "other";
}
/**
* Health Connect's own app ranking, per data category: the user's order in
* the Health Connect settings ("app priority"), stored as a comma-separated
* list of application row ids. Lower is better; an app the list does not
* name ranks after every app it does.
*/
export class AppRanking {
private readonly byCategory = new Map<number, Map<number, number>>();
constructor(rows: ReadonlyArray<{ category: unknown; order: unknown }>) {
for (const row of rows) {
const category = typeof row.category === "number" ? row.category : NaN;
if (!Number.isInteger(category) || typeof row.order !== "string") {
continue;
}
const ranks = new Map<number, number>();
row.order
.split(",")
.map((part) => Number(part.trim()))
.filter((id) => Number.isInteger(id))
.forEach((id, index) => {
if (!ranks.has(id)) ranks.set(id, index);
});
this.byCategory.set(category, ranks);
}
}
/** The app's rank in `category`; unlisted apps share the last place. */
rank(category: number, appId: number): number {
return this.byCategory.get(category)?.get(appId) ?? 1_000_000;
}
/** Whether the category has an explicit order naming `appId`. */
lists(category: number, appId: number): boolean {
return this.byCategory.get(category)?.has(appId) ?? false;
}
/**
* The better of two apps: the lower rank, then (both unlisted, or a tie)
* the lower application row id, so the choice is deterministic.
*/
better(category: number, a: number, b: number): number {
const ra = this.rank(category, a);
const rb = this.rank(category, b);
if (ra !== rb) return ra < rb ? a : b;
return Math.min(a, b);
}
/**
* Pick one app's total for a day. An app the priority list names wins over
* any it does not; among unlisted apps the largest total wins, the way the
* Apple Health importer picks a day's source.
*/
pickDaily<T extends { appId: number; total: number }>(
category: number,
candidates: readonly T[],
): T | null {
let best: T | null = null;
for (const c of candidates) {
if (!best) {
best = c;
continue;
}
const listedC = this.lists(category, c.appId);
const listedB = this.lists(category, best.appId);
if (listedC !== listedB) {
if (listedC) best = c;
continue;
}
if (listedC) {
if (this.rank(category, c.appId) < this.rank(category, best.appId)) {
best = c;
}
continue;
}
if (
c.total > best.total ||
(c.total === best.total && c.appId < best.appId)
) {
best = c;
}
}
return best;
}
}
+300
View File
@@ -0,0 +1,300 @@
/**
* Read-only access to a Health Connect export database
* (`health_connect_export.db`, the file the Android Health Connect app copies
* into its export ZIP).
*
* The file comes from an upload, so it is opened as untrusted input:
*
* - `immutable=1` in the URI. The export is a byte copy of the live
* database, and a database last written in WAL mode refuses a plain
* read-only open in a directory it cannot create the `-wal` / `-shm`
* companions in. Immutable tells SQLite the file cannot change, so it
* neither looks for nor creates them, and takes no locks.
* - `trusted_schema=OFF`, so a view, trigger or generated column in a
* crafted file cannot call a function with side effects; `query_only`,
* `cell_size_check` and no memory map on top.
* - Every table the importer reads must be a real table. A view under a
* record table's name is refused outright rather than read: it would
* let the file run its own query in place of the one written here.
* - Columns are found by name (`PRAGMA table_info`), never by position.
* The platform adds columns with `ALTER TABLE` as its schema version
* moves, so their order differs between devices.
* - `PRAGMA user_version` below 9 (the first version with 16-byte UUID
* blobs) is refused; above 27 (the newest the AOSP source defines) is
* read with a warning, because new versions have so far only added
* columns and tables.
*
* Nothing here holds a whole table in memory. Statements are walked with
* `iterate()` and handed out in batches; the aggregations (daily sums, hourly
* means) run inside SQLite.
*/
import type { DatabaseSync, SQLInputValue } from "node:sqlite";
import { pathToFileURL } from "node:url";
/** First schema version with 16-byte UUID blobs (`DatabaseVersions.MIN_SUPPORTED`). */
export const HC_MIN_USER_VERSION = 9;
/** Newest schema version the AOSP source defines at the time of writing. */
export const HC_MAX_KNOWN_USER_VERSION = 27;
/** Rows handed out per batch. */
export const HC_BATCH_SIZE = 500;
/** Machine-readable reasons an export is refused before anything is read. */
export type HealthConnectRefusal =
"unsupported_version" | "not_health_connect" | "unsafe_schema";
/** A refusal the worker turns into the job's failure reason. */
export class HealthConnectImportError extends Error {
readonly code: HealthConnectRefusal;
constructor(code: HealthConnectRefusal, message: string) {
super(`${code}: ${message}`);
this.name = "HealthConnectImportError";
this.code = code;
}
}
/** The tables the importer reads. Anything else in the file is ignored. */
export const HC_TABLES = [
"application_info_table",
"device_info_table",
"health_data_category_priority_table",
"weight_record_table",
"body_fat_record_table",
"lean_body_mass_record_table",
"blood_pressure_record_table",
"resting_heart_rate_record_table",
"heart_rate_variability_rmssd_record_table",
"oxygen_saturation_record_table",
"respiratory_rate_record_table",
"blood_glucose_record_table",
"body_temperature_record_table",
"vo2_max_record_table",
"steps_record_table",
"active_calories_burned_record_table",
"distance_record_table",
"hydration_record_table",
"nutrition_record_table",
"heart_rate_record_table",
"heart_rate_record_series_table",
"sleep_session_record_table",
"sleep_stages_table",
"exercise_session_record_table",
"menstruation_flow_record_table",
"menstruation_period_record_table",
] as const;
export type HcTable = (typeof HC_TABLES)[number];
/** One row as `node:sqlite` returns it. */
export type HcRow = Record<string, unknown>;
/** An opened export. */
export interface HealthConnectDb {
readonly userVersion: number;
/** Codes for what the importer could not read; never row content. */
readonly warnings: string[];
/**
* The columns of `table` when it is present, `null` when it is not. A
* missing table is ordinary (an app that never wrote that type), so it is
* not a warning.
*/
columns(table: HcTable): ReadonlySet<string> | null;
/**
* Whether `table` is present with every column in `required`. A table that
* is present but lacks one is noted in `warnings` once and treated as
* absent, so a schema drift skips that type instead of failing the import.
*/
usable(table: HcTable, required: readonly string[]): boolean;
/** Every row of `sql`, in batches of at most `size`. */
batches(
sql: string,
params?: readonly SQLInputValue[],
size?: number,
): Generator<HcRow[]>;
/** Every row of `sql` at once. For small, bounded results only. */
all(sql: string, params?: readonly SQLInputValue[]): HcRow[];
close(): void;
}
/**
* Open `path` read-only and check that it is a Health Connect export this
* importer can read. Throws {@link HealthConnectImportError} when it is not;
* the database is closed again in that case.
*/
export function openHealthConnectDb(path: string): HealthConnectDb {
const url = pathToFileURL(path);
url.searchParams.set("immutable", "1");
let db: DatabaseSync;
try {
// Loaded on first use rather than at import: the routes that share this
// module graph (upload, status) never open a database, and `node:sqlite`
// prints an experimental-feature warning the moment it loads.
const { DatabaseSync: Database } = process.getBuiltinModule(
"node:sqlite",
) as typeof import("node:sqlite");
db = new Database(url, { readOnly: true });
} catch (err) {
throw new HealthConnectImportError(
"not_health_connect",
`the export database could not be opened (${err instanceof Error ? err.message : String(err)})`,
);
}
try {
// Before the first statement reads the schema.
db.exec(
"PRAGMA trusted_schema = OFF; PRAGMA query_only = ON; PRAGMA cell_size_check = ON; PRAGMA mmap_size = 0;",
);
return inspect(db);
} catch (err) {
db.close();
if (err instanceof HealthConnectImportError) throw err;
throw new HealthConnectImportError(
"not_health_connect",
`the export database could not be read (${err instanceof Error ? err.message : String(err)})`,
);
}
}
function inspect(db: DatabaseSync): HealthConnectDb {
const versionRow = db.prepare("PRAGMA user_version").get() as
{ user_version?: unknown } | undefined;
const userVersion = Number(versionRow?.user_version ?? 0);
if (!Number.isSafeInteger(userVersion) || userVersion < HC_MIN_USER_VERSION) {
throw new HealthConnectImportError(
"unsupported_version",
`database version ${userVersion} is older than ${HC_MIN_USER_VERSION}`,
);
}
const warnings: string[] = [];
if (userVersion > HC_MAX_KNOWN_USER_VERSION) {
warnings.push(`user_version_newer:${userVersion}`);
}
// What each name the importer reads actually is in this file.
const kinds = new Map<string, string>();
const names = new Set<string>(HC_TABLES);
for (const row of db
.prepare(
"SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view')",
)
.all() as Array<{ type: string; name: string }>) {
if (names.has(row.name)) kinds.set(row.name, row.type);
}
for (const [name, type] of kinds) {
if (type !== "table") {
throw new HealthConnectImportError(
"unsafe_schema",
`${name} is a ${type}, not a table`,
);
}
}
if (!kinds.has("application_info_table")) {
throw new HealthConnectImportError(
"not_health_connect",
"the database has no application_info_table",
);
}
const columnCache = new Map<string, ReadonlySet<string>>();
const columns = (table: HcTable): ReadonlySet<string> | null => {
if (!kinds.has(table)) return null;
const cached = columnCache.get(table);
if (cached) return cached;
// `table` is one of the closed HC_TABLES names, never input.
const set = new Set(
(
db.prepare(`PRAGMA table_info(${table})`).all() as Array<{
name: string;
}>
).map((c) => c.name),
);
columnCache.set(table, set);
return set;
};
const appInfo = columns("application_info_table");
if (!appInfo?.has("row_id") || !appInfo.has("package_name")) {
throw new HealthConnectImportError(
"not_health_connect",
"application_info_table lacks row_id or package_name",
);
}
const warned = new Set<string>();
const usable = (table: HcTable, required: readonly string[]): boolean => {
const cols = columns(table);
if (!cols) return false;
const missing = required.filter((c) => !cols.has(c));
if (missing.length === 0) return true;
for (const column of missing) {
const code = `column_missing:${table}.${column}`;
if (!warned.has(code)) {
warned.add(code);
warnings.push(code);
}
}
return false;
};
return {
userVersion,
warnings,
columns,
usable,
*batches(sql, params = [], size = HC_BATCH_SIZE) {
let batch: HcRow[] = [];
for (const row of db.prepare(sql).iterate(...params)) {
batch.push(row as HcRow);
if (batch.length >= size) {
yield batch;
batch = [];
}
}
if (batch.length > 0) yield batch;
},
all(sql, params = []) {
return db.prepare(sql).all(...params) as HcRow[];
},
close() {
try {
db.close();
} catch {
// Already closed.
}
},
};
}
/** The 8-4-4-4-12 string of a 16-byte UUID blob, or null for anything else. */
export function hcUuid(value: unknown): string | null {
if (!(value instanceof Uint8Array) || value.length !== 16) return null;
const hex = Buffer.from(value).toString("hex");
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
}
/**
* A number from a column Health Connect declares as an enum. Several of them
* are TEXT columns the platform writes integers into, so SQLite hands back
* `'3'` rather than `3`; both read as 3. Anything that is not an integer
* reads as null.
*/
export function hcInt(value: unknown): number | null {
if (typeof value === "number") return Number.isInteger(value) ? value : null;
if (typeof value === "bigint") return Number(value);
if (typeof value === "string" && /^-?\d+$/.test(value.trim())) {
return Number(value.trim());
}
return null;
}
/** A finite number, or null. */
export function hcNumber(value: unknown): number | null {
if (typeof value === "number") return Number.isFinite(value) ? value : null;
if (typeof value === "bigint") return Number(value);
if (typeof value === "string" && value.trim() !== "") {
const n = Number(value);
return Number.isFinite(n) ? n : null;
}
return null;
}
@@ -0,0 +1,75 @@
/**
* Apps whose Health Connect data the import leaves out because HealthLog
* already receives the same readings from them directly.
*
* A Withings scale writes each weighing to Health Connect and to the Withings
* cloud. When the account has the Withings integration connected, the
* weighing arrives twice: once as `WITHINGS`, once inside the Health Connect
* export. The cross-source merge only pairs manual entries with Apple Health,
* so nothing would fold the two, and the reader's source ranking would show
* one and keep the other. Leaving the app's records out of the import is the
* exact fix; the ±2 s same-reading check in the importer stays as a safety
* net for anything this list does not name.
*
* The list is per integration, by Android package name. Only integrations the
* account has actually connected count; a person who uses the Fitbit app but
* not HealthLog's Fitbit integration gets their Fitbit data from the export.
*
* Google Health is the Fitbit Web API's successor and carries what the Fitbit
* app records (Fitbit devices, Pixel Watch), so it maps to the Fitbit app's
* package as well.
*/
import type { PrismaClient } from "@/generated/prisma/client";
export type DirectIntegration =
"withings" | "fitbit" | "googleHealth" | "oura" | "polar" | "whoop";
export const INTEGRATION_PACKAGES: Readonly<
Record<DirectIntegration, readonly string[]>
> = {
withings: ["com.withings.wiscale2"],
fitbit: ["com.fitbit.FitbitMobile"],
googleHealth: ["com.fitbit.FitbitMobile"],
oura: ["com.ouraring.oura"],
polar: ["fi.polar.polarflow"],
whoop: ["com.whoop.android"],
};
/** The integrations this account has connected right now. */
export async function connectedDirectIntegrations(
prisma: PrismaClient,
userId: string,
): Promise<DirectIntegration[]> {
const [withings, fitbit, googleHealth, whoop, user] = await Promise.all([
prisma.withingsConnection.count({ where: { userId } }),
prisma.fitbitConnection.count({ where: { userId } }),
prisma.googleHealthConnection.count({ where: { userId } }),
prisma.whoopConnection.count({ where: { userId } }),
prisma.user.findUnique({
where: { id: userId },
select: {
polarAccessTokenEncrypted: true,
ouraAccessTokenEncrypted: true,
},
}),
]);
const out: DirectIntegration[] = [];
if (withings > 0) out.push("withings");
if (fitbit > 0) out.push("fitbit");
if (googleHealth > 0) out.push("googleHealth");
if (whoop > 0) out.push("whoop");
if (user?.polarAccessTokenEncrypted) out.push("polar");
if (user?.ouraAccessTokenEncrypted) out.push("oura");
return out;
}
/** The package names to leave out for `integrations`. */
export function packagesToSkip(
integrations: readonly DirectIntegration[],
): Set<string> {
const out = new Set<string>();
for (const integration of integrations) {
for (const pkg of INTEGRATION_PACKAGES[integration]) out.add(pkg);
}
return out;
}
@@ -0,0 +1,58 @@
/**
* Health Connect sleep stages onto HealthLog's `SleepStage`.
*
* Health Connect (`SleepSessionRecord.StageType`):
* 0 unknown, 1 awake, 2 sleeping, 3 out of bed, 4 light, 5 deep, 6 REM,
* 7 awake in bed.
*
* Light sleep is what Apple calls core sleep, and both "awake" stages are
* awake time inside the session. "Out of bed" is time the person was not in
* bed at all, so it is no part of the night and is left out.
*
* "Unknown" (0) is the stage an app writes when it knows the person was in a
* session but not what kind of sleep. Inside a session that also carries
* real stages it is a gap the app could not classify, and counting it as
* sleep would inflate the night; it is left out there. In a session made of
* nothing but unknown stages it is the only record of the night, and it is
* counted as plain sleep (`ASLEEP`), the same way a session without any
* stages is.
*/
import type { SleepStage } from "@/generated/prisma/client";
export const HC_STAGE_UNKNOWN = 0;
export const HC_STAGE_OUT_OF_BED = 3;
const KNOWN_STAGES: Readonly<Record<number, SleepStage>> = {
1: "AWAKE",
2: "ASLEEP",
4: "CORE",
5: "DEEP",
6: "REM",
7: "AWAKE",
};
/** Why a stage was not mapped, for the result's skip counters. */
export type SleepStageSkip = "unknown_stage" | "out_of_bed" | "unmapped_stage";
/**
* Map one stage. `sessionHasKnownStages` is whether the stage's session
* carries at least one stage other than unknown and out of bed.
*/
export function mapHealthConnectSleepStage(
stageType: number,
sessionHasKnownStages: boolean,
): { stage: SleepStage } | { skip: SleepStageSkip } {
if (stageType === HC_STAGE_OUT_OF_BED) return { skip: "out_of_bed" };
if (stageType === HC_STAGE_UNKNOWN) {
return sessionHasKnownStages
? { skip: "unknown_stage" }
: { stage: "ASLEEP" };
}
const stage = KNOWN_STAGES[stageType];
return stage ? { stage } : { skip: "unmapped_stage" };
}
/** Whether `stageType` is a classified stage (not unknown, not out of bed). */
export function isKnownSleepStage(stageType: number): boolean {
return stageType in KNOWN_STAGES;
}
+108 -1
View File
@@ -32,6 +32,10 @@
* Peak memory is bounded by stream buffer sizes and the (small)
* central directory, independent of archive size.
*
* v1.42 (#972): the same walker pulls `health_connect_export.db` out of an
* Android Health Connect export (`extractHealthConnectDb`), under its own,
* lower size ceiling.
*
* Locks per `.planning/research/v1434-r-1-xml-import.md` §6.1.
*/
import {
@@ -317,6 +321,108 @@ export async function extractExportXml(
}
}
/** The one member of a Health Connect export ZIP the importer reads. */
export const HEALTH_CONNECT_DB_MEMBER = "health_connect_export.db";
/**
* Ceiling on the extracted Health Connect database. The export is a byte copy
* of the phone's own database; years of minute-by-minute heart rate from a
* watch put it in the hundreds of megabytes, and 4 GiB leaves room for that
* while keeping a forged archive from filling the temp directory.
*/
export const MAX_HEALTH_CONNECT_DB_BYTES = 4 * 1024 * 1024 * 1024;
/** Where the Health Connect database landed. */
export interface HealthConnectUnzipResult {
dbPath: string;
dbBytes: number;
}
/**
* Find `health_connect_export.db` among the archive's members. The Health
* Connect app writes it as the archive's single entry; any folder in front of
* it (an archive repacked on a computer) is tolerated.
*/
export function selectHealthConnectDbEntry<T extends { fileName: string }>(
entries: readonly T[],
): T | null {
return (
entries.find(
(e) =>
!e.fileName.startsWith("__MACOSX/") &&
!e.fileName.endsWith("/") &&
baseName(e.fileName).toLowerCase() === HEALTH_CONNECT_DB_MEMBER,
) ?? null
);
}
/**
* Write the `health_connect_export.db` member of `archivePath` to a temp file
* with an unguessable name. Refuses an archive without the member, an
* encrypted or unknown compression method, a declared size above
* `maxBytes`, and a declared ratio above the zip-bomb threshold; the streamed
* byte cap holds the real output to the same ceiling whatever the header
* claims.
*/
export async function extractHealthConnectDb(
archivePath: string,
options: { maxBytes?: number } = {},
): Promise<HealthConnectUnzipResult> {
const maxBytes = options.maxBytes ?? MAX_HEALTH_CONNECT_DB_BYTES;
const handle = await open(archivePath, "r");
try {
const fileSize = Number((await handle.stat()).size);
if (!Number.isSafeInteger(fileSize)) {
throw new Error("Archive is too large to address safely");
}
const entries = await readCentralDirectoryFromFile(handle, fileSize);
const entry = selectHealthConnectDbEntry(entries);
if (!entry) {
throw new Error(
`not_health_connect: the archive has no ${HEALTH_CONNECT_DB_MEMBER} member — is this a Health Connect export?`,
);
}
if (entry.compressionMethod !== 0 && entry.compressionMethod !== 8) {
throw new Error(
`Unsupported ZIP compression method ${entry.compressionMethod}` +
` for ${HEALTH_CONNECT_DB_MEMBER} (expected 0=stored or 8=deflate)`,
);
}
if (entry.uncompressedSize > maxBytes) {
throw new Error(
`too_large: ${HEALTH_CONNECT_DB_MEMBER} declares ${entry.uncompressedSize} bytes` +
` — refusing to extract (cap is ${maxBytes} bytes).`,
);
}
if (
entry.compressedSize > 0 &&
entry.uncompressedSize / entry.compressedSize > MAX_COMPRESSION_RATIO
) {
throw new Error(
`${HEALTH_CONNECT_DB_MEMBER} advertises a ${(
entry.uncompressedSize / entry.compressedSize
).toFixed(0)}× compression ratio (cap is ${MAX_COMPRESSION_RATIO}×)` +
" — refusing as a suspected zip bomb.",
);
}
const dbPath = join(
tmpdir(),
`healthlog-hc-import-${randomBytes(12).toString("hex")}.db`,
);
const dbBytes = await streamEntryToFile(
handle,
archivePath,
fileSize,
entry,
dbPath,
maxBytes,
);
return { dbPath, dbBytes };
} finally {
await handle.close();
}
}
/** Read exactly `length` bytes at `position` or throw on truncation. */
async function readExactly(
handle: FileHandle,
@@ -651,6 +757,7 @@ async function streamEntryToFile(
fileSize: number,
entry: CentralDirectoryEntry,
destPath: string,
maxOutputBytes: number = MAX_DECOMPRESSED_BYTES,
): Promise<number> {
const local = await readExactly(handle, 30, entry.localHeaderOffset);
if (local.readUInt32LE(0) !== LOCAL_FILE_HEADER) {
@@ -669,7 +776,7 @@ async function streamEntryToFile(
}
const dest = createWriteStream(destPath);
const cap = createByteCap(memberOutputCap(entry));
const cap = createByteCap(Math.min(memberOutputCap(entry), maxOutputBytes));
try {
if (entry.compressedSize === 0) {
@@ -158,17 +158,26 @@ describe("reconcileOrphanImportJobs", () => {
});
});
it("only ever scopes the candidate query to the current parser revision", async () => {
it("scopes Apple Health rows to the current parser revision and takes every Health Connect row", async () => {
// v1.42: the table holds both importers' rows. A revision filter alone
// would either skip the Health Connect rows (they carry their own
// revision) or, matched by accident, look them up on the wrong queue.
mocks.findMany.mockResolvedValue([]);
await reconcileOrphanImportJobs();
expect(mocks.findMany).toHaveBeenCalledWith({
where: {
parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
status: { in: ["unpacking", "parsing", "upserting"] },
OR: [
{
kind: "apple_health",
parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
},
{ kind: "health_connect" },
],
},
select: { id: true, pgBossJobId: true, updatedAt: true },
select: { id: true, pgBossJobId: true, updatedAt: true, kind: true },
});
});
});
@@ -240,16 +249,32 @@ describe("stagedImportFilesInUse", () => {
"/tmp/a.bin",
"/tmp/b.bin",
]);
expect(inUse?.xmlInUse).toBe(true);
expect(inUse?.extractedInUse).toBe(true);
await handleImportJobReconcileTick([] as never);
expect(sweepStaleImportStaging).toHaveBeenCalledWith(
undefined,
undefined,
expect.objectContaining({ xmlInUse: true }),
expect.objectContaining({ extractedInUse: true }),
);
});
it("looks a Health Connect upload up on its own queue", async () => {
const rows = [
{ status: "queued", pgBossJobId: "boss-hc", kind: "health_connect" },
];
mocks.findMany.mockResolvedValueOnce(rows);
mocks.getGlobalBoss.mockReturnValue({ getJobById: mocks.getJobById });
mocks.getJobById.mockImplementation(async (queue: string) =>
queue === "health-connect-import"
? { state: "created", data: { uploadPath: "/tmp/hc.bin" } }
: null,
);
const inUse = await stagedImportFilesInUse();
expect([...(inUse?.paths ?? [])]).toEqual(["/tmp/hc.bin"]);
});
it("sweeps nothing when the queue cannot be read", async () => {
const rows = [{ status: "queued", pgBossJobId: "boss-1" }];
mocks.findMany
+41 -12
View File
@@ -54,6 +54,26 @@ import {
sweepStaleImportStaging,
type StagingInUse,
} from "@/lib/import/apple-health-staging";
import {
HEALTH_CONNECT_IMPORT_KIND,
HEALTH_CONNECT_IMPORT_QUEUE,
} from "@/lib/jobs/health-connect-import-worker";
/** `ImportJob.kind` of the rows this worker owns. */
export const APPLE_HEALTH_IMPORT_KIND = "apple_health";
/**
* The pg-boss queues a job of each `ImportJob.kind` can live on, in lookup
* order. The reconcile and the staging sweep share one cron across both
* importers, and each row is checked against its own importer's queue: a
* Health Connect job looked up on the Apple Health queue reads as gone, and
* the reconcile used to fail it while it was still running.
*/
function queuesForKind(kind: string): readonly string[] {
return kind === HEALTH_CONNECT_IMPORT_KIND
? [HEALTH_CONNECT_IMPORT_QUEUE]
: [APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_LEGACY_QUEUE];
}
/**
* Queue + cron for the periodic orphan-ImportJob sweep. v1.32.1
@@ -669,26 +689,24 @@ const LIVE_PG_BOSS_STATES = new Set(["active", "created", "retry"]);
/**
* The staged files a queued or running import still owns: the upload each
* live pg-boss job names, and whether any import is past the queue (it has
* an extracted XML open). `null` when that cannot be read, so the caller
* sweeps nothing rather than guess.
* an extracted XML or database open). `null` when that cannot be read, so the
* caller sweeps nothing rather than guess. Covers both importers; each row is
* looked up on its own kind's queue.
*/
export async function stagedImportFilesInUse(): Promise<StagingInUse | null> {
const prisma = getWorkerPrisma();
const rows = await prisma.importJob.findMany({
where: { status: { in: [...ACTIVE_IMPORT_STATUSES] } },
select: { status: true, pgBossJobId: true },
select: { status: true, pgBossJobId: true, kind: true },
});
const paths = new Set<string>();
if (rows.length === 0) return { paths, xmlInUse: false };
if (rows.length === 0) return { paths, extractedInUse: false };
const boss = getGlobalBoss();
if (!boss) return null;
for (const row of rows) {
if (!row.pgBossJobId) continue;
let job: { state: string; data: unknown } | null = null;
for (const queue of [
APPLE_HEALTH_IMPORT_V2_QUEUE,
APPLE_HEALTH_IMPORT_LEGACY_QUEUE,
]) {
for (const queue of queuesForKind(row.kind)) {
job = await boss.getJobById(queue, row.pgBossJobId);
if (job) break;
}
@@ -699,7 +717,7 @@ export async function stagedImportFilesInUse(): Promise<StagingInUse | null> {
}
return {
paths,
xmlInUse: rows.some((row) => row.status !== "queued"),
extractedInUse: rows.some((row) => row.status !== "queued"),
};
}
@@ -711,6 +729,9 @@ export async function stagedImportFilesInUse(): Promise<StagingInUse | null> {
* dedup no longer short-circuits future re-uploads onto a dead job).
* Revision-1 rows are deliberately excluded: during a rolling deployment
* their backing jobs live on the legacy queue and remain owned by old workers.
* Health Connect rows (v1.42) are reconciled by the same rule against their
* own queue; they carry their own parser revision, so the Apple Health
* revision filter applies to Apple Health rows only.
*
* The reconcile is deliberately NOT unconditional. In a multi-replica
* or rolling-deploy topology a booting worker must not flip a row that
@@ -737,10 +758,16 @@ export async function reconcileOrphanImportJobs(): Promise<void> {
const prisma = getWorkerPrisma();
const candidates = await prisma.importJob.findMany({
where: {
parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
status: { in: ["unpacking", "parsing", "upserting"] },
OR: [
{
kind: APPLE_HEALTH_IMPORT_KIND,
parserRevision: APPLE_HEALTH_IMPORT_PARSER_REVISION,
},
{ kind: HEALTH_CONNECT_IMPORT_KIND },
],
},
select: { id: true, pgBossJobId: true, updatedAt: true },
select: { id: true, pgBossJobId: true, updatedAt: true, kind: true },
});
if (candidates.length === 0) return;
@@ -761,7 +788,9 @@ export async function reconcileOrphanImportJobs(): Promise<void> {
let live = false;
try {
const job = await boss.getJobById(
APPLE_HEALTH_IMPORT_V2_QUEUE,
row.kind === HEALTH_CONNECT_IMPORT_KIND
? HEALTH_CONNECT_IMPORT_QUEUE
: APPLE_HEALTH_IMPORT_V2_QUEUE,
row.pgBossJobId,
);
live = job !== null && LIVE_PG_BOSS_STATES.has(job.state);
+241 -9
View File
@@ -3,18 +3,51 @@
*
* Reads the `health_connect_export.db` SQLite file out of an uploaded export
* ZIP (read-only, `immutable=1`) and writes its records as `HEALTH_CONNECT`
* measurements through the same reconciliation path the other external
* sources use. Mirrors the Apple Health import: the upload is staged to disk
* by the route, an `ImportJob` row (`kind = "health_connect"`) carries the
* progress, and the run consumes its staged file, so it is a one-shot.
* rows (`src/lib/import/health-connect/`). Mirrors the Apple Health import:
* the upload is staged to disk by the route, an `ImportJob` row
* (`kind = "health_connect"`) carries the progress, and the run consumes its
* staged file, so it is a one-shot with no retries.
*
* Contract stub: the queue is registered and bound; the handler refuses to
* claim success until the importer is implemented.
* The person's export is plain health data on the worker's disk. Both files,
* the upload and the extracted database, are removed in `finally` on every
* way out, and the shared staging sweep in the Apple Health reconcile cron
* removes anything a crash left behind (it knows both names). The terminal
* result holds counts per type and per app, nothing else.
*/
import { jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { rm } from "node:fs/promises";
import type { Job } from "pg-boss";
import type { MeasurementType } from "@/generated/prisma/client";
import { prisma, toJson } from "@/lib/db";
import { extractHealthConnectDb } from "@/lib/import/unzip-export-xml";
import {
importHealthConnectExport,
type HealthConnectImportProgress,
} from "@/lib/import/health-connect/import";
import { HealthConnectImportError } from "@/lib/import/health-connect/reader";
import { connectedDirectIntegrations } from "@/lib/import/health-connect/skip-packages";
import { invalidateStatusInsightsForTypes } from "@/lib/insights/status-invalidation";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
import { annotate } from "@/lib/logging/context";
import {
recomputeUserRollups,
ROLLUP_FOLD_WINDOW_MS,
} from "@/lib/rollups/measurement-rollups";
import { DEFAULT_TIMEZONE } from "@/lib/tz/format";
export const HEALTH_CONNECT_IMPORT_QUEUE = "health-connect-import";
/** `ImportJob.kind` of the rows this worker owns. */
export const HEALTH_CONNECT_IMPORT_KIND = "health_connect";
/**
* Parser semantics carried on the job row. The upload dedupe pins it, so a
* change to what the importer writes for the same file bumps it and lets the
* same bytes be imported again.
*/
export const HEALTH_CONNECT_IMPORT_PARSER_REVISION = 1;
/** One import at a time per worker: the SQLite read and the writes are heavy. */
export const HEALTH_CONNECT_IMPORT_CONCURRENCY = 1;
@@ -34,6 +67,205 @@ export interface HealthConnectImportPayload {
uploadPath: string;
}
export async function handleHealthConnectImport(): Promise<JobOutcome> {
return jobFailed("health connect import not implemented");
/**
* Failure reasons the card translates. Anything else is a database or
* archive error and passes through as written.
*/
export type HealthConnectFailureCode =
| "unsupported_version"
| "not_health_connect"
| "unsafe_schema"
| "too_large"
| "staging_missing";
function failureReason(err: unknown): string {
if (err instanceof HealthConnectImportError) return err.message;
if (
typeof err === "object" &&
err !== null &&
(err as NodeJS.ErrnoException).code === "ENOENT"
) {
return "staging_missing: the uploaded file is no longer on the server (a restart, or web and worker do not share the staging directory). Upload the export again.";
}
return err instanceof Error ? err.message : String(err);
}
/**
* The queue's work callback. pg-boss hands over an array; each import is
* independent, so a failed one does not stop the rest, and the first failure
* is what the batch reports.
*/
export async function handleHealthConnectImport(
jobs: Job<HealthConnectImportPayload>[],
): Promise<JobOutcome> {
let failure: JobOutcome | null = null;
for (const job of jobs) {
const outcome = await runHealthConnectImport(job.data);
if (!outcome.ok && failure === null) failure = outcome;
}
return failure ?? jobDone({ jobs: jobs.length });
}
/** One import, from the staged upload to the terminal job row. */
export async function runHealthConnectImport(
payload: HealthConnectImportPayload,
): Promise<JobOutcome> {
const { userId, importJobId, uploadPath } = payload;
let dbPath: string | null = null;
try {
const row = await prisma.importJob.findUnique({
where: { id: importJobId },
select: { id: true, status: true, userId: true, kind: true },
});
// Gone with its account, or not ours: nothing to import for.
if (
!row ||
row.userId !== userId ||
row.kind !== HEALTH_CONNECT_IMPORT_KIND
) {
return jobDone({ skipped: "no_job_row" });
}
// A redelivery after the first run: its upload is consumed, and running
// again could only overwrite the real outcome.
if (row.status === "done" || row.status === "failed") {
return jobDone({ skipped: "already_terminal" });
}
const user = await prisma.user.findUnique({
where: { id: userId },
select: { timezone: true },
});
const userTimezone =
user?.timezone && user.timezone.length > 0
? user.timezone
: DEFAULT_TIMEZONE;
await writeProgress(importJobId, "unpacking", {
currentPhase: "parsing",
recordsRead: 0,
rowsUpserted: 0,
percent: null,
elapsedMs: 0,
});
const unzipped = await extractHealthConnectDb(uploadPath);
dbPath = unzipped.dbPath;
// The archive has served its purpose; the database is what is read.
await rm(uploadPath, { force: true });
await writeProgress(importJobId, "parsing", {
currentPhase: "parsing",
recordsRead: 0,
rowsUpserted: 0,
percent: null,
elapsedMs: 0,
});
const connectedIntegrations = await connectedDirectIntegrations(
prisma,
userId,
);
const result = await importHealthConnectExport({
prisma,
dbPath,
userId,
userTimezone,
connectedIntegrations,
onProgress: (snapshot) =>
writeProgress(importJobId, snapshot.currentPhase, snapshot),
});
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: "done",
completedAt: new Date(),
progress: toJson({
currentPhase: "upserting",
recordsRead: result.totals.recordsRead,
rowsUpserted: result.totals.rowsUpserted,
percent: 100,
elapsedMs: result.totals.durationMs,
}),
result: toJson(result),
},
});
// The per-row tails are skipped on a bulk import; refold the rollups over
// what this import wrote, inside the window every other path keeps to,
// and re-warm the cached assessments of the types it touched.
let rollupFailed = false;
const span = result.measuredSpan;
if (span) {
try {
const windowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS);
const spanFrom = new Date(span.from);
const from = spanFrom > windowStart ? spanFrom : windowStart;
const to = new Date(new Date(span.to).getTime() + 1);
if (to > from) await recomputeUserRollups(userId, { from, to });
} catch (err) {
rollupFailed = true;
console.warn(
`[health-connect-import] Rollup recompute failed for user ${userId}`,
err,
);
}
}
const touched = Object.entries(result.perType)
.filter(([, stat]) => stat.inserted + stat.updated > 0)
.map(([type]) => type as MeasurementType);
if (touched.length > 0) {
trackBackgroundTask(
invalidateStatusInsightsForTypes(userId, touched).catch((err) => {
console.warn(
`[health-connect-import] status-insight invalidate failed for user ${userId}`,
err,
);
}),
);
}
annotate({
action: { name: "import.health-connect.done" },
meta: {
records_read: result.totals.recordsRead,
rows_upserted: result.totals.rowsUpserted,
duration_ms: result.totals.durationMs,
user_version: result.userVersion,
},
});
return jobDone({
records_read: result.totals.recordsRead,
rows_upserted: result.totals.rowsUpserted,
duration_ms: result.totals.durationMs,
rollup_failed: rollupFailed,
});
} catch (err) {
const reason = failureReason(err);
try {
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: "failed",
failureReason: reason.slice(0, 1000),
completedAt: new Date(),
},
});
} catch {
// The row went with a deleted account; nothing left to record on.
}
return jobFailed("health connect import failed", err);
} finally {
await rm(uploadPath, { force: true }).catch(() => {});
if (dbPath) await rm(dbPath, { force: true }).catch(() => {});
}
}
async function writeProgress(
importJobId: string,
status: string,
progress: HealthConnectImportProgress,
): Promise<void> {
await prisma.importJob.update({
where: { id: importJobId },
data: { status, progress: toJson(progress) },
});
}
+14 -9
View File
@@ -20,7 +20,6 @@ import {
dataEnvelope,
errorEnvelope,
measurementTypeEnum,
notImplementedResponse,
stdResponses,
} from "./shared";
@@ -196,8 +195,15 @@ const healthConnectImportJob = z
result: z
.record(z.string(), z.unknown())
.nullable()
.describe("The worker's outcome summary, null until it finishes."),
failureReason: z.string().nullable(),
.describe(
"The worker's outcome summary, null until it finishes. Counts only: `perType` (read / inserted / updated / unchanged / skipped per measurement type), `perApp` (records per Android package, `leftOut` for an app whose integration the account has connected), `workouts`, `sleep`, `cycle`, `nutrients`, `skipped` (reason → count), `warnings` (codes for tables or columns the importer could not read), `totals`. Never a value, a timestamp or text from the file.",
),
failureReason: z
.string()
.nullable()
.describe(
"Why the job failed. Starts with a code the client can key on when the file was refused — `unsupported_version` (database older than version 9), `not_health_connect` (no `health_connect_export.db`, or not a Health Connect database), `unsafe_schema` (a record table is a view), `too_large`, `staging_missing` — else free text.",
),
})
.meta({ id: "HealthConnectImportJob" });
@@ -251,25 +257,25 @@ export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
},
"409": {
description:
"An import for this account is already queued or running with different bytes. The upload was discarded.",
"An import for this account (Health Connect or Apple Health) is already queued or running. The upload was discarded. `errorCode` is `import.apple_health.busy` for either kind, `jobId` names the running job.",
content: { "application/json": { schema: errorEnvelope } },
},
"413": {
description: "The declared `Content-Length` exceeds the upload cap.",
description:
"The declared `Content-Length` exceeds the 1 GiB upload cap.",
content: { "application/json": { schema: errorEnvelope } },
},
...stdResponses,
"422": {
description:
"The upload is not a Health Connect export the importer can read: no `file` part, no export database in the ZIP, or a database version older than the importer supports.",
"The multipart body could not be read: no `file` part, a malformed body, or more bytes than the cap. Whether the ZIP is a readable Health Connect export is decided by the worker and reported on the job (`status: failed`, `failureReason`).",
content: { "application/json": { schema: errorEnvelope } },
},
"503": {
description:
"No background worker is bound, so nothing would run the import.",
"No background worker is bound, or the queue did not take the job, so nothing would run the import.",
content: { "application/json": { schema: errorEnvelope } },
},
...notImplementedResponse,
},
},
},
@@ -292,7 +298,6 @@ export const importPaths: NonNullable<ZodOpenApiObject["paths"]> = {
},
},
...stdResponses,
...notImplementedResponse,
},
},
},
+8
View File
@@ -80,4 +80,12 @@ export const integrationKeys = {
*/
importJobStatus: (jobId: string) =>
["import", "apple-health", "status", jobId] as const,
/**
* v1.42 (#972) — the account's latest Health Connect export import. One key
* per account rather than per job: the status route answers the newest
* job, so the card shows the last outcome after a reload as well.
*/
healthConnectImportStatus: () =>
["import", "health-connect", "status"] as const,
};