diff --git a/CHANGELOG.md b/CHANGELOG.md index fd0d97d17..cba2145eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,14 +63,20 @@ the plain unit. Any other text in brackets still counts as part of the unit. (#1313) - **The sync feed no longer loses a reading written during a long - import.** A Garmin or Withings import writes a batch in one transaction - and stamps its readings before it commits them. When a reading logged by - hand committed in between and the app synced right then, the feed's - position moved past the import's readings, and they never reached the - app or Apple Health. A caught-up page of `GET /api/sync/changes` now - hands back a position a minute behind the present, like `start=now` - already does, so the next pull reads that minute again and anything - that committed late comes with it. + import, and no longer starts over for a quiet account.** A Garmin or + Withings import writes a batch in one transaction and stamps its + readings before it commits them. When a reading logged by hand + committed in between and the app synced right then, the feed's position + moved past the import's readings, and they never reached the app or + Apple Health. Separately, a kind of data that had not changed for more + than 75 days (mood, say, for someone who stopped logging it) kept its + old position in the feed, and from then on every sync was answered as + expired and the app started over from scratch, each time. Now every + kind of data that `GET /api/sync/changes` has fully caught up takes a + position two minutes behind the present, like `start=now` (which moves + from one minute to two): what changed in those two minutes is read + once more, so anything that committed late still arrives, and a quiet + kind of data never ages out. - **Changing the calendar subscription while its link is being renewed.** A subscription turned off at the same moment its link was renewed could come back under the new link, and an option changed at that moment diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index ab5b2953d..f00512ce5 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -4522,17 +4522,17 @@ paths: - in: query name: start schema: - description: "`now` answers an empty page (every domain empty, `hasMore: false`) whose `cursor` sits one minute before + description: "`now` answers an empty page (every domain empty, `hasMore: false`) whose `cursor` sits two minutes before `serverNow` in every domain, in the same opaque format as any other page. Paging forward from it yields - what changes from then on, plus anything written in that last minute (a write stamped before `serverNow` + what changes from then on, plus anything written in those two minutes (a write stamped before `serverNow` can commit after it, and is not skipped; upserts are idempotent, so a row seen twice is harmless), including `measurementMirror`, without walking the history first. Mutually exclusive with `cursor` (422). A server without it ignores the parameter and starts from the oldest row." type: string const: now - description: "`now` answers an empty page (every domain empty, `hasMore: false`) whose `cursor` sits one minute before + description: "`now` answers an empty page (every domain empty, `hasMore: false`) whose `cursor` sits two minutes before `serverNow` in every domain, in the same opaque format as any other page. Paging forward from it yields what - changes from then on, plus anything written in that last minute (a write stamped before `serverNow` can + changes from then on, plus anything written in those two minutes (a write stamped before `serverNow` can commit after it, and is not skipped; upserts are idempotent, so a row seen twice is harmless), including `measurementMirror`, without walking the history first. Mutually exclusive with `cursor` (422). A server without it ignores the parameter and starts from the oldest row." @@ -50202,9 +50202,10 @@ components: - "null" hasMore: type: boolean - description: False once the client is caught up as of `serverNow`. The cursor of such a page sits at most a minute - before `serverNow` in each row domain, so the next pull reads rows changed in that last minute again (a - write still in flight may commit a row stamped there); upserts are idempotent. + description: False once the client is caught up as of `serverNow`. Every row domain that ran dry on a page takes a + watermark two minutes before `serverNow` (on the last page of a walk, all of them), so the next pull reads + rows changed in that stretch again (a write still in flight may commit a row stamped there) and a domain + that stays quiet never ages its cursor past the tombstone horizon; upserts are idempotent. cursorExpired: type: boolean description: True when the supplied cursor predates tombstone retention, or was issued before the account's data was diff --git a/src/app/api/sync/changes/__tests__/start-now.test.ts b/src/app/api/sync/changes/__tests__/start-now.test.ts index 02f812fb1..1e7612ebb 100644 --- a/src/app/api/sync/changes/__tests__/start-now.test.ts +++ b/src/app/api/sync/changes/__tests__/start-now.test.ts @@ -1,7 +1,7 @@ /** * `GET /api/sync/changes?start=now` (#1339) — a cursor at the present. * - * The answer is an empty page whose cursor sits a minute before `serverNow` + * The answer is an empty page whose cursor sits two minutes before `serverNow` * in every domain (a write stamped before it and committed after it is not * skipped), in the feed's own opaque format, and nothing is read: the point is * to skip the history walk, so a read here would be the bug. The database is @@ -55,7 +55,7 @@ beforeEach(() => { }); describe("GET /api/sync/changes?start=now", () => { - it("answers an empty page with a cursor a minute before serverNow in every domain", async () => { + it("answers an empty page with a cursor two minutes before serverNow in every domain", async () => { const before = Date.now(); const res = await GET(req("?start=now")); const after = Date.now(); @@ -82,7 +82,7 @@ describe("GET /api/sync/changes?start=now", () => { expect(cursor).not.toBeNull(); for (const domain of SYNC_DOMAINS) { expect(cursor![domain], domain).toEqual({ - updatedAtMs: nowMs - 60_000, + updatedAtMs: nowMs - 120_000, id: "0", }); } diff --git a/src/app/api/sync/changes/route.ts b/src/app/api/sync/changes/route.ts index 4d6574219..e9c31b87c 100644 --- a/src/app/api/sync/changes/route.ts +++ b/src/app/api/sync/changes/route.ts @@ -30,11 +30,13 @@ * server `id` (the client mirrors those by server id). * - `hasMore` is true when ANY served domain still has rows past its * page; the next `cursor` advances every domain that returned rows. - * When `hasMore` is false the client is caught up as of `serverNow`, - * and that page's cursor is held back to a minute before it in every - * row domain (`holdBackDrainedCursor`): a row stamped in that minute - * whose write commits after the pull still arrives on the next one, at - * the cost of reading that minute twice. + * When `hasMore` is false the client is caught up as of `serverNow`. + * A row domain that ran dry takes a watermark two minutes before + * `serverNow` (`drainedWatermark`; on the last page every one does): + * a row stamped in that stretch whose write commits after the pull + * still arrives on the next one, at the cost of reading the stretch + * twice, and a domain that stays quiet keeps a fresh watermark instead + * of ageing into `cursorExpired`. * - `cursorExpired: true` when ANY domain watermark predates the * tombstone-retention horizon — the client drops its cursor and does * a clean initial sync (a deletion older than retention may have @@ -47,7 +49,7 @@ * cycle tracking off gets both blocks empty (never a 403 — see the * gate note in the handler) and their cursor watermarks untouched. * - * `start=now` (#1339) answers an empty page whose cursor sits a minute + * `start=now` (#1339) answers an empty page whose cursor sits two minutes * before `serverNow` in every domain (`START_NOW_OVERLAP_MS`), in the same * opaque format, so a client that * only wants what changes from now on (the Apple Health write-back) pages diff --git a/src/lib/openapi/routes/sync.ts b/src/lib/openapi/routes/sync.ts index bfc9207a7..dc2134410 100644 --- a/src/lib/openapi/routes/sync.ts +++ b/src/lib/openapi/routes/sync.ts @@ -256,7 +256,7 @@ const syncChangesQuery = z .literal("now") .optional() .describe( - "`now` answers an empty page (every domain empty, `hasMore: false`) whose `cursor` sits one minute before `serverNow` in every domain, in the same opaque format as any other page. Paging forward from it yields what changes from then on, plus anything written in that last minute (a write stamped before `serverNow` can commit after it, and is not skipped; upserts are idempotent, so a row seen twice is harmless), including `measurementMirror`, without walking the history first. Mutually exclusive with `cursor` (422). A server without it ignores the parameter and starts from the oldest row.", + "`now` answers an empty page (every domain empty, `hasMore: false`) whose `cursor` sits two minutes before `serverNow` in every domain, in the same opaque format as any other page. Paging forward from it yields what changes from then on, plus anything written in those two minutes (a write stamped before `serverNow` can commit after it, and is not skipped; upserts are idempotent, so a row seen twice is harmless), including `measurementMirror`, without walking the history first. Mutually exclusive with `cursor` (422). A server without it ignores the parameter and starts from the oldest row.", ), limit: z.coerce .number() @@ -286,7 +286,7 @@ export const syncChangesResponse = z hasMore: z .boolean() .describe( - "False once the client is caught up as of `serverNow`. The cursor of such a page sits at most a minute before `serverNow` in each row domain, so the next pull reads rows changed in that last minute again (a write still in flight may commit a row stamped there); upserts are idempotent.", + "False once the client is caught up as of `serverNow`. Every row domain that ran dry on a page takes a watermark two minutes before `serverNow` (on the last page of a walk, all of them), so the next pull reads rows changed in that stretch again (a write still in flight may commit a row stamped there) and a domain that stays quiet never ages its cursor past the tombstone horizon; upserts are idempotent.", ), cursorExpired: z .boolean() diff --git a/src/lib/sync/__tests__/cursor.test.ts b/src/lib/sync/__tests__/cursor.test.ts index adaae6cfa..55980b04c 100644 --- a/src/lib/sync/__tests__/cursor.test.ts +++ b/src/lib/sync/__tests__/cursor.test.ts @@ -139,16 +139,16 @@ describe("cursor issue time (sync reset)", () => { }); describe("encodeCursorAtNow", () => { - it("places every domain a minute back, before any id in that millisecond", () => { + it("places every domain two minutes back, before any id in that millisecond", () => { const nowMs = 1_790_000_000_123; const token = encodeCursorAtNow(nowMs); const cursor = decodeCursor(token); // A write stamped before the instant but committed after it is still // ahead of the watermark. - expect(START_NOW_OVERLAP_MS).toBe(60_000); + expect(START_NOW_OVERLAP_MS).toBe(120_000); for (const domain of SYNC_DOMAINS) { expect(cursor?.[domain], domain).toEqual({ - updatedAtMs: nowMs - 60_000, + updatedAtMs: nowMs - 120_000, id: "0", }); } diff --git a/src/lib/sync/changes/pull.ts b/src/lib/sync/changes/pull.ts index 4b5ca8be6..0735a647b 100644 --- a/src/lib/sync/changes/pull.ts +++ b/src/lib/sync/changes/pull.ts @@ -14,7 +14,7 @@ import { decodeCursorIssuedAt, encodeCursor, encodeCursorAtNow, - holdBackDrainedCursor, + drainedWatermark, SYNC_DOMAINS, type SyncCursor, type SyncDomain, @@ -111,7 +111,7 @@ function emptyChanges(): SyncChangesBody["changes"] { } /** - * `start=now` (#1339): an empty page with a cursor a minute before + * `start=now` (#1339): an empty page with a cursor two minutes before * `serverNow` in every domain (`encodeCursorAtNow`), for a client that wants changes from now on (the Apple Health * write-back) without paging through the history first. `hasMore` is false: * the client is caught up as of `serverNow` by definition. @@ -233,7 +233,39 @@ export async function pullSyncChanges(input: { cd.hasMore || cy.hasMore || mm.hasMore; - if (!hasMore) holdBackDrainedCursor(nextCursor, serverNow.getTime()); + + // A row domain whose page ran dry takes the overlap watermark + // (`drainedWatermark`). On the last page of a walk every one does. On an + // earlier page only one whose watermark has not yet passed the overlap + // line: one that walked into the last stretch on a full page keeps its + // place, because reading that stretch again while it holds more than a + // page, with another domain still paging, would never end. The cycle + // domains are left alone while the module is off, and `measurementMirror` + // keeps its token stamps: re-reading its stretch would send every reading + // of a token whose switch just flipped once more on every pull. + const drained = drainedWatermark(serverNow.getTime()); + const rowDomains: Array<[SyncDomain, boolean]> = [ + ["measurements", m.hasMore], + ["mood", md.hasMore], + ["intakes", ik.hasMore], + ...(cycleEnabled + ? ([ + ["cycleDays", cd.hasMore], + ["cycles", cy.hasMore], + ] as Array<[SyncDomain, boolean]>) + : []), + ]; + for (const [domain, domainHasMore] of rowDomains) { + if (domainHasMore) continue; + const before = cursor[domain]; + if ( + !hasMore || + before === undefined || + before.updatedAtMs <= drained.updatedAtMs + ) { + nextCursor[domain] = drained; + } + } annotate({ action: { name: "sync.changes.pull" }, diff --git a/src/lib/sync/cursor.ts b/src/lib/sync/cursor.ts index 7cacd33ac..4f6f76e21 100644 --- a/src/lib/sync/cursor.ts +++ b/src/lib/sync/cursor.ts @@ -103,13 +103,16 @@ export function encodeCursor(cursor: SyncCursor, issuedAtMs?: number): string { } /** - * How far behind the present a `start=now` cursor is placed. `updatedAt` is - * stamped by the application when the write is built, not when it commits, so - * a transaction in flight at `nowMs` can commit a row stamped before it after - * the cursor was handed out; a watermark at `nowMs` would skip that row for - * good. A minute covers any write the app keeps open. + * How far behind the present a caught-up watermark is placed: a `start=now` + * cursor, and every domain a page has drained. `updatedAt` is stamped by the + * application when the write is built, not when it commits, so a transaction + * in flight can commit a row stamped before a watermark after it was handed + * out; a watermark at the present would skip that row for good. The longest + * interactive transaction that writes served rows (the provider syncs and the + * intake import) times out at 60 s, so two minutes covers any of them with + * room to spare. */ -export const START_NOW_OVERLAP_MS = 60_000; +export const START_NOW_OVERLAP_MS = 120_000; /** * A token positioned just before `nowMs` in every domain (#1339), the same @@ -130,31 +133,24 @@ export function encodeCursorAtNow(nowMs: number): string { } /** - * Hold a drained walk's watermarks back to `START_NOW_OVERLAP_MS` before - * `nowMs`, in place. The watermark of a page is the stamp of its last row, - * and that stamp says nothing about rows still in flight: a long write (a - * Garmin or Withings chunk holds its transaction for seconds) stamps its rows - * first and commits them last, so a quick write that commits in between can - * carry the watermark past them, and the keyset would never return them. - * Held back a minute, the next pull reads the last minute again and the - * late rows come with it; anything older is passed as before. Only the - * cursor of a walk that is done is held back: one that still has pages keeps - * moving forward, or a minute holding more than a page could never drain. + * The watermark for a domain whose page has run dry: `START_NOW_OVERLAP_MS` + * before `nowMs`, whichever way that moves it. * - * `measurementMirror` is left alone. Its watermark is a token's mirror - * stamp, written by one short update, and reading its last minute again - * would send every reading of a token whose mirror switch just flipped - * once more on every pull in that minute. + * Back, when the last row delivered is newer: a long write (a provider sync + * holds its transaction for seconds) stamps its rows first and commits them + * last, so a quick write committing in between carried the watermark past + * them and the keyset never returned them. Held back, the next pull reads + * that stretch again and the late rows come with it. + * + * Forward, when the last row is older or there is none: the page just read + * everything past the old watermark, so nothing older than the overlap is + * left behind it. A domain that stays quiet would otherwise keep the stamp + * of its last row for good, and once that is older than the tombstone + * horizon every pull answers `cursorExpired` and the client resyncs from + * scratch, again and again. */ -export function holdBackDrainedCursor(cursor: SyncCursor, nowMs: number): void { - const ceiling = nowMs - START_NOW_OVERLAP_MS; - for (const domain of SYNC_DOMAINS) { - if (domain === "measurementMirror") continue; - const wm = cursor[domain]; - if (wm && wm.updatedAtMs > ceiling) { - cursor[domain] = { updatedAtMs: ceiling, id: "0" }; - } - } +export function drainedWatermark(nowMs: number): DomainWatermark { + return { updatedAtMs: nowMs - START_NOW_OVERLAP_MS, id: "0" }; } function parseWatermark(raw: unknown): DomainWatermark | null { diff --git a/tests/integration/routes/parity/__snapshots__/sync-changes.01-first-page.json.snap b/tests/integration/routes/parity/__snapshots__/sync-changes.01-first-page.json.snap index 6e7ef10b0..33795a4de 100644 --- a/tests/integration/routes/parity/__snapshots__/sync-changes.01-first-page.json.snap +++ b/tests/integration/routes/parity/__snapshots__/sync-changes.01-first-page.json.snap @@ -3,7 +3,7 @@ "body": { "data": { "serverNow": "", - "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDQ4MjAwMDAwLCJpIjoibS0wMiJ9LCJtb29kIjp7InUiOjE3OTIwNTE4MDAwMDAsImkiOiJtb29kLTAyIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzAwMDAwMCwiaSI6ImluLTAyIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU2NjAwMDAwLCJpIjoiY2QtMDIifSwiY3ljbGVzIjp7InUiOjE3OTIwNTQ4MDAwMDAsImkiOiJjeS0wMiJ9fX0", + "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDQ4MjAwMDAwLCJpIjoibS0wMiJ9LCJtb29kIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzAwMDAwMCwiaSI6ImluLTAyIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU2NjAwMDAwLCJpIjoiY2QtMDIifSwiY3ljbGVzIjp7InUiOjE3OTIwNTQ4MDAwMDAsImkiOiJjeS0wMiJ9fX0", "hasMore": true, "cursorExpired": false, "changes": { diff --git a/tests/integration/routes/parity/__snapshots__/sync-changes.02-second-page.json.snap b/tests/integration/routes/parity/__snapshots__/sync-changes.02-second-page.json.snap index 735d06770..2ae2df810 100644 --- a/tests/integration/routes/parity/__snapshots__/sync-changes.02-second-page.json.snap +++ b/tests/integration/routes/parity/__snapshots__/sync-changes.02-second-page.json.snap @@ -3,7 +3,7 @@ "body": { "data": { "serverNow": "", - "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDQ5NDAwMDAwLCJpIjoibS0wNCJ9LCJtb29kIjp7InUiOjE3OTIwNTE4MDAwMDAsImkiOiJtb29kLTAyIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzYwMDAwMCwiaSI6ImluLTAzIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU3MjAwMDAwLCJpIjoiY2QtMDMifSwiY3ljbGVzIjp7InUiOjE3OTIwNTU0MDAwMDAsImkiOiJjeS0wMyJ9fX0", + "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDQ5NDAwMDAwLCJpIjoibS0wNCJ9LCJtb29kIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImludGFrZXMiOnsidSI6MTc5MjA2NTQ4MDAwMCwiaSI6IjAifSwiY3ljbGVEYXlzIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImN5Y2xlcyI6eyJ1IjoxNzkyMDY1NDgwMDAwLCJpIjoiMCJ9fX0", "hasMore": true, "cursorExpired": false, "changes": { diff --git a/tests/integration/routes/parity/__snapshots__/sync-changes.03-the-rest.json.snap b/tests/integration/routes/parity/__snapshots__/sync-changes.03-the-rest.json.snap index 97841d250..c8a077f78 100644 --- a/tests/integration/routes/parity/__snapshots__/sync-changes.03-the-rest.json.snap +++ b/tests/integration/routes/parity/__snapshots__/sync-changes.03-the-rest.json.snap @@ -3,7 +3,7 @@ "body": { "data": { "serverNow": "", - "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDUwNjAwMDAwLCJpIjoibS0wNiJ9LCJtb29kIjp7InUiOjE3OTIwNTE4MDAwMDAsImkiOiJtb29kLTAyIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzYwMDAwMCwiaSI6ImluLTAzIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU3MjAwMDAwLCJpIjoiY2QtMDMifSwiY3ljbGVzIjp7InUiOjE3OTIwNTU0MDAwMDAsImkiOiJjeS0wMyJ9fX0", + "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDY1NDgwMDAwLCJpIjoiMCJ9LCJtb29kIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImludGFrZXMiOnsidSI6MTc5MjA2NTQ4MDAwMCwiaSI6IjAifSwiY3ljbGVEYXlzIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImN5Y2xlcyI6eyJ1IjoxNzkyMDY1NDgwMDAwLCJpIjoiMCJ9fX0", "hasMore": false, "cursorExpired": false, "changes": { diff --git a/tests/integration/routes/parity/__snapshots__/sync-changes.04-caught-up.json.snap b/tests/integration/routes/parity/__snapshots__/sync-changes.04-caught-up.json.snap index e74d39368..abfb90e2c 100644 --- a/tests/integration/routes/parity/__snapshots__/sync-changes.04-caught-up.json.snap +++ b/tests/integration/routes/parity/__snapshots__/sync-changes.04-caught-up.json.snap @@ -3,7 +3,7 @@ "body": { "data": { "serverNow": "", - "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDUwNjAwMDAwLCJpIjoibS0wNiJ9LCJtb29kIjp7InUiOjE3OTIwNTE4MDAwMDAsImkiOiJtb29kLTAyIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzYwMDAwMCwiaSI6ImluLTAzIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU3MjAwMDAwLCJpIjoiY2QtMDMifSwiY3ljbGVzIjp7InUiOjE3OTIwNTU0MDAwMDAsImkiOiJjeS0wMyJ9fX0", + "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDY1NDgwMDAwLCJpIjoiMCJ9LCJtb29kIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImludGFrZXMiOnsidSI6MTc5MjA2NTQ4MDAwMCwiaSI6IjAifSwiY3ljbGVEYXlzIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImN5Y2xlcyI6eyJ1IjoxNzkyMDY1NDgwMDAwLCJpIjoiMCJ9fX0", "hasMore": false, "cursorExpired": false, "changes": { diff --git a/tests/integration/routes/parity/__snapshots__/sync-changes.06-before-reset.json.snap b/tests/integration/routes/parity/__snapshots__/sync-changes.06-before-reset.json.snap index 52d0edffe..2f116f524 100644 --- a/tests/integration/routes/parity/__snapshots__/sync-changes.06-before-reset.json.snap +++ b/tests/integration/routes/parity/__snapshots__/sync-changes.06-before-reset.json.snap @@ -3,7 +3,7 @@ "body": { "data": { "serverNow": "", - "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDQ4MjAwMDAwLCJpIjoibS0wMiJ9LCJtb29kIjp7InUiOjE3OTIwNTE4MDAwMDAsImkiOiJtb29kLTAyIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzAwMDAwMCwiaSI6ImluLTAyIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU2NjAwMDAwLCJpIjoiY2QtMDIifSwiY3ljbGVzIjp7InUiOjE3OTIwNTQ4MDAwMDAsImkiOiJjeS0wMiJ9fX0", + "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDQ4MjAwMDAwLCJpIjoibS0wMiJ9LCJtb29kIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzAwMDAwMCwiaSI6ImluLTAyIn0sImN5Y2xlRGF5cyI6eyJ1IjoxNzkyMDU2NjAwMDAwLCJpIjoiY2QtMDIifSwiY3ljbGVzIjp7InUiOjE3OTIwNTQ4MDAwMDAsImkiOiJjeS0wMiJ9fX0", "hasMore": false, "cursorExpired": true, "changes": { diff --git a/tests/integration/routes/parity/__snapshots__/sync-changes.07-cycle-off.json.snap b/tests/integration/routes/parity/__snapshots__/sync-changes.07-cycle-off.json.snap index f4c382042..4cf382c69 100644 --- a/tests/integration/routes/parity/__snapshots__/sync-changes.07-cycle-off.json.snap +++ b/tests/integration/routes/parity/__snapshots__/sync-changes.07-cycle-off.json.snap @@ -3,7 +3,7 @@ "body": { "data": { "serverNow": "", - "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDUwNjAwMDAwLCJpIjoibS0wNiJ9LCJtb29kIjp7InUiOjE3OTIwNTE4MDAwMDAsImkiOiJtb29kLTAyIn0sImludGFrZXMiOnsidSI6MTc5MjA1MzYwMDAwMCwiaSI6ImluLTAzIn19fQ", + "cursor": "eyJ2IjoyLCJ0IjoxNzkyMDY1NjAwMDAwLCJkIjp7Im1lYXN1cmVtZW50cyI6eyJ1IjoxNzkyMDY1NDgwMDAwLCJpIjoiMCJ9LCJtb29kIjp7InUiOjE3OTIwNjU0ODAwMDAsImkiOiIwIn0sImludGFrZXMiOnsidSI6MTc5MjA2NTQ4MDAwMCwiaSI6IjAifX19", "hasMore": false, "cursorExpired": false, "changes": { diff --git a/tests/integration/sync-changes-start-now.test.ts b/tests/integration/sync-changes-start-now.test.ts index 72a63bb5d..9a424949e 100644 --- a/tests/integration/sync-changes-start-now.test.ts +++ b/tests/integration/sync-changes-start-now.test.ts @@ -8,7 +8,7 @@ * that token's readings on `measurementMirror`, because the switch happens * after the cursor. * - * The cursor sits a minute behind the present: `updatedAt` is stamped when a + * The cursor sits two minutes behind the present: `updatedAt` is stamped when a * write is built, not when it commits, so a row stamped just before the * cursor can become visible just after it, and must not be skipped. */ @@ -138,7 +138,7 @@ describe("a cursor that starts now", () => { ); expect(mirrored.every((r) => r.appleHealthMirror === true)).toBe(true); - // The drained cursor it hands back sits a minute back like the start + // The drained cursor it hands back sits two minutes back like the start // one, so the row written in that minute comes once more and nothing // else does. The mirror domain is not held back. const after = await pull({ cursor: next.cursor! }); diff --git a/tests/integration/sync-changes.test.ts b/tests/integration/sync-changes.test.ts index b7092d344..f5f8b1f2f 100644 --- a/tests/integration/sync-changes.test.ts +++ b/tests/integration/sync-changes.test.ts @@ -585,6 +585,86 @@ describe("GET /api/sync/changes — multi-domain (real Postgres)", () => { expect(j.data.hasMore).toBe(true); }); + it("keeps catching up when a domain has had no change since before the retention horizon", async () => { + // Someone who stopped logging mood months ago: the newest mood row is + // older than the horizon. A drained walk proved nothing changed since, + // so its cursor must not read as expired on the next pull. + const prisma = getPrismaClient(); + await prisma.moodEntry.create({ + data: { + userId: TEST_USER_ID, + date: "2026-01-02", + mood: "GUT", + score: 4, + moodLoggedAt: new Date( + Date.now() - (TOMBSTONE_RETENTION_DAYS + 10) * 86_400_000, + ), + updatedAt: new Date( + Date.now() - (TOMBSTONE_RETENTION_DAYS + 10) * 86_400_000, + ), + }, + }); + await seedLive(1); + const { GET } = await import("@/app/api/sync/changes/route"); + const first = (await (await GET(makeRequest())).json()) as { + data: ChangesData; + }; + expect(first.data.hasMore).toBe(false); + expect(first.data.changes.mood.upserts).toHaveLength(1); + const second = (await ( + await GET( + makeRequest(`?cursor=${encodeURIComponent(first.data.cursor!)}`), + ) + ).json()) as { data: ChangesData }; + expect(second.data.cursorExpired).toBe(false); + }); + + it("delivers a late row in a domain that ran dry on an earlier page of the walk", async () => { + // Two measurements keep the walk going at limit=1; the mood domain runs + // dry on the first page. A mood row stamped before that page's mood row + // and committed after it still arrives before the walk ends. + await seedLive(2); + const prisma = getPrismaClient(); + const quick = await prisma.moodEntry.create({ + data: { + userId: TEST_USER_ID, + date: fixtureDay(1), + mood: "GUT", + score: 4, + moodLoggedAt: new Date(), + }, + }); + const { GET } = await import("@/app/api/sync/changes/route"); + const page1 = (await (await GET(makeRequest("?limit=1"))).json()) as { + data: ChangesData; + }; + expect(page1.data.hasMore).toBe(true); + expect(page1.data.changes.mood.upserts.map((r) => r.id)).toEqual([ + quick.id, + ]); + const late = await prisma.moodEntry.create({ + data: { + userId: TEST_USER_ID, + date: fixtureDay(2), + mood: "GUT", + score: 3, + moodLoggedAt: new Date(), + updatedAt: new Date(quick.updatedAt.getTime() - 1_000), + }, + }); + const seen = new Set(); + let cursor = page1.data.cursor!; + for (let i = 0; i < 5; i++) { + const page = (await ( + await GET(makeRequest(`?limit=1&cursor=${encodeURIComponent(cursor)}`)) + ).json()) as { data: ChangesData }; + for (const row of page.data.changes.mood.upserts) seen.add(row.id); + cursor = page.data.cursor!; + if (!page.data.hasMore) break; + } + expect(seen).toContain(late.id); + }); + it("treats a garbage cursor as a clean initial sync", async () => { await seedMood(1); const { GET } = await import("@/app/api/sync/changes/route");