feat(environment): fill air quality for every past day with entries

A background job fetches air quality, pollen and UV for every earlier
local day with a reading, mood entry, dose, symptom or workout: after the
update (boot and nightly discovery), when the home is set and when air
quality is switched on. A day with a weather row gets its air quality; a
day without one gets both, at the covering location period or else at the
home, also before the home was set. Days the source does not reach (before
2013 in Europe, before August 2022 elsewhere) stay empty and are not
counted.

Pending days are grouped by coarse location and cut into ranges of up to
90 days, joining days at most 14 apart, newest first. A run holds itself to
300 of the account's 400 daily calls and half of each instance window,
stores its progress on the user (migration 0391) and sends a follow-up while
it makes progress; a budget stop waits for the next discovery. The
Location & weather card shows the progress.

The air-quality client now asks pollen and dust from 2021 and skips the
request outside Europe before CAMS global starts.
This commit is contained in:
Marc-André Bombeck committed 2026-10-09 09:43:01 +02:00
1 parent 7340cd3ab7
commit d81a6ddf25
30 files changed
+1747 -52

No files matched your search

+4
View File
@@ -18,6 +18,7 @@ import { homeLocationSchema } from "@/lib/validations/environment";
import { roundCoarse } from "@/lib/environment/open-meteo";
import { sealLocation } from "@/lib/environment/location-cipher";
import { enqueueEnvironmentFetch } from "@/lib/jobs/environment-fetch";
import { enqueueAirQualityHistory } from "@/lib/jobs/environment-air-quality-history";
export const PUT = apiHandler(async (request: NextRequest) => {
const { user } = await requireAuth();
@@ -67,6 +68,9 @@ export const PUT = apiHandler(async (request: NextRequest) => {
// Kick a lookback refresh so recent days populate promptly. No-ops cleanly
// when no worker is bound; the nightly cron still covers it.
await enqueueEnvironmentFetch({ userId: user.id });
// And the past: air quality for every earlier day with entries, in the
// background and inside the request budget. A no-op while it is off.
await enqueueAirQualityHistory(user.id);
annotate({ action: { name: "environment.home.set" } });
@@ -20,6 +20,7 @@ import { prisma } from "@/lib/db";
import { environmentPreferencesSchema } from "@/lib/validations/environment";
import { isAirQualityOperatorDisabled } from "@/lib/environment/open-meteo-air-quality";
import { enqueueEnvironmentFetch } from "@/lib/jobs/environment-fetch";
import { enqueueAirQualityHistory } from "@/lib/jobs/environment-air-quality-history";
export const PATCH = apiHandler(async (request: NextRequest) => {
const { user } = await requireAuth();
@@ -57,6 +58,8 @@ export const PATCH = apiHandler(async (request: NextRequest) => {
before?.environmentAirQualityEnabled === false
) {
await enqueueEnvironmentFetch({ userId: user.id });
// Switched on: fill the past as well, every earlier day with entries.
await enqueueAirQualityHistory(user.id);
}
annotate({
+18 -1
View File
@@ -11,7 +11,8 @@
* (the account switch, the operator switch, how many days carry air-quality
* values), the newest stored day for the dashboard chips (with its air
* quality only while the part is on), and every attribution line in
* `attributions`.
* `attributions`, and the progress of the air-quality history backfill
* (`airQuality.history`).
*/
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { apiSuccess } from "@/lib/api-response";
@@ -23,6 +24,7 @@ import { isAirQualityOperatorDisabled } from "@/lib/environment/open-meteo-air-q
import { environmentAttributionLines } from "@/lib/environment/air-quality-contract";
import { readLocation } from "@/lib/environment/location-cipher";
import { isAirQualityActive } from "@/lib/environment/service";
import { readAirQualityHistoryState } from "@/lib/environment/air-quality-history";
import { ENVIRONMENT_FETCH_QUEUE } from "@/lib/jobs/environment-fetch";
import { readQueueFailureForUser } from "@/lib/jobs/job-failures";
@@ -54,6 +56,7 @@ export const GET = apiHandler(async () => {
homeSince: true,
timezone: true,
environmentAirQualityEnabled: true,
environmentAqHistoryJson: true,
},
}),
prisma.environmentTravelLocation.findMany({
@@ -157,6 +160,19 @@ export const GET = apiHandler(async () => {
const accountEnabled = profile?.environmentAirQualityEnabled ?? true;
const operatorDisabled = isAirQualityOperatorDisabled();
const airActive = isAirQualityActive(accountEnabled);
// The history backfill's progress, while the part is on: what the last run
// counted, so the card can say how far the past has been filled.
const historyState = airActive
? readAirQualityHistoryState(profile?.environmentAqHistoryJson)
: null;
const history = historyState
? {
total: historyState.total,
done: historyState.done,
complete: historyState.complete,
checkedAt: historyState.checkedAt,
}
: null;
const latestYear = Number(
(latestAir?.date ?? latest?.date ?? new Date().toISOString()).slice(0, 4),
);
@@ -219,6 +235,7 @@ export const GET = apiHandler(async () => {
days: airDays,
latestDate: latestAir?.date ?? null,
domain: latestAir?.aqDomain ?? null,
history,
},
latestDay,
// null = the last background runs did not fail (or there is no queue to
@@ -92,6 +92,13 @@ interface EnvironmentOverview {
days: number;
latestDate: string | null;
domain: string | null;
/** The history backfill's progress; null before its first run. */
history: {
total: number;
done: number;
complete: boolean;
checkedAt: string;
} | null;
};
/** v1.42 — the newest stored day; `airQuality` null while off/unfetched. */
latestDay: {
@@ -638,6 +645,26 @@ export function EnvironmentSection() {
? ` ${t("settings.sections.environment.airQuality.pollenEuropeOnly")}`
: ""}
</p>
{data.airQuality.history && data.airQuality.history.total > 0 && (
<p
className="text-muted-foreground text-xs"
data-testid="environment-aq-history"
data-complete={data.airQuality.history.complete}
>
{data.airQuality.history.complete
? t(
"settings.sections.environment.airQuality.historyComplete",
{ total: data.airQuality.history.total },
)
: t(
"settings.sections.environment.airQuality.historyProgress",
{
done: data.airQuality.history.done,
total: data.airQuality.history.total,
},
)}
</p>
)}
{data.latestDay?.airQuality && (
<div className="space-y-2">
<p className="text-sm font-medium">
+1
View File
@@ -378,6 +378,7 @@ export const USER_RESET = {
homeSince: null,
homeLocationEncrypted: null,
environmentAirQualityEnabled: true,
environmentAqHistoryJson: Prisma.DbNull,
// Derived AI output cached on the row
insightsPrivacyMode: "aggregated",
@@ -0,0 +1,201 @@
/**
* v1.42 — the air-quality history backfill's pure parts: how pending days
* are cut into request ranges, what a range weighs, the ceilings that keep
* the history from taking the nightly fetch's room, where a day without a
* row is placed, and when an account is offered a run again.
*/
import { describe, expect, it } from "vitest";
import {
AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS,
historyRangeWeight,
historyRequestFits,
planHistoryRanges,
readAirQualityHistoryState,
resolveHistoryLocation,
} from "../air-quality-history";
import { historyDue } from "@/lib/jobs/environment-air-quality-history";
import { shiftDateKey } from "@/lib/tz/format";
function daysFrom(start: string, count: number, step = 1): string[] {
return Array.from({ length: count }, (_, i) => shiftDateKey(start, i * step));
}
describe("planHistoryRanges", () => {
it("joins days up to 14 apart into one range and splits on a longer gap", () => {
const ranges = planHistoryRanges([
"2024-01-01",
"2024-01-15", // 14 days later: joined
"2024-01-30", // 15 days later: new range
"2024-01-31",
]);
expect(ranges.map((r) => [r.startDate, r.endDate, r.days.length])).toEqual([
["2024-01-01", "2024-01-15", 2],
["2024-01-30", "2024-01-31", 2],
]);
});
it("never lets a range grow past 90 days", () => {
const ranges = planHistoryRanges(daysFrom("2023-01-01", 200));
expect(ranges).toHaveLength(3);
for (const r of ranges) {
expect(shiftDateKey(r.startDate, 89) >= r.endDate).toBe(true);
}
expect(ranges.flatMap((r) => r.days)).toHaveLength(200);
});
it("sorts and de-duplicates its input", () => {
const ranges = planHistoryRanges([
"2024-03-02",
"2024-03-01",
"2024-03-02",
]);
expect(ranges).toEqual([
{
startDate: "2024-03-01",
endDate: "2024-03-02",
days: ["2024-03-01", "2024-03-02"],
},
]);
});
it("spends fewer calls on a sparse history than one request per day", () => {
// A day every three days for a year: 122 days.
const days = daysFrom("2023-01-01", 122, 3);
const ranges = planHistoryRanges(days);
const ranged = ranges.reduce(
(sum, r) => sum + historyRangeWeight(r, []),
0,
);
const perDay =
days.length *
historyRangeWeight(
{ startDate: days[0], endDate: days[0], days: [days[0]] },
[],
);
expect(ranged).toBeLessThan(perDay / 4);
});
});
describe("historyRangeWeight", () => {
it("adds the weather only over the span of the days without a row", () => {
const range = {
startDate: "2023-01-01",
endDate: "2023-03-31",
days: ["2023-01-01", "2023-03-31"],
};
const airOnly = historyRangeWeight(range, []);
expect(airOnly).toBeCloseTo((1.7 * 90) / 14);
// One missing day: one weather request at its minimum weight (12 vars).
expect(historyRangeWeight(range, ["2023-03-31"])).toBeCloseTo(
airOnly + 1.2,
);
});
});
describe("historyRequestFits", () => {
it("admits a request under every ceiling", () => {
expect(historyRequestFits({}, 10)).toBe(true);
});
it("holds the account to three quarters of its daily share", () => {
expect(AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS).toBe(300);
expect(historyRequestFits({ "account-day": 295 }, 5)).toBe(true);
expect(historyRequestFits({ "account-day": 295 }, 5.1)).toBe(false);
});
it("holds the history to half of each instance window", () => {
expect(historyRequestFits({ minute: 245 }, 5)).toBe(true);
expect(historyRequestFits({ minute: 246 }, 5)).toBe(false);
expect(historyRequestFits({ hour: 1999 }, 2)).toBe(false);
expect(historyRequestFits({ day: 3995 }, 10)).toBe(false);
});
});
describe("resolveHistoryLocation", () => {
const home = {
lat: 51.5,
lon: 7.2,
label: "Home",
timezone: "Europe/Berlin",
since: "2026-09-01",
};
const trip = {
startDate: "2024-07-01",
endDate: "2024-07-14",
lat: 38.7,
lon: -9.1,
label: "Lisbon",
};
it("places a day of a trip at the trip", () => {
expect(resolveHistoryLocation("2024-07-05", home, [trip])).toMatchObject({
lat: 38.7,
source: "TRAVEL",
});
});
it("places a day before the home was set at the home", () => {
expect(resolveHistoryLocation("2020-03-01", home, [trip])).toMatchObject({
lat: 51.5,
source: "HOME",
});
});
it("places nothing without a home or a period", () => {
expect(resolveHistoryLocation("2020-03-01", null, [trip])).toBeNull();
});
});
describe("stored progress", () => {
const now = new Date("2026-10-09T03:00:00Z");
it("reads a valid state and caps done at total", () => {
expect(
readAirQualityHistoryState({
version: 1,
total: 10,
done: 12,
checkedAt: now.toISOString(),
complete: false,
}),
).toMatchObject({ total: 10, done: 10 });
expect(readAirQualityHistoryState({ total: "x" })).toBeNull();
expect(readAirQualityHistoryState(null)).toBeNull();
});
it("offers an account with no, an unfinished or a stale history", () => {
expect(historyDue(null, now)).toBe(true);
expect(
historyDue(
{ total: 5, done: 2, checkedAt: now.toISOString(), complete: false },
now,
),
).toBe(true);
expect(
historyDue(
{
total: 5,
done: 5,
checkedAt: "2026-09-30T03:00:00Z",
complete: true,
},
now,
),
).toBe(true);
});
it("leaves a history that is through and checked this week alone", () => {
expect(
historyDue(
{
total: 5,
done: 5,
checkedAt: "2026-10-07T03:00:00Z",
complete: true,
},
now,
),
).toBe(false);
});
});
@@ -29,6 +29,8 @@ import {
ozoneMax8h,
uvIndexMax,
variablesForChunk,
airQualityChunkWeight,
earliestAirQualityDay,
type AirQualityHourly,
} from "../open-meteo-air-quality";
@@ -172,14 +174,24 @@ describe("aggregateAirQuality", () => {
});
describe("requests", () => {
it("splits into 90-day chunks and asks the pre-2022 ones for the pollutants only", () => {
it("splits into 90-day chunks and asks each era only for what it carries", () => {
expect(chunkDays("2022-01-01", "2022-07-01")).toEqual([
{ startDate: "2022-01-01", endDate: "2022-03-31" },
{ startDate: "2022-04-01", endDate: "2022-06-29" },
{ startDate: "2022-06-30", endDate: "2022-07-01" },
]);
expect(variablesForChunk("2022-03-31")).toHaveLength(8);
expect(variablesForChunk("2022-06-29")).toHaveLength(17);
// Pollutants and indices alone before 2021.
expect(variablesForChunk("2020-12-31")).toHaveLength(8);
// Pollen and dust join in 2021 (Europe).
expect(variablesForChunk("2021-01-01")).toHaveLength(15);
expect(variablesForChunk("2022-07-31")).toHaveLength(15);
// UV and aerosol depth from CAMS global, August 2022 on.
expect(variablesForChunk("2022-08-01")).toHaveLength(17);
// Weight: 90 days of the full set is 1.7 × 90 / 14 calls.
expect(airQualityChunkWeight("2023-01-01", "2023-03-31")).toBeCloseTo(
(1.7 * 90) / 14,
);
expect(airQualityChunkWeight("2015-01-01", "2015-01-01")).toBe(1);
});
it("sends the coarse coordinates, the timezone and domains=auto, nothing else", async () => {
@@ -268,6 +280,28 @@ describe("requests", () => {
]);
});
it("answers days before August 2022 outside Europe as uncoverable without a request", async () => {
const fetchSpy = vi.fn(async (_url: unknown) =>
Response.json({ hourly: dayFixture("2022-08-01", 10) }),
);
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchDailyAirQuality({
lat: 40.7,
lon: -74,
timezone: "UTC",
startDate: "2022-07-30",
endDate: "2022-08-01",
});
expect(fetchSpy).toHaveBeenCalledTimes(1);
const url = new URL(String(fetchSpy.mock.calls[0]![0]));
expect(url.searchParams.get("start_date")).toBe("2022-08-01");
expect(
result.days.filter((d) => d.aqHours === 0).map((d) => d.date),
).toEqual(["2022-07-30", "2022-07-31"]);
expect(earliestAirQualityDay(40.7, -74)).toBe("2022-08-01");
expect(earliestAirQualityDay(51.5, 7.2)).toBe("2013-01-01");
});
it("honours a self-hosted endpoint", async () => {
vi.resetModules();
vi.stubEnv("OPENMETEO_AIR_QUALITY_URL", "https://aq.example.test/");
+583
View File
@@ -0,0 +1,583 @@
/**
* v1.42 — the air-quality history backfill: air quality, pollen and UV for
* every past local day an account has entries on, not only from the day the
* module was switched on.
*
* ## Which days
*
* Every local day (in the zone the module keys its rows on: the home's, else
* the account's) with at least one measurement, mood entry, taken dose,
* symptom or workout, from the first day the feed serves up to eight days
* ago. The last week belongs to the nightly fetch, whose lookback already
* covers it (`ENVIRONMENT_LOOKBACK_DAYS`). A day is placed:
*
* - on its stored row's location when the day already has a weather row
* (only the air-quality part is fetched then, as the nightly gap fill
* does);
* - otherwise in the dated location period that covers it, else at the
* home. Unlike the nightly resolver this also places days before
* `homeSince` at the home: those are exactly the days the history is for,
* and a period entered for a trip still wins. Such a day gets its weather
* row with the air quality, because the air quality is stored on it.
*
* A day inside a period whose sealed location does not open is skipped (fail
* closed, as in the service), and so is a day with no place at all. A day the
* source does not reach at its location (before 2013 in Europe, before August
* 2022 elsewhere, `earliestAirQualityDay`) is neither fetched nor counted: it
* stays without a value, which says nothing about the air, rather than
* carrying a zero.
*
* ## Requests
*
* The pending days are grouped by coarse location (the stored or resolved
* one, never finer: the same rounded coordinates the module always sends)
* and cut into ranges of at most 90 days, joining two days into one range
* when they lie at most 14 days apart. The feed weighs a request by days
* over 14 (`openMeteoCallWeight`), so a lone day costs as much as two weeks:
* joining short gaps is free, splitting long ones saves the gap. The newest
* ranges go first, so the window the correlations read fills first. The
* weather of a range is asked only over the span of its days without a row.
*
* ## Budget
*
* Every request still passes the instance budget and the account's daily
* share (`request-budget.ts`). On top of that the history holds itself to
* {@link AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS} of the account's 400 daily
* calls and to half of each instance window, so the nightly fetch, the gap
* fill and a backfill someone asked for keep their room. A run that reaches a
* ceiling stops; the nightly discovery picks the account up again, so a long
* history spreads over several days by construction.
*
* ## Idempotence and progress
*
* A day is done once its row carries `aqFetchedAt`; done days are never asked
* for again, and the writes only touch rows still without it (or create the
* row). Each run stores `{ total, done, checkedAt, complete }` on
* `User.environmentAqHistoryJson`, which the settings card shows as
* "Backfilling air quality: x of y days".
*/
import { Prisma } from "@/generated/prisma/client";
import { prisma, toJson } from "@/lib/db";
import {
airQualityChunkWeight,
earliestAirQualityDay,
fetchDailyAirQuality,
isAirQualityActive,
uncoverableDay,
AIR_QUALITY_CHUNK_DAYS,
AIR_QUALITY_EARLIEST_DAY,
type DailyAirQualityObservation,
} from "@/lib/environment/open-meteo-air-quality";
import {
archiveRequestWeight,
fetchDailyEnvironment,
type DailyEnvironmentObservation,
} from "@/lib/environment/open-meteo";
import {
OPEN_METEO_ACCOUNT_DAY_WINDOW,
OPEN_METEO_BUDGET_WINDOWS,
OpenMeteoBudgetExhaustedError,
readOpenMeteoBudgetUsage,
type BudgetUsage,
} from "@/lib/environment/request-budget";
import { readLocation, sealLocation } from "@/lib/environment/location-cipher";
import {
ENVIRONMENT_LOOKBACK_DAYS,
HOME_SELECT,
airQualityColumns,
buildUpsert,
inAnyPeriod,
readTravelOverrides,
resolveHome,
utcDayKey,
type HomeLocation,
type ResolvedLocation,
type TravelOverride,
} from "@/lib/environment/service";
import { shiftDateKey } from "@/lib/tz/format";
import { startOfLocalDayKey } from "@/lib/tz/local-day";
/** Days before today the history stops at; the nightly lookback has the rest. */
export const AIR_QUALITY_HISTORY_SETTLE_DAYS = ENVIRONMENT_LOOKBACK_DAYS + 1;
/** Longest range one request covers (one feed chunk). */
export const AIR_QUALITY_HISTORY_RANGE_DAYS = AIR_QUALITY_CHUNK_DAYS;
/** Two days at most this far apart share a range (14 days weigh one call). */
export const AIR_QUALITY_HISTORY_JOIN_GAP_DAYS = 14;
/** Ranges one run works through before it hands over to its follow-up. */
export const AIR_QUALITY_HISTORY_RANGES_PER_RUN = 8;
/** The history's ceiling inside the account's 400 daily calls. */
export const AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS = Math.floor(
OPEN_METEO_ACCOUNT_DAY_WINDOW.limit * 0.75,
);
/** The history's share of each instance-wide window. */
export const AIR_QUALITY_HISTORY_INSTANCE_SHARE = 0.5;
/** One pending day and where it is fetched. */
export interface HistoryDay {
date: string;
location: ResolvedLocation;
/** True when a weather row exists and only its air quality is missing. */
hasRow: boolean;
}
/** A contiguous request range over a sorted set of pending days. */
export interface HistoryRange {
startDate: string;
endDate: string;
days: string[];
}
/**
* Cut sorted day keys into request ranges: a new range when the next day lies
* more than `joinGapDays` after the previous one, or when the range would
* grow past `maxRangeDays`. Pure; exported for tests.
*/
export function planHistoryRanges(
days: readonly string[],
opts: { joinGapDays?: number; maxRangeDays?: number } = {},
): HistoryRange[] {
const joinGap = opts.joinGapDays ?? AIR_QUALITY_HISTORY_JOIN_GAP_DAYS;
const maxRange = opts.maxRangeDays ?? AIR_QUALITY_HISTORY_RANGE_DAYS;
const ranges: HistoryRange[] = [];
for (const day of [...new Set(days)].sort()) {
const current = ranges[ranges.length - 1];
if (
current &&
shiftDateKey(current.endDate, joinGap) >= day &&
shiftDateKey(current.startDate, maxRange - 1) >= day
) {
current.endDate = day;
current.days.push(day);
} else {
ranges.push({ startDate: day, endDate: day, days: [day] });
}
}
return ranges;
}
/**
* The weight of one range in calls: its air-quality request, plus the
* weather request over the span of its days without a row, when it has any.
*/
export function historyRangeWeight(
range: HistoryRange,
missingRowDays: readonly string[],
): number {
const air = airQualityChunkWeight(range.startDate, range.endDate);
if (missingRowDays.length === 0) return air;
const sorted = [...missingRowDays].sort();
return air + archiveRequestWeight(sorted[0], sorted[sorted.length - 1]);
}
/**
* Whether a request of `weight` calls fits under the history's ceilings, given
* what each window already holds. Pure; exported for tests.
*/
export function historyRequestFits(
usage: BudgetUsage,
weight: number,
): boolean {
for (const w of OPEN_METEO_BUDGET_WINDOWS) {
const ceiling = w.limit * AIR_QUALITY_HISTORY_INSTANCE_SHARE;
if ((usage[w.name] ?? 0) + weight > ceiling) return false;
}
return (
(usage["account-day"] ?? 0) + weight <=
AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS
);
}
/**
* Where a day without a row is placed: the period that covers it, else the
* home, also before `homeSince` (see the header). Pure; exported for tests.
*/
export function resolveHistoryLocation(
day: string,
home: HomeLocation | null,
travels: readonly TravelOverride[],
): ResolvedLocation | null {
for (const t of travels) {
if (day >= t.startDate && day <= t.endDate) {
return { lat: t.lat, lon: t.lon, label: t.label, source: "TRAVEL" };
}
}
if (home) {
return { lat: home.lat, lon: home.lon, label: home.label, source: "HOME" };
}
return null;
}
/** The stored progress, as the settings card reads it. */
export interface AirQualityHistoryState {
version: 1;
/** Past days with entries the source can cover at their place. */
total: number;
/** Of those, the days whose air quality is stored. */
done: number;
/** ISO instant of the run that counted them. */
checkedAt: string;
/** True when nothing is left that the source can serve. */
complete: boolean;
}
/** Tolerant read of the stored progress; null for anything malformed. */
export function readAirQualityHistoryState(
raw: unknown,
): AirQualityHistoryState | null {
if (raw === null || typeof raw !== "object") return null;
const r = raw as Record<string, unknown>;
const count = (v: unknown) =>
typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : null;
const total = count(r.total);
const done = count(r.done);
if (
total === null ||
done === null ||
typeof r.checkedAt !== "string" ||
typeof r.complete !== "boolean"
) {
return null;
}
return {
version: 1,
total,
done: Math.min(done, total),
checkedAt: r.checkedAt,
complete: r.complete,
};
}
/**
* The local days in `[fromKey, toKey]` with at least one entry: a reading, a
* mood entry, a taken dose, a symptom or a workout. Distinct in SQL, so a
* dense record costs a scan, not a transfer. `tz` is a zone the account
* stored (home or profile), bound as a parameter.
*/
export async function readEntryDays(
userId: string,
tz: string,
fromKey: string,
toKey: string,
): Promise<string[]> {
// The instants of the first local day's start and the day after the last.
const from = startOfLocalDayKey(fromKey, tz);
const to = startOfLocalDayKey(shiftDateKey(toKey, 1), tz);
const rows = await prisma.$queryRaw<Array<{ day: string }>>`
SELECT day FROM (
SELECT DISTINCT to_char(("measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS day
FROM "measurements"
WHERE "user_id" = ${userId} AND "deleted_at" IS NULL
AND "measured_at" >= ${from} AND "measured_at" < ${to}
UNION
SELECT "date" AS day FROM "mood_entries"
WHERE "user_id" = ${userId} AND "deleted_at" IS NULL
AND "date" >= ${fromKey} AND "date" <= ${toKey}
UNION
SELECT to_char(("taken_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS day
FROM "medication_intake_events"
WHERE "user_id" = ${userId} AND "deleted_at" IS NULL
AND "taken_at" >= ${from} AND "taken_at" < ${to}
UNION
SELECT to_char(("occurred_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS day
FROM "symptom_events"
WHERE "user_id" = ${userId}
AND "occurred_at" >= ${from} AND "occurred_at" < ${to}
UNION
SELECT to_char(("started_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS day
FROM "workouts"
WHERE "user_id" = ${userId}
AND "started_at" >= ${from} AND "started_at" < ${to}
) entry_days
`;
return rows
.map((r) => r.day)
.filter((d) => d >= fromKey && d <= toKey)
.sort();
}
/** What the history still has to do for an account, and how far it is. */
export interface HistoryWork {
pending: HistoryDay[];
total: number;
done: number;
timezone: string;
}
/**
* Collect the pending days and the counts. Null when the account is gone or
* its air quality is off (for the account or the instance).
*/
export async function collectAirQualityHistory(
userId: string,
now: Date = new Date(),
): Promise<HistoryWork | null> {
const user = await prisma.user.findUnique({
where: { id: userId },
select: HOME_SELECT,
});
if (!user || !isAirQualityActive(user.environmentAirQualityEnabled)) {
return null;
}
const home = resolveHome(user);
const travel = await readTravelOverrides(userId);
const timezone = home?.timezone ?? user.timezone;
const cutoff = shiftDateKey(utcDayKey(now), -AIR_QUALITY_HISTORY_SETTLE_DAYS);
if (cutoff < AIR_QUALITY_EARLIEST_DAY) {
return { pending: [], total: 0, done: 0, timezone };
}
const [entryDays, rows] = await Promise.all([
readEntryDays(userId, timezone, AIR_QUALITY_EARLIEST_DAY, cutoff),
prisma.environmentContext.findMany({
where: { userId, date: { gte: AIR_QUALITY_EARLIEST_DAY, lte: cutoff } },
select: {
date: true,
aqFetchedAt: true,
lat: true,
lon: true,
locationLabel: true,
locationEncrypted: true,
source: true,
},
}),
]);
const rowByDate = new Map(rows.map((r) => [r.date, r]));
const pending: HistoryDay[] = [];
let total = 0;
let done = 0;
for (const date of entryDays) {
const row = rowByDate.get(date);
let location: ResolvedLocation | null;
if (row) {
const stored = readLocation({
sealed: row.locationEncrypted,
lat: row.lat,
lon: row.lon,
label: row.locationLabel,
});
location = stored ? { ...stored, source: row.source } : null;
} else if (inAnyPeriod(date, travel.unreadable)) {
location = null;
} else {
location = resolveHistoryLocation(date, home, travel.overrides);
}
if (!location) continue;
// Outside what the source serves at this place: left empty, not counted.
if (date < earliestAirQualityDay(location.lat, location.lon)) continue;
total += 1;
if (row?.aqFetchedAt) {
done += 1;
} else {
pending.push({ date, location, hasRow: row != null });
}
}
return { pending, total, done, timezone };
}
/** A weather observation with nothing in it, for a day the archive skipped. */
function emptyWeather(date: string): DailyEnvironmentObservation {
return {
date,
tempMin: null,
tempMax: null,
tempMean: null,
apparentMean: null,
apparentMax: null,
sunshineSec: null,
daylightSec: null,
precipSum: null,
pressureMean: null,
pressureDelta: null,
humidityMean: null,
cloudMean: null,
weatherCode: null,
};
}
export interface AirQualityHistoryRunResult {
/** `inactive`: the account is gone or its air quality is off. */
status: "inactive" | "complete" | "progress" | "budget" | "error";
/** Days whose air quality this run stored. */
filled: number;
/** Requests sent. */
fetches: number;
total: number;
done: number;
/** Days still pending after this run. */
remaining: number;
}
/**
* One run of the history for an account: up to `maxRanges` ranges, newest
* first, inside the history's budget ceilings. Stores the progress and says
* why it stopped.
*/
export async function runAirQualityHistory(
userId: string,
opts: { now?: Date; maxRanges?: number } = {},
): Promise<AirQualityHistoryRunResult> {
const now = opts.now ?? new Date();
const maxRanges = opts.maxRanges ?? AIR_QUALITY_HISTORY_RANGES_PER_RUN;
const work = await collectAirQualityHistory(userId, now);
if (!work) {
return {
status: "inactive",
filled: 0,
fetches: 0,
total: 0,
done: 0,
remaining: 0,
};
}
// Group by the coarse coordinates the request carries, then plan ranges.
const groups = new Map<
string,
{ lat: number; lon: number; byDate: Map<string, HistoryDay> }
>();
for (const day of work.pending) {
const key = `${day.location.lat},${day.location.lon}`;
const group = groups.get(key) ?? {
lat: day.location.lat,
lon: day.location.lon,
byDate: new Map(),
};
group.byDate.set(day.date, day);
groups.set(key, group);
}
const planned = [...groups.values()]
.flatMap((group) =>
planHistoryRanges([...group.byDate.keys()]).map((range) => ({
group,
range,
})),
)
.sort((a, b) => b.range.endDate.localeCompare(a.range.endDate));
let filled = 0;
let fetches = 0;
let stop: "budget" | "error" | null = null;
let errors = 0;
let attempted = 0;
for (const { group, range } of planned) {
if (attempted >= maxRanges) break;
attempted += 1;
const days = range.days.map((d) => group.byDate.get(d)!);
const missing = days.filter((d) => !d.hasRow).map((d) => d.date);
const usage = await readOpenMeteoBudgetUsage(userId, new Date());
if (!historyRequestFits(usage, historyRangeWeight(range, missing))) {
stop = "budget";
break;
}
const air = await fetchDailyAirQuality(
{
lat: group.lat,
lon: group.lon,
timezone: work.timezone,
startDate: range.startDate,
endDate: range.endDate,
},
{ accountId: userId },
);
if (air.stopped === "budget") {
stop = "budget";
break;
}
if (air.stopped === "error") {
errors += 1;
continue;
}
fetches += 1;
const airByDay = new Map<string, DailyAirQualityObservation>(
air.days.map((d) => [d.date, d]),
);
const airFor = (date: string) => airByDay.get(date) ?? uncoverableDay(date);
let weather: Map<string, DailyEnvironmentObservation> | null = null;
if (missing.length > 0) {
try {
const observations = await fetchDailyEnvironment(
{
lat: group.lat,
lon: group.lon,
timezone: work.timezone,
startDate: missing[0],
endDate: missing[missing.length - 1],
},
{ accountId: userId },
);
fetches += 1;
weather = new Map(observations.map((o) => [o.date, o]));
} catch (err) {
if (err instanceof OpenMeteoBudgetExhaustedError) {
stop = "budget";
} else {
errors += 1;
}
// The days with a row are stored below; those without one stay
// pending until the weather can be fetched with them.
}
}
const fetchedAt = new Date();
const ops: Prisma.PrismaPromise<unknown>[] = [];
for (const day of days) {
if (day.hasRow) {
ops.push(
prisma.environmentContext.updateMany({
// Still unfetched, so a weather rewrite that landed meanwhile
// (with its own air quality) is not overwritten.
where: { userId, date: day.date, aqFetchedAt: null },
data: airQualityColumns(airFor(day.date), fetchedAt),
}),
);
} else if (weather) {
ops.push(
buildUpsert(
userId,
day.location,
sealLocation(day.location),
weather.get(day.date) ?? emptyWeather(day.date),
{ active: true, day: airFor(day.date), now: fetchedAt },
),
);
}
}
if (ops.length > 0) {
const results = await prisma.$transaction(ops);
for (const r of results) {
filled +=
r && typeof r === "object" && "count" in r
? Number((r as { count: number }).count)
: 1;
}
}
if (stop) break;
}
const done = Math.min(work.total, work.done + filled);
const remaining = work.total - done;
const state: AirQualityHistoryState = {
version: 1,
total: work.total,
done,
checkedAt: now.toISOString(),
complete: remaining === 0,
};
await prisma.user.update({
where: { id: userId },
data: { environmentAqHistoryJson: toJson(state) },
});
const status: AirQualityHistoryRunResult["status"] =
remaining === 0
? "complete"
: stop === "budget"
? "budget"
: errors > 0 && filled === 0
? "error"
: "progress";
return { status, filled, fetches, total: work.total, done, remaining };
}
+66 -33
View File
@@ -22,14 +22,20 @@
*
* ## Coverage, and why a missing value is null and never zero
*
* The feed starts on 2013-01-01; pollen, UV, dust and aerosol depth only from
* about mid 2022, and pollen only in Europe. A value the feed did not cover is
* stored as null. A chunk that ends before {@link AIR_QUALITY_FULL_FEED_FROM}
* asks only for the eight pollutant and index variables, which the feed has
* for that era; that keeps the request weight at 1.0 instead of 1.7 per ten
* variables. A day before {@link AIR_QUALITY_EARLIEST_DAY} cannot be fetched
* at all and is returned as uncoverable (every value null, no hours), so the
* nightly gap fill does not ask for it again.
* What the feed serves (open-meteo.com/en/docs/air-quality-api, checked
* against the live API in v1.42): inside Europe the CAMS European reanalysis
* reaches back to 2013-01-01 and the forecast takes over from October 2023;
* outside Europe only CAMS global exists, from 2022-08-01. Pollen is Europe
* only and present from 2021 (in season); UV and aerosol depth come from CAMS
* global, so from 2022-08-01. A value the feed did not cover is stored as
* null. The variables asked for follow the era of a chunk's last day
* ({@link variablesForChunk}): the eight pollutant and index variables before
* {@link AIR_QUALITY_POLLEN_FROM}, plus pollen and dust until
* {@link AIR_QUALITY_GLOBAL_FROM}, everything after, which keeps the
* request weight down where the extra variables would come back empty. A day
* before the first day the feed serves at the location
* ({@link earliestAirQualityDay}) is not requested at all and is returned as
* uncoverable (every value null, no hours), so nobody asks for it again.
*
* ## Daily rules (a local day of hourly values)
*
@@ -90,13 +96,15 @@ export function isAirQualityActive(accountEnabled: boolean): boolean {
return accountEnabled && !isAirQualityOperatorDisabled();
}
/** The first day the feed serves. */
/** The first day the feed serves (the CAMS European reanalysis). */
export const AIR_QUALITY_EARLIEST_DAY = "2013-01-01";
/**
* From about here the feed carries pollen, UV, dust and aerosol depth. A
* chunk ending before it asks for the pollutant and index variables only.
* The first day CAMS global serves, the only model outside Europe, and so the
* first day with every variable (UV and aerosol depth come from it).
*/
export const AIR_QUALITY_FULL_FEED_FROM = "2022-06-01";
export const AIR_QUALITY_GLOBAL_FROM = "2022-08-01";
/** From here the European feed carries pollen (and dust). */
export const AIR_QUALITY_POLLEN_FROM = "2021-01-01";
/** Days per request: bounds the response (about 300 KB) and the heap. */
export const AIR_QUALITY_CHUNK_DAYS = 90;
@@ -112,11 +120,9 @@ const POLLUTANT_VARIABLES = [
"us_aqi",
] as const;
/** The nine variables the feed fills only from about mid 2022. */
const RECENT_VARIABLES = [
"uv_index",
/** The seven variables the European feed fills from 2021. */
const POLLEN_ERA_VARIABLES = [
"dust",
"aerosol_optical_depth",
"alder_pollen",
"birch_pollen",
"grass_pollen",
@@ -125,13 +131,23 @@ const RECENT_VARIABLES = [
"ragweed_pollen",
] as const;
/** The two variables that come from CAMS global, so from August 2022. */
const GLOBAL_ERA_VARIABLES = ["uv_index", "aerosol_optical_depth"] as const;
type HourlyVariable =
(typeof POLLUTANT_VARIABLES)[number] | (typeof RECENT_VARIABLES)[number];
| (typeof POLLUTANT_VARIABLES)[number]
| (typeof POLLEN_ERA_VARIABLES)[number]
| (typeof GLOBAL_ERA_VARIABLES)[number];
const POLLEN_ERA_REQUEST: readonly HourlyVariable[] = [
...POLLUTANT_VARIABLES,
...POLLEN_ERA_VARIABLES,
];
/** All 17 hourly variables, in request order. */
export const AIR_QUALITY_VARIABLES: readonly HourlyVariable[] = [
...POLLUTANT_VARIABLES,
...RECENT_VARIABLES,
...POLLEN_ERA_REQUEST,
...GLOBAL_ERA_VARIABLES,
];
/** One stored day's air-quality part, as the feed produced it. */
@@ -211,6 +227,16 @@ export function insideCamsEurope(lat: number, lon: number): boolean {
return lat >= 30 && lat <= 72 && lon >= -25 && lon <= 45;
}
/**
* The first day the feed has air quality for at a coarse location: 2013
* inside the CAMS Europe domain, August 2022 (CAMS global) outside it.
*/
export function earliestAirQualityDay(lat: number, lon: number): string {
return insideCamsEurope(lat, lon)
? AIR_QUALITY_EARLIEST_DAY
: AIR_QUALITY_GLOBAL_FROM;
}
/**
* Fold an hourly response into one observation per local day present in it.
* Pure; exported for tests. Times are the feed's local `YYYY-MM-DDTHH:MM`;
@@ -326,11 +352,22 @@ export function chunkDays(
return chunks;
}
/** The variables a chunk asks for: the pollutant set alone before the full era. */
/** The variables a chunk asks for, by the era of its last day. */
export function variablesForChunk(endDate: string): readonly HourlyVariable[] {
return endDate < AIR_QUALITY_FULL_FEED_FROM
? POLLUTANT_VARIABLES
: AIR_QUALITY_VARIABLES;
if (endDate < AIR_QUALITY_POLLEN_FROM) return POLLUTANT_VARIABLES;
if (endDate < AIR_QUALITY_GLOBAL_FROM) return POLLEN_ERA_REQUEST;
return AIR_QUALITY_VARIABLES;
}
/** The request weight of one chunk, in calls (`openMeteoCallWeight`). */
export function airQualityChunkWeight(
startDate: string,
endDate: string,
): number {
return openMeteoCallWeight(
variablesForChunk(endDate).length,
enumerateDayCount(startDate, endDate),
);
}
/** Why a fetch stopped before its last chunk. */
@@ -361,27 +398,23 @@ export async function fetchDailyAirQuality(
budget: { accountId?: string } = {},
): Promise<AirQualityFetchResult> {
const days: DailyAirQualityObservation[] = [];
// Days before the feed's first day are answered here, without a request.
// Days before the feed's first day at this location are answered here,
// without a request.
const earliest = earliestAirQualityDay(args.lat, args.lon);
for (
let day = args.startDate;
day <= args.endDate && day < AIR_QUALITY_EARLIEST_DAY;
day <= args.endDate && day < earliest;
day = shiftDateKey(day, 1)
) {
days.push(uncoverableDay(day));
}
const start =
args.startDate < AIR_QUALITY_EARLIEST_DAY
? AIR_QUALITY_EARLIEST_DAY
: args.startDate;
const start = args.startDate < earliest ? earliest : args.startDate;
if (start > args.endDate) return { days, stopped: null };
const chunks = chunkDays(start, args.endDate).reverse();
for (const chunk of chunks) {
const variables = variablesForChunk(chunk.endDate);
const weight = openMeteoCallWeight(
variables.length,
enumerateDayCount(chunk.startDate, chunk.endDate),
);
const weight = airQualityChunkWeight(chunk.startDate, chunk.endDate);
if (!(await reserveOpenMeteoCalls(weight, budget.accountId))) {
return { days, stopped: "budget" };
}
+12 -4
View File
@@ -149,6 +149,17 @@ const HOURLY_FIELDS = HOURLY_FIELD_LIST.join(",");
const ARCHIVE_VARIABLE_COUNT =
DAILY_FIELD_LIST.length + HOURLY_FIELD_LIST.length;
/** The request weight of one archive request over a day range, in calls. */
export function archiveRequestWeight(
startDate: string,
endDate: string,
): number {
return openMeteoCallWeight(
ARCHIVE_VARIABLE_COUNT,
enumerateDayCount(startDate, endDate),
);
}
/** Per-day aggregation accumulator for the hourly-only fields. */
interface HourlyDayAgg {
pressures: number[];
@@ -269,10 +280,7 @@ export async function fetchDailyEnvironment(
},
budget: { accountId?: string } = {},
): Promise<DailyEnvironmentObservation[]> {
const weight = openMeteoCallWeight(
ARCHIVE_VARIABLE_COUNT,
enumerateDayCount(args.startDate, args.endDate),
);
const weight = archiveRequestWeight(args.startDate, args.endDate);
if (!(await reserveOpenMeteoCalls(weight, budget.accountId))) {
throw new OpenMeteoBudgetExhaustedError();
}
+36
View File
@@ -171,3 +171,39 @@ export class OpenMeteoBudgetExhaustedError extends Error {
this.name = "OpenMeteoBudgetExhaustedError";
}
}
/** Calls already charged to each live window, by window name. */
export type BudgetUsage = Partial<Record<BudgetWindow["name"], number>>;
/**
* Read how much of each window is in use, in calls, without charging
* anything. A window never written or already reset reads as absent (zero).
* With `accountId` the account's daily share is read as well. Used by the
* air-quality history backfill, which holds itself under a ceiling of its own
* so the nightly fetch and on-demand requests keep their room.
*/
export async function readOpenMeteoBudgetUsage(
accountId?: string,
now: Date = new Date(),
): Promise<BudgetUsage> {
const keys = new Map<string, BudgetWindow["name"]>(
OPEN_METEO_BUDGET_WINDOWS.map((w) => [
`${BUCKET_PREFIX}:${w.name}`,
w.name,
]),
);
if (accountId) {
keys.set(`${BUCKET_PREFIX}:account:${accountId}:day`, "account-day");
}
const rows = await prisma.rateLimit.findMany({
where: { key: { in: [...keys.keys()] } },
select: { key: true, count: true, resetAt: true },
});
const usage: BudgetUsage = {};
for (const row of rows) {
const name = keys.get(row.key);
if (!name || row.resetAt.getTime() <= now.getTime()) continue;
usage[name] = row.count / CENTI;
}
return usage;
}
+14 -10
View File
@@ -56,14 +56,14 @@ export const ENVIRONMENT_LOOKBACK_DAYS = 7;
export const ENVIRONMENT_MAX_BACKFILL_DAYS = 730;
/** A coarse resolved location for a given day. */
interface ResolvedLocation {
export interface ResolvedLocation {
lat: number;
lon: number;
label: string;
source: EnvironmentLocationSource;
}
interface HomeLocation {
export interface HomeLocation {
lat: number;
lon: number;
label: string;
@@ -72,7 +72,7 @@ interface HomeLocation {
since: string | null;
}
interface TravelOverride {
export interface TravelOverride {
startDate: string;
endDate: string;
lat: number;
@@ -164,7 +164,7 @@ export interface FetchAndStoreResult {
}
/** The user columns every environment read resolves the home from. */
const HOME_SELECT = {
export const HOME_SELECT = {
homeLat: true,
homeLon: true,
homeLabel: true,
@@ -208,7 +208,7 @@ export function resolveHome(user: {
}
/** A location period whose sealed location this host cannot open. */
interface UnreadablePeriod {
export interface UnreadablePeriod {
startDate: string;
endDate: string;
}
@@ -220,7 +220,7 @@ interface UnreadablePeriod {
* period it cannot read still says "this account was not at home on these
* days", so those days must not fall through to the home location.
*/
async function readTravelOverrides(userId: string): Promise<{
export async function readTravelOverrides(userId: string): Promise<{
overrides: TravelOverride[];
unreadable: UnreadablePeriod[];
}> {
@@ -260,7 +260,7 @@ async function readTravelOverrides(userId: string): Promise<{
}
/** True when `day` falls inside one of `periods`. */
function inAnyPeriod(day: string, periods: readonly UnreadablePeriod[]) {
export function inAnyPeriod(day: string, periods: readonly UnreadablePeriod[]) {
return periods.some((p) => day >= p.startDate && day <= p.endDate);
}
@@ -401,7 +401,10 @@ export async function fetchAndStoreEnvironment(args: {
}
/** The air-quality columns of a stored day, from one observation. */
function airQualityColumns(day: DailyAirQualityObservation, fetchedAt: Date) {
export function airQualityColumns(
day: DailyAirQualityObservation,
fetchedAt: Date,
) {
return {
pm25Mean: day.pm25Mean,
pm25Max: day.pm25Max,
@@ -453,7 +456,7 @@ const AIR_QUALITY_UNFETCHED = {
} as const;
/** Build the idempotent per-day upsert operation (un-awaited, for a batch). */
function buildUpsert(
export function buildUpsert(
userId: string,
loc: ResolvedLocation,
sealed: Uint8Array<ArrayBuffer>,
@@ -523,7 +526,8 @@ export interface AirQualityGapFillResult {
* null), newest first, so the correlation window fills first and an account
* with two years of weather catches up within about nine nights. Each day is
* fetched at the location it was stored for (opened from its sealed copy),
* never re-resolved. A day the feed cannot serve (before 2013) is marked
* never re-resolved. A day the feed cannot serve (before 2013 in Europe,
* before August 2022 elsewhere) is marked
* fetched with every value null, so it is not asked for again. A no-op when
* the air-quality part is off for the account or the instance.
*/
+3
View File
@@ -909,6 +909,9 @@ export const USER_COLUMN_BACKUP_CLASS = {
// The restore stamps its own: paired clients' cursors from before it must
// be refused, which a carried value would undo.
syncResetAt: "OPERATIONAL",
// v1.42 — the air-quality history backfill's progress; the restored
// account's job recounts it on its next run.
environmentAqHistoryJson: "OPERATIONAL",
// ── Settings ──────────────────────────────────────────────────────────────
// The body and the person, for the reference ranges, the score and the
@@ -0,0 +1,246 @@
/**
* v1.42 — the air-quality history backfill queue
* (`src/lib/environment/air-quality-history.ts` does the work).
*
* Three ways in, all coalesced per account by `singletonKey`:
* - the boot discovery after an update and the nightly discovery tick
* (02:40 Europe/Berlin, after the 02:10 environment fetch), which offer
* every account with the module and its air quality on, a place to
* resolve days to, and a history that is not through or was last checked
* more than a week ago (old entries imported since);
* - setting the home location;
* - switching the air-quality part on.
*
* One run works through a bounded number of ranges and stores its progress.
* When it made progress and the history is not through, and no budget
* ceiling stopped it, it sends its own follow-up half a minute later. A run
* the budget stopped sends none: the next nightly discovery resumes it, which
* is what spreads a long history over several days. The module gate and the
* air-quality switches are re-read on every run, so switching either off
* ends the chain.
*
* The queue name MUST be registered in `allQueues` in
* `src/lib/jobs/reminder/register-maintenance.ts`, or the sends silently
* no-op.
*/
import type { PgBoss } from "pg-boss";
import { prisma } from "@/lib/db";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
getOperatorModuleAvailability,
isModuleEnabled,
normalisePrefs,
resolveModuleEnabled,
} from "@/lib/modules/gate";
import { isAirQualityOperatorDisabled } from "@/lib/environment/open-meteo-air-quality";
import {
readAirQualityHistoryState,
runAirQualityHistory,
} from "@/lib/environment/air-quality-history";
import { workerLog } from "./reminder/shared";
export const ENVIRONMENT_AQ_HISTORY_QUEUE = "environment-aq-history";
/** Daily 02:40 Europe/Berlin, after the nightly environment fetch. */
export const ENVIRONMENT_AQ_HISTORY_CRON = "40 2 * * *";
/** Delay before a run's follow-up, in seconds. */
export const AQ_HISTORY_CONTINUATION_DELAY_SECONDS = 30;
/** Follow-ups one chain may send before it waits for the next discovery. */
export const AQ_HISTORY_MAX_CONTINUATIONS = 50;
/** A history checked longer ago than this is offered again (new old entries). */
const AQ_HISTORY_RECHECK_MS = 7 * 24 * 60 * 60 * 1000;
export interface EnvironmentAqHistoryPayload {
/** Absent on the discovery tick. */
userId?: string;
continuation?: number;
}
function singletonKey(userId: string): string {
return `environment-aq-history:${userId}`;
}
/**
* Queue a history run for one account. No-ops cleanly without a worker (the
* nightly discovery still covers it). A send while a run is queued collapses
* into it.
*/
export async function enqueueAirQualityHistory(
userId: string,
startAfterSeconds: number = 0,
): Promise<boolean> {
const boss = getGlobalBoss();
if (!boss) return false;
const id = await boss.send(
ENVIRONMENT_AQ_HISTORY_QUEUE,
{ userId } satisfies EnvironmentAqHistoryPayload,
{
singletonKey: singletonKey(userId),
...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}),
},
);
return id != null;
}
/** Whether a stored progress means the account should be offered a run. */
export function historyDue(raw: unknown, now: Date = new Date()): boolean {
const state = readAirQualityHistoryState(raw);
if (!state || !state.complete) return true;
const checked = Date.parse(state.checkedAt);
return (
!Number.isFinite(checked) || now.getTime() - checked > AQ_HISTORY_RECHECK_MS
);
}
/**
* Discovery: one run per account with the module on, its air quality on, a
* home or a location period, and a history that is due. Nothing at all when
* the operator turned air quality off.
*/
export async function discoverAirQualityHistory(
boss: PgBoss,
startAfterSeconds: number = 0,
): Promise<{ enqueued: number; skipped: number }> {
if (isAirQualityOperatorDisabled()) return { enqueued: 0, skipped: 0 };
const [candidates, operatorAvailability] = await Promise.all([
prisma.user.findMany({
where: {
environmentAirQualityEnabled: true,
OR: [
{ homeLocationEncrypted: { not: null } },
{ homeLat: { not: null }, homeLon: { not: null } },
{ environmentTravelLocations: { some: {} } },
],
},
select: {
id: true,
modulePreferencesJson: true,
environmentAqHistoryJson: true,
},
}),
getOperatorModuleAvailability(),
]);
const now = new Date();
let enqueued = 0;
let skipped = 0;
for (const candidate of candidates) {
const moduleOn = resolveModuleEnabled(
"environment",
{
gender: null,
disableCoach: false,
modulePreferences: normalisePrefs(candidate.modulePreferencesJson),
cycleTrackingEnabled: null,
},
false,
operatorAvailability,
);
if (!moduleOn || !historyDue(candidate.environmentAqHistoryJson, now)) {
skipped += 1;
continue;
}
await boss.send(
ENVIRONMENT_AQ_HISTORY_QUEUE,
{ userId: candidate.id } satisfies EnvironmentAqHistoryPayload,
{
singletonKey: singletonKey(candidate.id),
...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}),
},
);
enqueued += 1;
}
return { enqueued, skipped };
}
/**
* Boot discovery, staggered past the startup storm. Best-effort: the error
* comes back in the result so worker boot never fails on it.
*/
export async function enqueueBootTimeAirQualityHistory(
startAfterSeconds: number,
): Promise<{ enqueued: number; skipped: number; error: string | null }> {
const boss = getGlobalBoss();
if (!boss) return { enqueued: 0, skipped: 0, error: null };
try {
return {
...(await discoverAirQualityHistory(boss, startAfterSeconds)),
error: null,
};
} catch (err) {
return {
enqueued: 0,
skipped: 0,
error: err instanceof Error ? err.message : String(err),
};
}
}
/** The queue handler: discovery for an empty payload, else one account's run. */
export async function handleEnvironmentAqHistory(
boss: PgBoss,
payload: EnvironmentAqHistoryPayload,
): Promise<JobOutcome> {
if (!payload.userId) {
const { enqueued, skipped } = await discoverAirQualityHistory(boss);
workerLog(
"info",
`[environment-aq-history] discovery enqueued=${enqueued} skipped=${skipped}`,
);
return jobDone({
discovery_enqueued: enqueued,
discovery_skipped: skipped,
});
}
const userId = payload.userId;
if (!(await isModuleEnabled(userId, "environment"))) {
return jobDone({ skipped: "environment_module_disabled" });
}
const result = await runAirQualityHistory(userId);
const continuation = payload.continuation ?? 0;
let continued = false;
if (
result.status === "progress" &&
result.filled > 0 &&
continuation < AQ_HISTORY_MAX_CONTINUATIONS
) {
await boss.send(
ENVIRONMENT_AQ_HISTORY_QUEUE,
{
userId,
continuation: continuation + 1,
} satisfies EnvironmentAqHistoryPayload,
{
singletonKey: singletonKey(userId),
startAfter: AQ_HISTORY_CONTINUATION_DELAY_SECONDS,
},
);
continued = true;
}
workerLog(
"info",
`[environment-aq-history] user=${userId} status=${result.status} filled=${result.filled} done=${result.done}/${result.total} fetches=${result.fetches}${continued ? " continued" : ""}`,
);
// Every range failed at the feed: say so, so the failing-jobs card sees a
// feed that keeps refusing. A budget stop is not a failure: nothing was
// sent past the ceiling, and the next discovery resumes where it stopped.
if (result.status === "error") {
return jobFailed("air-quality history: every range failed", undefined, {
fetches: result.fetches,
});
}
return jobDone({
outcome: result.status,
days_stored: result.filled,
fetches: result.fetches,
budget_blocked: result.status === "budget",
continued,
});
}
+3
View File
@@ -466,4 +466,7 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
"environment-fetch": short(
"A fan-out of sends, or one account's bounded archive range.",
),
"environment-aq-history": short(
"A fan-out of sends, or one account's history run: at most eight ranges of up to 90 days, two requests each, then a follow-up.",
),
};
@@ -197,6 +197,13 @@ import {
handleEnvironmentFetch,
type EnvironmentFetchPayload,
} from "@/lib/jobs/environment-fetch";
import {
ENVIRONMENT_AQ_HISTORY_CRON,
ENVIRONMENT_AQ_HISTORY_QUEUE,
enqueueBootTimeAirQualityHistory,
handleEnvironmentAqHistory,
type EnvironmentAqHistoryPayload,
} from "@/lib/jobs/environment-air-quality-history";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
@@ -419,6 +426,9 @@ const allQueues = [
// surface. Without this entry pg-boss never provisions it and both the cron
// and the backfill button silently no-op.
ENVIRONMENT_FETCH_QUEUE,
// v1.42 — the air-quality history backfill: per account, from boot and
// nightly discovery, the home being set and air quality switched on.
ENVIRONMENT_AQ_HISTORY_QUEUE,
DATA_BACKUP_QUEUE,
RATE_LIMIT_CLEANUP_QUEUE,
IDEMPOTENCY_CLEANUP_QUEUE,
@@ -600,6 +610,9 @@ const schedules: ScheduleEntry[] = [
// v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload)
// that fans out one per-user environment fetch per opted-in account.
[ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON],
// v1.42 — daily 02:40 discovery for the air-quality history backfill, after
// the night's environment fetch has taken its share of the budget.
[ENVIRONMENT_AQ_HISTORY_QUEUE, ENVIRONMENT_AQ_HISTORY_CRON],
// The send options are the point of this tuple: without them the weekly
// snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on
// any sizeable record. See `data-backup-policy.ts` for why the window rides
@@ -823,6 +836,15 @@ const queuePolicies: QueuePolicyTable = {
// active. A boot's send for a queued account collapses into it; a duplicate
// beside an active run rewrites nothing, since only differing means are
// written.
// v1.42 — the air-quality history backfill. `short` for the same reason: a
// run with work left sends its own follow-up while it is still active. A
// send for an account whose run is queued collapses into it; one beside an
// active run only re-reads what is done, since done days are never fetched.
[ENVIRONMENT_AQ_HISTORY_QUEUE]: {
policy: "short",
reason:
"Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.",
},
[MEASUREMENT_FOLD_REPAIR_QUEUE]: {
policy: "short",
reason:
@@ -1769,6 +1791,22 @@ export async function registerMaintenanceQueues(
},
);
// v1.42 — the air-quality history backfill. Serial: every run draws on the
// same instance-wide Open-Meteo budget as the nightly fetch.
await createAndWork<EnvironmentAqHistoryPayload>(
boss,
ENVIRONMENT_AQ_HISTORY_QUEUE,
{ localConcurrency: 1 },
async (jobs) => {
let failure: JobOutcome | null = null;
for (const job of jobs) {
const outcome = await handleEnvironmentAqHistory(boss, job.data);
if (!outcome.ok && failure === null) failure = outcome;
}
return failure ?? jobDone({ jobs: jobs.length });
},
);
// v1.25 (W-ENV) — nightly environment fetch. The daily discovery tick (empty
// payload) fans out one per-user job per opted-in account; the queue also
// serves the on-demand backfill payloads from the settings surface. Serial
@@ -1925,6 +1963,31 @@ export async function enqueueMaintenanceBootDiscovery(): Promise<void> {
);
}
// v1.42 — the air-quality history backfill after an update, staggered past
// the boot storm. Accounts whose history is through are not offered.
try {
const { enqueued, skipped, error } = await enqueueBootTimeAirQualityHistory(
BOOT_BACKFILL_STAGGER_SECONDS * 7,
);
if (error) {
workerLog(
"error",
`[environment-aq-history] boot discovery failed: ${error}`,
);
} else {
workerLog(
"info",
`[environment-aq-history] boot discovery: enqueued=${enqueued} skipped=${skipped}`,
);
}
} catch (err) {
workerLog(
"error",
"[environment-aq-history] boot discovery threw an unexpected error",
err,
);
}
// v1.42 — the fold repair, once per account; the purge below waits for it.
try {
const { enqueued, skipped, error } =
+1
View File
@@ -94,6 +94,7 @@ const subsystemSurface = {
// register-maintenance.ts
"environment-fetch": { audience: "account" },
"environment-aq-history": { audience: "system" },
"data-backup": { audience: "system" },
"rate-limit-cleanup": { audience: "system" },
"idempotency-cleanup": { audience: "system" },
+23
View File
@@ -163,6 +163,29 @@ const environmentOverview = z
.describe(
"The model domain of that day: `cams_europe` (about 11 km, with pollen) or `cams_global` (about 45 km, no pollen).",
),
history: z
.object({
total: z
.number()
.int()
.describe(
"Past days with entries the source can serve at their place (from 2013 in Europe, from August 2022 elsewhere). Older days stay without values.",
),
done: z
.number()
.int()
.describe("Of those, the days whose air quality is stored."),
complete: z
.boolean()
.describe("True when nothing the source serves is left."),
checkedAt: z
.string()
.describe("ISO instant of the run that counted them."),
})
.nullable()
.describe(
"Progress of the background backfill that fills air quality for every past day with entries. Null before its first run and while air quality is off.",
),
})
.describe("v1.42 — the air-quality part of the module."),
latestDay: z