feat(environment): air quality, pollen and UV per day, locations sealed at rest

Fetch the Open-Meteo air-quality feed (CAMS, domains=auto) with the weather:
17 hourly variables folded into local days by fixed rules (means and maxima
only with 18 of 24 hours, ozone as the highest 8-hour running mean, UV only
with the midday hours), a value the feed did not cover stored as null, never
as zero. Requests are cut into 90-day chunks, newest first, and chunks before
mid 2022 ask for the eight pollutant and index variables only. An air-quality
failure never costs the weather row; the day stays unfetched and the nightly
gap fill (up to 90 days per account, newest first) asks for it again.

Every Open-Meteo request asks an instance-wide budget kept in Postgres first
(500 a minute, 4000 an hour, 8000 a day, weighted by the hosted pricing
rule). A refused request is not sent and the run ends as done with
budget_blocked. OPENMETEO_AIR_QUALITY_URL and ENVIRONMENT_AIR_QUALITY_DISABLED
are honoured; the account switch lives on PATCH /api/environment/preferences.

The home, each dated location period and each stored day keep their coarse
location sealed as one value under healthlog/environment-location/v1. The
writers write the sealed value only, the readers open it and fall back to the
readable columns for rows the free-text encryption backfill has not reached,
and the backfill now seals all three and clears the readable columns. The
backup carries the location readable in a portable file and sealed in a
disaster-recovery file; a restore seals either under the receiving key.

Three correlation channels join the environment family (PM2.5, the ozone
8-hour high, the day's highest pollen count), read through the averaged
same-day window and left empty while air quality is off. The Coach and MCP
gain get_environment: module-gated, clamped to the lookback limit, no
coordinate or place name anywhere, with provenance and the attribution
lines. The settings section gets the switch, the coverage and the newest
day's values; the dashboard shows a quiet chip on a day with high pollen,
a warm night or very poor air. Copernicus and Open-Meteo are credited next
to the values.
This commit is contained in:
Marc-André Bombeck committed 2026-10-08 10:02:39 +02:00
1 parent aca497d9ac
commit 3c98d95a40
73 files changed
+6061 -385

No files matched your search

@@ -5,6 +5,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
* 11 km), not 2 (about 1 km). The geocoder is where a picked city's
* coordinates enter, so its results must already be coarse.
*/
// The instance-wide request budget lives in Postgres; these tests are about
// the request itself, so the budget admits every call.
vi.mock("@/lib/environment/request-budget", async (importOriginal) => ({
...(await importOriginal<
typeof import("@/lib/environment/request-budget")
>()),
reserveOpenMeteoCalls: vi.fn(async () => true),
}));
beforeEach(() => {
vi.resetModules();
});
@@ -0,0 +1,52 @@
/**
* v1.42 (#615) — the notable-day flags the chips and the Coach summary share.
* Null is "not covered" and never raises a flag; each mark is inclusive.
*/
import { describe, expect, it } from "vitest";
import {
highPollenKinds,
isHotNight,
isVeryPoorAir,
pollenMax,
} from "../day-flags";
const NONE = {
alder: null,
birch: null,
grass: null,
mugwort: null,
olive: null,
ragweed: null,
};
describe("day flags", () => {
it("marks a tropical night at 20 °C and not below, and never for null", () => {
expect(isHotNight(20)).toBe(true);
expect(isHotNight(19.9)).toBe(false);
expect(isHotNight(null)).toBe(false);
});
it("marks very poor air from the index's very-poor band", () => {
expect(isVeryPoorAir(80)).toBe(true);
expect(isVeryPoorAir(79)).toBe(false);
expect(isVeryPoorAir(null)).toBe(false);
});
it("marks pollen per kind at its own high mark", () => {
expect(highPollenKinds({ ...NONE, birch: 100, grass: 49 })).toEqual([
"birch",
]);
expect(highPollenKinds({ ...NONE, grass: 50, ragweed: 50 })).toEqual([
"grass",
"ragweed",
]);
expect(highPollenKinds(NONE)).toEqual([]);
});
it("takes the pollen high over covered kinds only", () => {
expect(pollenMax(NONE)).toBeNull();
expect(pollenMax({ ...NONE, alder: 0 })).toBe(0);
expect(pollenMax({ ...NONE, alder: 3, grass: 12 })).toBe(12);
});
});
@@ -0,0 +1,82 @@
/**
* v1.42 (#615) — the sealed environment location.
*
* Round trip, rounding to the privacy floor before sealing, the label that
* keeps a location from opening as any other sealed value, and the reader's
* order: the sealed value first, the readable columns only without one, and
* a value that does not open read as no location rather than replaced.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { Buffer } from "node:buffer";
import { decryptBytes, encryptBytes } from "@/lib/crypto";
import { ENVIRONMENT_LOCATION_AAD } from "@/lib/crypto/encrypted-columns";
import { openLocation, readLocation, sealLocation } from "../location-cipher";
beforeEach(() => {
vi.stubEnv("ENCRYPTION_KEYS", "");
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
vi.stubEnv("ENCRYPTION_KEY", "b".repeat(64));
});
afterEach(() => vi.unstubAllEnvs());
describe("sealLocation / openLocation", () => {
it("round-trips a location and rounds it to one decimal first", () => {
const sealed = sealLocation({
lat: 51.48165,
lon: 7.21648,
label: "Bochum, Germany",
});
expect(openLocation(sealed)).toEqual({
lat: 51.5,
lon: 7.2,
label: "Bochum, Germany",
});
// The bytes are ciphertext, not the JSON.
expect(Buffer.from(sealed).toString("utf8")).not.toContain("Bochum");
});
it("is labelled: the value does not open under another label", () => {
const sealed = sealLocation({ lat: 1, lon: 2, label: "x" });
expect(() =>
decryptBytes(Buffer.from(sealed), "healthlog/workout-route-geometry/v1"),
).toThrow();
expect(() =>
decryptBytes(Buffer.from(sealed), ENVIRONMENT_LOCATION_AAD),
).not.toThrow();
});
it("refuses a sealed value that is not a location", () => {
const other = encryptBytes(
Buffer.from(JSON.stringify({ lat: "51" })),
ENVIRONMENT_LOCATION_AAD,
);
expect(() => openLocation(new Uint8Array(other))).toThrow();
});
});
describe("readLocation", () => {
it("prefers the sealed value over readable columns", () => {
const sealed = sealLocation({ lat: 48.1, lon: 11.6, label: "Munich" });
expect(
readLocation({ sealed, lat: 52.5, lon: 13.4, label: "Berlin" }),
).toEqual({ lat: 48.1, lon: 11.6, label: "Munich" });
});
it("falls back to the readable columns for a row the backfill has not reached", () => {
expect(
readLocation({ sealed: null, lat: 52.5, lon: 13.4, label: "Berlin" }),
).toEqual({ lat: 52.5, lon: 13.4, label: "Berlin" });
expect(
readLocation({ sealed: null, lat: 52.5, lon: null, label: "Berlin" }),
).toBeNull();
});
it("reads a value that does not open as no location, never as the readable columns", () => {
const sealed = sealLocation({ lat: 48.1, lon: 11.6, label: "Munich" });
vi.stubEnv("ENCRYPTION_KEY", "c".repeat(64));
expect(
readLocation({ sealed, lat: 52.5, lon: 13.4, label: "Berlin" }),
).toBeNull();
});
});
@@ -0,0 +1,297 @@
/**
* v1.42 (#615) — the air-quality client: how hourly values become a day, and
* what leaves the host.
*
* Fixtures only, no live call: a day is built hour by hour so each rule is
* pinned at its edge (18 of 24 hours, 6 of 8 in an ozone window, 13 of 17
* windows, the midday UV hours), a missing value stays null and is never read
* as zero, and the request carries the coarse coordinates, `domains=auto`
* and nothing else about the person.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const reserve = vi.hoisted(() => vi.fn(async (_weight: number) => true));
vi.mock("@/lib/environment/request-budget", async (importOriginal) => ({
...(await importOriginal<
typeof import("@/lib/environment/request-budget")
>()),
reserveOpenMeteoCalls: reserve,
}));
import {
AIR_QUALITY_VARIABLES,
aggregateAirQuality,
chunkDays,
coveredMax,
coveredMean,
fetchDailyAirQuality,
insideCamsEurope,
ozoneMax8h,
uvIndexMax,
variablesForChunk,
type AirQualityHourly,
} from "../open-meteo-air-quality";
/** 24 hourly stamps of one local day. */
function hoursOf(day: string): string[] {
return Array.from(
{ length: 24 },
(_, h) => `${day}T${String(h).padStart(2, "0")}:00`,
);
}
/** A full day with every variable at `value`, then `edit` applied. */
function dayFixture(
day: string,
value: number,
edit: (h: AirQualityHourly) => void = () => {},
): AirQualityHourly {
const hourly: AirQualityHourly = { time: hoursOf(day) };
for (const variable of AIR_QUALITY_VARIABLES) {
hourly[variable] = Array.from({ length: 24 }, () => value);
}
edit(hourly);
return hourly;
}
const BOCHUM = { lat: 51.5, lon: 7.2 };
beforeEach(() => {
reserve.mockReset();
reserve.mockResolvedValue(true);
});
afterEach(() => {
vi.unstubAllGlobals();
vi.unstubAllEnvs();
});
describe("daily rules", () => {
it("averages a day with 18 hours and refuses one with 17", () => {
const eighteen = [...Array(18).fill(10), ...Array(6).fill(null)];
const seventeen = [...Array(17).fill(10), ...Array(7).fill(null)];
expect(coveredMean(eighteen)).toBe(10);
expect(coveredMean(seventeen)).toBeNull();
expect(coveredMax(eighteen)).toBe(10);
expect(coveredMax(seventeen)).toBeNull();
});
it("never reads a missing hour as zero", () => {
// 18 hours at 20 and six missing: a zero-fill would give 15.
expect(coveredMean([...Array(18).fill(20), ...Array(6).fill(null)])).toBe(
20,
);
});
it("takes the highest 8-hour running mean inside the day", () => {
const byHour = Array.from({ length: 24 }, (_, h) =>
h >= 12 && h < 20 ? 120 : 40,
);
expect(ozoneMax8h(byHour)).toBe(120);
});
it("counts an ozone window with 6 of 8 hours and drops the day under 13 windows", () => {
// Hours 0–5 missing: windows starting 0..3 have fewer than 6 hours…
const sparse = Array.from({ length: 24 }, (_, h) => (h < 6 ? null : 50));
// windows 0,1 have 2/3 hours; 2,3 have 4/5; 4..16 have ≥6 → 13 windows.
expect(ozoneMax8h(sparse)).toBe(50);
const sparser = Array.from({ length: 24 }, (_, h) => (h < 7 ? null : 50));
expect(ozoneMax8h(sparser)).toBeNull();
});
it("gives a UV maximum only when the midday hours are all there", () => {
const full: (number | null)[] = Array.from({ length: 24 }, (_, h) =>
h === 13 ? 6.5 : 1,
);
expect(uvIndexMax(full)).toBe(6.5);
const gap = [...full];
gap[11] = null;
expect(uvIndexMax(gap)).toBeNull();
});
it("decides the model domain by the CAMS Europe box", () => {
expect(insideCamsEurope(51.5, 7.2)).toBe(true);
expect(insideCamsEurope(40.7, -74)).toBe(false);
expect(insideCamsEurope(-33.9, 151.2)).toBe(false);
});
});
describe("aggregateAirQuality", () => {
it("folds a full day into one observation", () => {
const [day] = aggregateAirQuality(
dayFixture("2025-05-01", 10, (h) => {
h.pm2_5![15] = 40;
h.european_aqi![15] = 85;
}),
BOCHUM,
);
expect(day).toMatchObject({
date: "2025-05-01",
pm25Mean: 11.3,
pm25Max: 40,
eaqiMax: 85,
pollenBirchMax: 10,
aqDomain: "cams_europe",
aqHours: 24,
});
});
it("keeps a variable the feed left empty as null (pollen outside Europe)", () => {
const [day] = aggregateAirQuality(
dayFixture("2023-06-07", 30, (h) => {
for (const kind of [
"alder_pollen",
"birch_pollen",
"grass_pollen",
"mugwort_pollen",
"olive_pollen",
"ragweed_pollen",
] as const) {
h[kind] = Array(24).fill(null);
}
}),
{ lat: 40.7, lon: -74 },
);
expect(day.pm25Mean).toBe(30);
expect(day.pollenGrassMax).toBeNull();
expect(day.pollenBirchMax).toBeNull();
expect(day.aqDomain).toBe("cams_global");
});
it("keeps a variable missing from the response as null (the pre-2022 era)", () => {
const hourly: AirQualityHourly = {
time: hoursOf("2016-03-01"),
pm2_5: Array(24).fill(12),
};
const [day] = aggregateAirQuality(hourly, BOCHUM);
expect(day.pm25Mean).toBe(12);
expect(day.uvIndexMax).toBeNull();
expect(day.pollenBirchMax).toBeNull();
expect(day.dustMax).toBeNull();
});
});
describe("requests", () => {
it("splits into 90-day chunks and asks the pre-2022 ones for the pollutants only", () => {
expect(chunkDays("2022-01-01", "2022-07-01")).toEqual([
{ startDate: "2022-01-01", endDate: "2022-03-31" },
{ startDate: "2022-04-01", endDate: "2022-06-29" },
{ startDate: "2022-06-30", endDate: "2022-07-01" },
]);
expect(variablesForChunk("2022-03-31")).toHaveLength(8);
expect(variablesForChunk("2022-06-29")).toHaveLength(17);
});
it("sends the coarse coordinates, the timezone and domains=auto, nothing else", async () => {
const fetchSpy = vi.fn(async (_url: unknown) =>
Response.json({ hourly: dayFixture("2025-05-01", 10) }),
);
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchDailyAirQuality({
...BOCHUM,
timezone: "Europe/Berlin",
startDate: "2025-05-01",
endDate: "2025-05-01",
});
expect(result.stopped).toBeNull();
expect(result.days).toHaveLength(1);
const url = new URL(String(fetchSpy.mock.calls[0]![0]));
expect(url.origin + url.pathname).toBe(
"https://air-quality-api.open-meteo.com/v1/air-quality",
);
expect([...url.searchParams.keys()].sort()).toEqual(
[
"domains",
"end_date",
"hourly",
"latitude",
"longitude",
"start_date",
"timezone",
].sort(),
);
expect(url.searchParams.get("latitude")).toBe("51.5");
expect(url.searchParams.get("longitude")).toBe("7.2");
expect(url.searchParams.get("domains")).toBe("auto");
expect(url.searchParams.get("hourly")!.split(",")).toHaveLength(17);
});
it("asks the budget with the request's weight and sends nothing when it refuses", async () => {
reserve.mockResolvedValue(false);
const fetchSpy = vi.fn();
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchDailyAirQuality({
...BOCHUM,
timezone: "UTC",
startDate: "2025-01-01",
endDate: "2025-03-31",
});
expect(result).toEqual({ days: [], stopped: "budget" });
expect(fetchSpy).not.toHaveBeenCalled();
// 17 variables over 90 days: 1.7 × 90/14.
expect(reserve.mock.calls[0]![0]).toBeCloseTo(1.7 * (90 / 14), 5);
});
it("fetches newest first and keeps what it has when a chunk fails", async () => {
const fetchSpy = vi
.fn()
.mockResolvedValueOnce(
Response.json({ hourly: dayFixture("2025-06-30", 5) }),
)
.mockResolvedValueOnce(new Response("busy", { status: 503 }));
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchDailyAirQuality({
...BOCHUM,
timezone: "UTC",
startDate: "2025-01-01",
endDate: "2025-06-30",
});
expect(result.stopped).toBe("error");
expect(result.days.map((d) => d.date)).toEqual(["2025-06-30"]);
const first = new URL(String(fetchSpy.mock.calls[0]![0]));
expect(first.searchParams.get("end_date")).toBe("2025-06-30");
});
it("answers days before 2013 as uncoverable without a request", async () => {
const fetchSpy = vi.fn();
vi.stubGlobal("fetch", fetchSpy);
const result = await fetchDailyAirQuality({
...BOCHUM,
timezone: "UTC",
startDate: "2012-12-30",
endDate: "2012-12-31",
});
expect(fetchSpy).not.toHaveBeenCalled();
expect(result.days.map((d) => [d.date, d.aqHours, d.pm25Mean])).toEqual([
["2012-12-30", 0, null],
["2012-12-31", 0, null],
]);
});
it("honours a self-hosted endpoint", async () => {
vi.resetModules();
vi.stubEnv("OPENMETEO_AIR_QUALITY_URL", "https://aq.example.test/");
const fetchSpy = vi.fn(async (_url: unknown) =>
Response.json({ hourly: {} }),
);
vi.stubGlobal("fetch", fetchSpy);
const fresh = await import("../open-meteo-air-quality");
await fresh.fetchDailyAirQuality({
...BOCHUM,
timezone: "UTC",
startDate: "2025-05-01",
endDate: "2025-05-01",
});
expect(String(fetchSpy.mock.calls[0]![0])).toMatch(
/^https:\/\/aq\.example\.test\/v1\/air-quality\?/,
);
});
it("reads the operator switch", async () => {
const { isAirQualityActive } = await import("../open-meteo-air-quality");
expect(isAirQualityActive(true)).toBe(true);
expect(isAirQualityActive(false)).toBe(false);
vi.stubEnv("ENVIRONMENT_AIR_QUALITY_DISABLED", "true");
expect(isAirQualityActive(true)).toBe(false);
});
});
@@ -6,6 +6,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
* hosted defaults must still apply; before, the empty string was kept and
* every request died on `new URL("/v1/search")` with ERR_INVALID_URL.
*/
// The instance-wide request budget lives in Postgres; these tests are about
// the request itself, so the budget admits every call.
vi.mock("@/lib/environment/request-budget", async (importOriginal) => ({
...(await importOriginal<
typeof import("@/lib/environment/request-budget")
>()),
reserveOpenMeteoCalls: vi.fn(async () => true),
}));
beforeEach(() => {
vi.resetModules();
});
@@ -0,0 +1,115 @@
/**
* v1.42 (#615) — the instance-wide Open-Meteo request budget.
*
* The weight rule (the hosted pricing formula), the three windows checked
* together, a window that expired starting over, and the transaction that
* charges all three windows or none.
*/
import { describe, expect, it, vi } from "vitest";
const store = vi.hoisted(
() => new Map<string, { count: number; resetAt: Date }>(),
);
vi.mock("@/lib/db", () => {
const tx = {
$queryRaw: vi.fn(async () => [{ locked: 1 }]),
rateLimit: {
findMany: vi.fn(async (args: { where: { key: { in: string[] } } }) =>
args.where.key.in
.filter((key) => store.has(key))
.map((key) => ({ key, ...store.get(key)! })),
),
upsert: vi.fn(
async (args: {
where: { key: string };
create: { count: number; resetAt: Date };
}) => {
store.set(args.where.key, {
count: args.create.count,
resetAt: args.create.resetAt,
});
},
),
},
};
return {
prisma: {
$transaction: async (fn: (t: typeof tx) => unknown) => fn(tx),
},
};
});
import {
OPEN_METEO_BUDGET_WINDOWS,
admitWeight,
openMeteoCallWeight,
reserveOpenMeteoCalls,
type BudgetBucket,
} from "../request-budget";
const NOW = new Date("2026-10-08T02:10:00Z");
describe("call weight", () => {
it("follows the hosted rule: more than 10 variables or 14 days weighs more", () => {
expect(openMeteoCallWeight(3, 1)).toBe(1);
expect(openMeteoCallWeight(10, 14)).toBe(1);
expect(openMeteoCallWeight(15, 14)).toBe(1.5);
expect(openMeteoCallWeight(17, 90)).toBeCloseTo(10.93, 2);
expect(openMeteoCallWeight(12, 730)).toBeCloseTo(62.57, 2);
});
});
describe("admitWeight", () => {
it("admits into empty windows and stamps each window's reset", () => {
const decision = admitWeight(new Map(), 150, NOW);
expect(decision.admitted).toBe(true);
if (!decision.admitted) return;
for (const w of OPEN_METEO_BUDGET_WINDOWS) {
expect(decision.next.get(w.name)).toEqual({
count: 150,
resetAt: new Date(NOW.getTime() + w.windowMs),
});
}
});
it("refuses when the day is full even though the minute and hour are not", () => {
const later = new Date(NOW.getTime() + 3_600_000);
const buckets = new Map<"minute" | "hour" | "day", BudgetBucket>([
["day", { count: 8_000 * 100 - 50, resetAt: later }],
]);
expect(admitWeight(buckets, 49, NOW).admitted).toBe(true);
expect(admitWeight(buckets, 51, NOW)).toEqual({
admitted: false,
window: "day",
});
});
it("starts a window over once it has expired", () => {
const expired = new Date(NOW.getTime() - 1);
const buckets = new Map<"minute" | "hour" | "day", BudgetBucket>([
["minute", { count: 500 * 100, resetAt: expired }],
]);
const decision = admitWeight(buckets, 100, NOW);
expect(decision.admitted).toBe(true);
if (decision.admitted) {
expect(decision.next.get("minute")!.count).toBe(100);
}
});
});
describe("reserveOpenMeteoCalls", () => {
it("charges every window, and charges nothing once one is full", async () => {
store.clear();
expect(await reserveOpenMeteoCalls(1.7)).toBe(true);
expect(store.get("open-meteo-budget:day")!.count).toBe(170);
expect(store.get("open-meteo-budget:minute")!.count).toBe(170);
// Fill the minute to the brim: the next call is refused and the hour
// and the day are not charged for it.
const minute = store.get("open-meteo-budget:minute")!;
store.set("open-meteo-budget:minute", { ...minute, count: 500 * 100 });
const dayBefore = store.get("open-meteo-budget:day")!.count;
expect(await reserveOpenMeteoCalls(1)).toBe(false);
expect(store.get("open-meteo-budget:day")!.count).toBe(dayBefore);
});
});
@@ -0,0 +1,297 @@
/**
* v1.42 (#615) — the fetch-and-store run with air quality and sealed
* locations.
*
* - the weather row is stored even when the air-quality fetch fails, and
* its air-quality part is left unfetched for the gap fill;
* - with air quality on, the day's values and `aqFetchedAt` ride the same
* upsert; off by the account or by the operator, the feed is not asked;
* - the home is opened from its sealed copy and only its coarse
* coordinates go into a request; the row is written sealed, the readable
* location columns empty;
* - a refused budget stops the run without failing it;
* - the gap fill fetches the stored days newest first at the location each
* day was stored for.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const db = vi.hoisted(() => ({
user: null as Record<string, unknown> | null,
travel: [] as Array<Record<string, unknown>>,
days: [] as Array<Record<string, unknown>>,
upserts: [] as Array<{
create: Record<string, unknown>;
update: Record<string, unknown>;
}>,
updates: [] as Array<{
where: Record<string, unknown>;
data: Record<string, unknown>;
}>,
}));
vi.mock("@/lib/db", () => ({
prisma: {
user: { findUnique: vi.fn(async () => db.user) },
environmentTravelLocation: { findMany: vi.fn(async () => db.travel) },
environmentContext: {
upsert: vi.fn(
(args: {
create: Record<string, unknown>;
update: Record<string, unknown>;
}) => {
db.upserts.push(args);
return args;
},
),
findMany: vi.fn(async () => db.days),
updateMany: vi.fn(
(args: {
where: Record<string, unknown>;
data: Record<string, unknown>;
}) => {
db.updates.push(args);
return { count: 1 };
},
),
},
$transaction: vi.fn(async (ops: unknown[]) =>
ops.map((op) =>
op && typeof op === "object" && "count" in op ? op : op,
),
),
},
}));
const weather = vi.hoisted(() => vi.fn());
vi.mock("@/lib/environment/open-meteo", async (importOriginal) => ({
...(await importOriginal<typeof import("@/lib/environment/open-meteo")>()),
fetchDailyEnvironment: weather,
}));
const air = vi.hoisted(() => vi.fn());
vi.mock("@/lib/environment/open-meteo-air-quality", async (importOriginal) => ({
...(await importOriginal<
typeof import("@/lib/environment/open-meteo-air-quality")
>()),
fetchDailyAirQuality: air,
}));
import { openLocation, sealLocation } from "../location-cipher";
import { OpenMeteoBudgetExhaustedError } from "../request-budget";
import { uncoverableDay } from "../open-meteo-air-quality";
import { fetchAndStoreEnvironment, fillAirQualityGaps } from "../service";
function obs(date: string) {
return {
date,
tempMin: 14,
tempMax: 24,
tempMean: 19,
apparentMean: 18,
apparentMax: 25,
sunshineSec: 3600,
daylightSec: 50_000,
precipSum: 0,
pressureMean: 1012,
pressureDelta: 3,
humidityMean: 60,
cloudMean: 40,
weatherCode: 1,
};
}
function airDay(date: string) {
return {
...uncoverableDay(date),
pm25Mean: 9.5,
aqHours: 24,
aqDomain: "cams_europe",
};
}
beforeEach(() => {
vi.stubEnv("ENCRYPTION_KEYS", "");
vi.stubEnv("ENCRYPTION_ACTIVE_KEY_ID", "");
vi.stubEnv("ENCRYPTION_KEY", "d".repeat(64));
db.user = {
homeLat: null,
homeLon: null,
homeLabel: null,
homeLocationEncrypted: sealLocation({
lat: 51.48,
lon: 7.21,
label: "Bochum, Germany",
}),
homeTimezone: "Europe/Berlin",
homeSince: new Date("2026-01-01T00:00:00Z"),
timezone: "Europe/Berlin",
environmentAirQualityEnabled: true,
};
db.travel = [];
db.days = [];
db.upserts = [];
db.updates = [];
weather.mockReset();
air.mockReset();
weather.mockImplementation(async (args: { startDate: string }) => [
obs(args.startDate),
]);
});
afterEach(() => vi.unstubAllEnvs());
describe("fetchAndStoreEnvironment", () => {
it("asks both feeds with the coarse home only and writes the day sealed", async () => {
air.mockResolvedValue({ days: [airDay("2026-05-01")], stopped: null });
const result = await fetchAndStoreEnvironment({
userId: "u1",
startDate: "2026-05-01",
endDate: "2026-05-01",
});
expect(result).toMatchObject({ stored: 1, budgetBlocked: false });
for (const call of [weather.mock.calls[0][0], air.mock.calls[0][0]]) {
expect(Object.keys(call).sort()).toEqual(
["endDate", "lat", "lon", "startDate", "timezone"].sort(),
);
expect([call.lat, call.lon]).toEqual([51.5, 7.2]);
}
const { create } = db.upserts[0];
expect(create).toMatchObject({
lat: null,
lon: null,
locationLabel: null,
source: "HOME",
apparentMax: 25,
pm25Mean: 9.5,
aqHours: 24,
});
expect(create.aqFetchedAt).toBeInstanceOf(Date);
expect(openLocation(create.locationEncrypted as Uint8Array)).toEqual({
lat: 51.5,
lon: 7.2,
label: "Bochum, Germany",
});
});
it("stores the weather when the air-quality fetch fails, and leaves the air part unfetched", async () => {
air.mockResolvedValue({ days: [], stopped: "error" });
const result = await fetchAndStoreEnvironment({
userId: "u1",
startDate: "2026-05-01",
endDate: "2026-05-01",
});
expect(result.stored).toBe(1);
const { update } = db.upserts[0];
expect(update).toMatchObject({
tempMin: 14,
aqFetchedAt: null,
pm25Mean: null,
});
});
it("does not ask the air-quality feed when the account turned it off", async () => {
db.user!.environmentAirQualityEnabled = false;
await fetchAndStoreEnvironment({
userId: "u1",
startDate: "2026-05-01",
endDate: "2026-05-01",
});
expect(air).not.toHaveBeenCalled();
// The stored air part is left as it is: no air column in the write.
expect(Object.keys(db.upserts[0].update)).not.toContain("aqFetchedAt");
});
it("does not ask it when the operator turned it off", async () => {
vi.stubEnv("ENVIRONMENT_AIR_QUALITY_DISABLED", "1");
await fetchAndStoreEnvironment({
userId: "u1",
startDate: "2026-05-01",
endDate: "2026-05-01",
});
expect(air).not.toHaveBeenCalled();
});
it("stops without failing when the budget refuses", async () => {
weather.mockRejectedValue(new OpenMeteoBudgetExhaustedError());
const result = await fetchAndStoreEnvironment({
userId: "u1",
startDate: "2026-05-01",
endDate: "2026-05-01",
});
expect(result).toMatchObject({ stored: 0, budgetBlocked: true });
expect(air).not.toHaveBeenCalled();
});
it("resolves a period from its sealed copy", async () => {
db.travel = [
{
startDate: "2026-05-01",
endDate: "2026-05-02",
lat: null,
lon: null,
label: null,
locationEncrypted: sealLocation({
lat: 38.72,
lon: -9.14,
label: "Lisbon, Portugal",
}),
},
];
air.mockResolvedValue({ days: [], stopped: null });
await fetchAndStoreEnvironment({
userId: "u1",
startDate: "2026-05-01",
endDate: "2026-05-01",
});
expect([
weather.mock.calls[0][0].lat,
weather.mock.calls[0][0].lon,
]).toEqual([38.7, -9.1]);
expect(db.upserts[0].create.source).toBe("TRAVEL");
});
});
describe("fillAirQualityGaps", () => {
it("fetches each stored day at its own sealed location and marks it fetched", async () => {
const sealed = sealLocation({ lat: 51.5, lon: 7.2, label: "Home" });
db.days = [
{
date: "2026-05-03",
lat: null,
lon: null,
locationLabel: null,
locationEncrypted: sealed,
},
{
date: "2026-05-02",
lat: null,
lon: null,
locationLabel: null,
locationEncrypted: sealed,
},
];
air.mockResolvedValue({
days: [airDay("2026-05-02"), airDay("2026-05-03")],
stopped: null,
});
const result = await fillAirQualityGaps("u1");
expect(result).toEqual({ filled: 2, stopped: null });
expect(air.mock.calls[0][0]).toMatchObject({
lat: 51.5,
lon: 7.2,
startDate: "2026-05-02",
endDate: "2026-05-03",
});
expect(db.updates.map((u) => u.where)).toEqual([
{ userId: "u1", date: "2026-05-02", aqFetchedAt: null },
{ userId: "u1", date: "2026-05-03", aqFetchedAt: null },
]);
});
it("is a no-op while air quality is off for the account", async () => {
db.user!.environmentAirQualityEnabled = false;
expect(await fillAirQualityGaps("u1")).toEqual({
filled: 0,
stopped: null,
});
expect(air).not.toHaveBeenCalled();
});
});
@@ -60,3 +60,25 @@ export const AIR_QUALITY_ATTRIBUTIONS = [
url: "https://open-meteo.com/",
},
] as const;
/**
* The attribution lines, in display order, for wherever environment values
* are shown (settings, the Coach and MCP read, the docs). The weather line
* always; the two air-quality lines when air-quality values are part of what
* is shown. "Contains modified" because hourly values are folded into days
* here (the Copernicus licence asks for that wording on derived data); the
* year is the newest year the shown values come from.
*/
export function environmentAttributionLines(args: {
airQuality: boolean;
year: number;
}): string[] {
const lines = ["Weather data by Open-Meteo.com (CC BY 4.0)"];
if (args.airQuality) {
lines.push(
"Air quality data by Open-Meteo.com (CC BY 4.0)",
`Contains modified Copernicus Atmosphere Monitoring Service information ${args.year}`,
);
}
return lines;
}
+77
View File
@@ -0,0 +1,77 @@
/**
* The three notable-day flags of the environment module (v1.42, #615): a
* high pollen count, a hot night, and very poor air. The dashboard chips, the
* Coach and MCP read's summary all count with these, so a day is "notable"
* by one definition everywhere. Leaf module, safe on the client.
*
* Deliberately few and deliberately high: a flag is a quiet note about the
* weather a day had, shown beside the person's own data, and a chip that is
* on most summer days says nothing. None of them claims an effect on the
* person: they describe the modelled outdoor air at a coarse location, which
* is not anyone's personal exposure.
*
* - Hot night: the day's minimum temperature at or above 20 °C, the
* meteorological "tropical night".
* - Very poor air: the European air-quality index reaching 80 at some hour
* of the day, the index's "very poor" band (60–80 is "poor", which a
* summer ozone afternoon reaches routinely).
* - High pollen: any kind at or above a coarse "high" mark, 100 grains/m³
* for the tree pollens (alder, birch, olive), 50 for grass, mugwort and
* ragweed. Approximate bands of the kind European pollen services use;
* individual sensitivity varies far more than these marks do.
*/
/** Minimum temperature (°C) at or above which a night counts as hot. */
export const HOT_NIGHT_MIN_C = 20;
/** European AQI at or above which the air counts as very poor. */
export const VERY_POOR_AIR_EAQI = 80;
/** The pollen kinds, in display order. */
export const POLLEN_KINDS = [
"alder",
"birch",
"grass",
"mugwort",
"olive",
"ragweed",
] as const;
export type PollenKind = (typeof POLLEN_KINDS)[number];
/** The grains/m³ at or above which each kind counts as high. */
export const POLLEN_HIGH_MARK: Readonly<Record<PollenKind, number>> = {
alder: 100,
birch: 100,
olive: 100,
grass: 50,
mugwort: 50,
ragweed: 50,
};
/** Pollen maxima of one day by kind; null = not covered. */
export type PollenByKind = Readonly<Record<PollenKind, number | null>>;
/** The day's highest pollen count over all kinds, or null when none covered. */
export function pollenMax(pollen: PollenByKind): number | null {
let best: number | null = null;
for (const kind of POLLEN_KINDS) {
const value = pollen[kind];
if (value != null && (best === null || value > best)) best = value;
}
return best;
}
/** The kinds at or above their high mark, in display order. */
export function highPollenKinds(pollen: PollenByKind): PollenKind[] {
return POLLEN_KINDS.filter((kind) => {
const value = pollen[kind];
return value != null && value >= POLLEN_HIGH_MARK[kind];
});
}
export function isHotNight(tempMin: number | null): boolean {
return tempMin != null && tempMin >= HOT_NIGHT_MIN_C;
}
export function isVeryPoorAir(eaqiMax: number | null): boolean {
return eaqiMax != null && eaqiMax >= VERY_POOR_AIR_EAQI;
}
+17
View File
@@ -0,0 +1,17 @@
/**
* Day-key arithmetic for the environment module. Leaf module (no imports) so
* the feed clients, the service and the job share one definition.
*/
const MS_PER_DAY = 86_400_000;
/** Inclusive number of days from `start` to `end` (`YYYY-MM-DD` keys). */
export function enumerateDayCount(start: string, end: string): number {
const [sy, sm, sd] = start.split("-").map(Number);
const [ey, em, ed] = end.split("-").map(Number);
return (
Math.round(
(Date.UTC(ey, em - 1, ed) - Date.UTC(sy, sm - 1, sd)) / MS_PER_DAY,
) + 1
);
}
+91 -2
View File
@@ -56,13 +56,39 @@ export type EnvironmentContextNumericColumn =
| "pressureMean"
| "pressureDelta"
| "humidityMean"
| "cloudMean";
| "cloudMean"
// v1.42 (#615) — air quality.
| "pm25Mean"
| "o3Max8h";
/**
* A channel value that is not one column: the day's highest pollen count over
* the six kinds (`pollenAlderMax` … `pollenRagweedMax`), null when no kind was
* covered. Read by {@link environmentFieldValue}.
*/
export type EnvironmentDerivedColumn = "pollenMax";
/** The six pollen columns `pollenMax` folds. */
export const POLLEN_COLUMNS = [
"pollenAlderMax",
"pollenBirchMax",
"pollenGrassMax",
"pollenMugwortMax",
"pollenOliveMax",
"pollenRagweedMax",
] as const;
export interface EnvironmentField {
/** Stable key — the registry signal key AND the correlation channel key. */
key: string;
/** The `EnvironmentContext` column this field's daily value comes from. */
column: EnvironmentContextNumericColumn;
column: EnvironmentContextNumericColumn | EnvironmentDerivedColumn;
/**
* v1.42 — the field comes from the air-quality feed, so it is read only
* while the account's air-quality part is on (and the operator's is not
* off). The weather fields do not set it.
*/
airQuality?: true;
/** Canonical unit (English, for prose + registry metadata). */
unit: string;
/** Stable English display name (the UI localises via `i18nLabelKey`). */
@@ -136,4 +162,67 @@ export const ENVIRONMENT_FIELDS: readonly EnvironmentField[] = [
narrationLabel: "intraday pressure swing",
i18nLabelKey: "environment.fields.pressureDelta",
},
// v1.42 (#615) — three air-quality channels, deliberately no more: every
// channel widens the false-discovery family for all the others. Fine
// particles and the ozone high carry the evidence for blood pressure, HRV
// and sleep; the pollen high is the one with a plausible personal signal,
// against symptoms. UV, NO2 and dust are shown and read by the Coach but
// not correlated (UV is almost pure season, NO2 moves with PM2.5). The
// night-time heat channel is the existing overnight low above.
{
key: "ENV_PM25",
column: "pm25Mean",
unit: "µg/m³",
displayName: "Fine particles (PM2.5)",
narrationLabel: "fine particles (PM2.5)",
i18nLabelKey: "environment.fields.pm25",
airQuality: true,
},
{
key: "ENV_OZONE_8H",
column: "o3Max8h",
unit: "µg/m³",
displayName: "Ozone (8-hour high)",
narrationLabel: "ozone",
i18nLabelKey: "environment.fields.ozone8h",
airQuality: true,
},
{
key: "ENV_POLLEN_MAX",
column: "pollenMax",
unit: "grains/m³",
displayName: "Pollen (highest)",
narrationLabel: "pollen",
i18nLabelKey: "environment.fields.pollenMax",
airQuality: true,
},
] as const;
/** The columns a day row must carry for {@link environmentFieldValue}. */
export type EnvironmentFieldRow = Readonly<
Record<EnvironmentContextNumericColumn, number | null> &
Record<(typeof POLLEN_COLUMNS)[number], number | null>
>;
/**
* One field's raw value on one stored day (before any unit conversion), or
* null when the day does not cover it. The pollen high is the maximum over
* the kinds that were covered, never a zero for the ones that were not.
*/
export function environmentFieldValue(
field: EnvironmentField,
row: EnvironmentFieldRow,
): number | null {
if (field.column === "pollenMax") {
let best: number | null = null;
for (const column of POLLEN_COLUMNS) {
const value = row[column];
if (value != null && Number.isFinite(value)) {
best = best === null ? value : Math.max(best, value);
}
}
return best;
}
const value = row[field.column];
return value != null && Number.isFinite(value) ? value : null;
}
+114
View File
@@ -0,0 +1,114 @@
/**
* The environment module's coarse locations, sealed at rest (v1.42, #615).
*
* Three places held a location in readable columns: the home on the user row
* (`homeLat` / `homeLon` / `homeLabel`), each dated location period
* (`EnvironmentTravelLocation.lat` / `lon` / `label`), and each stored day
* (`EnvironmentContext.lat` / `lon` / `locationLabel`). Coarse as they are
* (one decimal, about 11 km), a home city and a dated list of trips are still
* a movement profile. From v1.42 each row stores ONE sealed value instead, the
* JSON `{ lat, lon, label }` under AES-256-GCM with the associated-data label
* {@link ENVIRONMENT_LOCATION_AAD}: one decrypt per row, and the three parts
* can never drift apart.
*
* Writers seal through {@link sealLocation}; the readable columns are written
* as null. Readers go through {@link readLocation}: the sealed value when a
* row has one, the readable columns only for a row the boot-time encryption
* backfill (`free-text-encryption-backfill.ts`) has not reached yet. A sealed
* value that does not open reads as no location, with a warning on the
* request's event, and is never replaced by the readable columns.
*
* The coordinates are rounded to the privacy floor BEFORE sealing, so the
* sealed value never carries more precision than the readable column did,
* and nothing that leaves the host (the weather and air-quality requests) is
* ever finer than that.
*/
import { Buffer } from "node:buffer";
import { decryptBytes, encryptBytes } from "@/lib/crypto";
import { ENVIRONMENT_LOCATION_AAD } from "@/lib/crypto/encrypted-columns";
import { roundCoarse } from "@/lib/environment/open-meteo";
import { getEvent } from "@/lib/logging/context";
/** A coarse location as the module uses it. */
export interface CoarseLocation {
lat: number;
lon: number;
label: string;
}
/** Seal a location for one of the three `*Encrypted` location columns. */
export function sealLocation(
location: CoarseLocation,
): Uint8Array<ArrayBuffer> {
const sealed = encryptBytes(
Buffer.from(
JSON.stringify({
lat: roundCoarse(location.lat),
lon: roundCoarse(location.lon),
label: location.label,
}),
"utf8",
),
ENVIRONMENT_LOCATION_AAD,
);
const out = new Uint8Array(new ArrayBuffer(sealed.byteLength));
out.set(sealed);
return out;
}
/**
* Open a sealed location. Throws on a bad key id, a tampered value, or a
* payload that is not a location.
*/
export function openLocation(sealed: Uint8Array): CoarseLocation {
const plain = decryptBytes(Buffer.from(sealed), ENVIRONMENT_LOCATION_AAD);
const parsed = JSON.parse(plain.toString("utf8")) as unknown;
if (
!parsed ||
typeof parsed !== "object" ||
typeof (parsed as CoarseLocation).lat !== "number" ||
typeof (parsed as CoarseLocation).lon !== "number" ||
typeof (parsed as CoarseLocation).label !== "string"
) {
throw new Error("sealed environment location has an unexpected shape");
}
const { lat, lon, label } = parsed as CoarseLocation;
return { lat, lon, label };
}
/**
* The location a row holds: the sealed value first, the legacy readable
* columns only when there is no sealed value. Null when the row holds
* neither (or only part of the readable triple), and null with a warning when
* the sealed value does not open.
*
* `fallbackLabel` stands in for a missing readable label, which the home has
* always allowed (`homeLabel` was nullable from the start).
*/
export function readLocation(
row: {
sealed: Uint8Array | null | undefined;
lat: number | null | undefined;
lon: number | null | undefined;
label: string | null | undefined;
},
fallbackLabel?: string,
): CoarseLocation | null {
if (row.sealed && row.sealed.byteLength > 0) {
try {
return openLocation(row.sealed);
} catch (err) {
getEvent()?.addWarning(
`environment location decrypt failed: ${
err instanceof Error ? err.message : String(err)
}`,
);
return null;
}
}
if (row.lat == null || row.lon == null) return null;
const label = row.label ?? fallbackLabel;
if (label == null) return null;
return { lat: row.lat, lon: row.lon, label };
}
@@ -0,0 +1,438 @@
/**
* v1.42 (#615) — Open-Meteo air-quality client: particles, gases, the two
* air-quality indices, UV, Saharan dust, aerosol depth and six pollen kinds
* per local day, from the CAMS models.
*
* ## The request
*
* `GET /v1/air-quality` with the 17 hourly variables below, the coarse
* coordinates (one decimal, the module's privacy floor), the day range and the
* same timezone the weather archive is asked in, so a stored day means the
* same local day for both. `domains=auto` is required: CAMS Europe (0.1°,
* about 11 km) alone never returns UV or aerosol depth, CAMS global (0.4°,
* about 45 km) alone never returns pollen; `auto` takes each variable from the
* model that has it. The feed has no daily aggregates (asking for one is an
* error), so it is read hourly and folded into days here.
*
* Egress goes through `safeFetch` like the archive client, without
* `requirePublicHost`: the base URL is operator configuration
* (`OPENMETEO_AIR_QUALITY_URL`), and an operator may point it at a
* self-hosted Open-Meteo on a private host. `ENVIRONMENT_AIR_QUALITY_DISABLED`
* turns the whole feed off for the instance; nothing is sent then.
*
* ## Coverage, and why a missing value is null and never zero
*
* The feed starts on 2013-01-01; pollen, UV, dust and aerosol depth only from
* about mid 2022, and pollen only in Europe. A value the feed did not cover is
* stored as null. A chunk that ends before {@link AIR_QUALITY_FULL_FEED_FROM}
* asks only for the eight pollutant and index variables, which the feed has
* for that era; that keeps the request weight at 1.0 instead of 1.7 per ten
* variables. A day before {@link AIR_QUALITY_EARLIEST_DAY} cannot be fetched
* at all and is returned as uncoverable (every value null, no hours), so the
* nightly gap fill does not ask for it again.
*
* ## Daily rules (a local day of hourly values)
*
* - PM2.5, PM10, NO2, SO2, CO: the mean, only when at least 18 of the day's
* hours have a value (the 75 % completeness rule air-quality reporting
* uses); PM2.5 also its hourly maximum under the same rule (smoke shows in
* the peak, not the mean).
* - Ozone: the highest 8-hour running mean whose window lies inside the day
* (windows starting 00:00 to 16:00, 17 of them), a window counting when 6
* of its 8 hours have a value; the day only when at least 13 windows
* count. A simplified form of the EU 8-hour ozone metric.
* - European and US AQI, dust, aerosol depth, each pollen kind: the day's
* maximum, under the 18-hour rule.
* - UV index: the day's maximum, only when 10:00 to 16:00 are all present,
* because a UV maximum with the midday hours missing is not a maximum.
* - `aqHours`: the number of hours with a PM2.5 value, the day's coverage.
* - `aqDomain`: `cams_europe` when the coordinates lie inside the CAMS
* Europe domain (30–72° N, 25° W–45° E), else `cams_global`. The response
* does not say which model `auto` used; the domain box is what decides it.
*
* ## Attribution
*
* CAMS data is free to use with the notice "Contains modified Copernicus
* Atmosphere Monitoring Service information" (modified, because hourly
* values are folded into days here), and Open-Meteo's terms ask for credit
* (CC BY 4.0). Both lines come from `airQualityAttributionLines` in the
* contract module and are shown wherever the values are.
*/
import { envFlag, envOr } from "@/lib/env";
import { safeFetch, SafeFetchError } from "@/lib/safe-fetch";
import { shiftDateKey } from "@/lib/tz/format";
import type { AirQualityDay } from "@/lib/environment/air-quality-contract";
import { enumerateDayCount } from "@/lib/environment/day-span";
import {
openMeteoCallWeight,
reserveOpenMeteoCalls,
} from "@/lib/environment/request-budget";
/** Hosted default; override with `OPENMETEO_AIR_QUALITY_URL` (self-host). */
const AIR_QUALITY_BASE_URL = envOr(
"OPENMETEO_AIR_QUALITY_URL",
"https://air-quality-api.open-meteo.com",
).replace(/\/$/, "");
const FETCH_TIMEOUT_MS = 15_000;
/** True when the operator turned the air-quality feed off for the instance. */
export function isAirQualityOperatorDisabled(): boolean {
return envFlag("ENVIRONMENT_AIR_QUALITY_DISABLED");
}
/**
* Whether the air-quality part runs for an account: the operator has not
* turned it off for the instance and the account has not turned it off
* (`User.environmentAirQualityEnabled`).
*/
export function isAirQualityActive(accountEnabled: boolean): boolean {
return accountEnabled && !isAirQualityOperatorDisabled();
}
/** The first day the feed serves. */
export const AIR_QUALITY_EARLIEST_DAY = "2013-01-01";
/**
* From about here the feed carries pollen, UV, dust and aerosol depth. A
* chunk ending before it asks for the pollutant and index variables only.
*/
export const AIR_QUALITY_FULL_FEED_FROM = "2022-06-01";
/** Days per request: bounds the response (about 300 KB) and the heap. */
export const AIR_QUALITY_CHUNK_DAYS = 90;
/** The eight pollutant and index variables, served from 2013. */
const POLLUTANT_VARIABLES = [
"pm2_5",
"pm10",
"ozone",
"nitrogen_dioxide",
"sulphur_dioxide",
"carbon_monoxide",
"european_aqi",
"us_aqi",
] as const;
/** The nine variables the feed fills only from about mid 2022. */
const RECENT_VARIABLES = [
"uv_index",
"dust",
"aerosol_optical_depth",
"alder_pollen",
"birch_pollen",
"grass_pollen",
"mugwort_pollen",
"olive_pollen",
"ragweed_pollen",
] as const;
type HourlyVariable =
(typeof POLLUTANT_VARIABLES)[number] | (typeof RECENT_VARIABLES)[number];
/** All 17 hourly variables, in request order. */
export const AIR_QUALITY_VARIABLES: readonly HourlyVariable[] = [
...POLLUTANT_VARIABLES,
...RECENT_VARIABLES,
];
/** One stored day's air-quality part, as the feed produced it. */
export type DailyAirQualityObservation = { date: string } & Omit<
AirQualityDay,
"apparentMax" | "aqFetchedAt"
>;
/** The hourly block of a response: local ISO times and one array per variable. */
export type AirQualityHourly = { time?: string[] } & Partial<
Record<HourlyVariable, (number | null)[]>
>;
/** Hours a day needs for a mean or a maximum (75 % of 24). */
const MIN_HOURS = 18;
/** Valid hours an 8-hour ozone window needs. */
const MIN_O3_WINDOW_HOURS = 6;
/** Valid 8-hour windows (of 17) an ozone day needs. */
const MIN_O3_WINDOWS = 13;
/** Local hours whose values a UV maximum needs. */
const UV_MIDDAY_HOURS = [10, 11, 12, 13, 14, 15, 16];
function finite(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function round1(value: number): number {
return Math.round(value * 10) / 10;
}
function present(values: readonly (number | null)[]): number[] {
return values.filter((v): v is number => v !== null);
}
/** Mean when the day is covered, else null. Rounded to 0.1. */
export function coveredMean(values: readonly (number | null)[]): number | null {
const xs = present(values);
if (xs.length < MIN_HOURS) return null;
return round1(xs.reduce((a, b) => a + b, 0) / xs.length);
}
/** Maximum when the day is covered, else null. */
export function coveredMax(values: readonly (number | null)[]): number | null {
const xs = present(values);
if (xs.length < MIN_HOURS) return null;
return Math.max(...xs);
}
/**
* Highest 8-hour running mean of the day, from values indexed by local hour
* (0–23, a missing hour null). Null when fewer than 13 of the 17 windows have
* 6 of their 8 hours.
*/
export function ozoneMax8h(byHour: readonly (number | null)[]): number | null {
let best: number | null = null;
let validWindows = 0;
for (let start = 0; start <= 16; start += 1) {
const window = present(byHour.slice(start, start + 8));
if (window.length < MIN_O3_WINDOW_HOURS) continue;
validWindows += 1;
const mean = window.reduce((a, b) => a + b, 0) / window.length;
if (best === null || mean > best) best = mean;
}
if (validWindows < MIN_O3_WINDOWS || best === null) return null;
return round1(best);
}
/** The day's UV maximum when the midday hours are all present, else null. */
export function uvIndexMax(byHour: readonly (number | null)[]): number | null {
if (UV_MIDDAY_HOURS.some((h) => byHour[h] == null)) return null;
const xs = present(byHour);
return xs.length === 0 ? null : Math.max(...xs);
}
/** Whether a coarse location lies inside the CAMS Europe model domain. */
export function insideCamsEurope(lat: number, lon: number): boolean {
return lat >= 30 && lat <= 72 && lon >= -25 && lon <= 45;
}
/**
* Fold an hourly response into one observation per local day present in it.
* Pure; exported for tests. Times are the feed's local `YYYY-MM-DDTHH:MM`;
* a repeated hour (the autumn clock change) keeps its first value.
*/
export function aggregateAirQuality(
hourly: AirQualityHourly,
location: { lat: number; lon: number },
): DailyAirQualityObservation[] {
const times = hourly.time ?? [];
// day → variable → value per local hour (24 slots, null when absent).
const byDay = new Map<string, Map<HourlyVariable, (number | null)[]>>();
for (let i = 0; i < times.length; i += 1) {
const stamp = times[i];
if (typeof stamp !== "string" || stamp.length < 13) continue;
const day = stamp.slice(0, 10);
const hour = Number(stamp.slice(11, 13));
if (!Number.isInteger(hour) || hour < 0 || hour > 23) continue;
let vars = byDay.get(day);
if (!vars) {
vars = new Map();
byDay.set(day, vars);
}
for (const variable of AIR_QUALITY_VARIABLES) {
let slots = vars.get(variable);
if (!slots) {
slots = new Array<number | null>(24).fill(null);
vars.set(variable, slots);
}
if (slots[hour] === null) {
slots[hour] = finite(hourly[variable]?.[i]);
}
}
}
const domain = insideCamsEurope(location.lat, location.lon)
? "cams_europe"
: "cams_global";
const out: DailyAirQualityObservation[] = [];
for (const day of [...byDay.keys()].sort()) {
const vars = byDay.get(day)!;
const v = (name: HourlyVariable) =>
vars.get(name) ?? new Array<number | null>(24).fill(null);
const pm25 = v("pm2_5");
out.push({
date: day,
pm25Mean: coveredMean(pm25),
pm25Max: coveredMax(pm25),
pm10Mean: coveredMean(v("pm10")),
no2Mean: coveredMean(v("nitrogen_dioxide")),
so2Mean: coveredMean(v("sulphur_dioxide")),
coMean: coveredMean(v("carbon_monoxide")),
o3Max8h: ozoneMax8h(v("ozone")),
eaqiMax: coveredMax(v("european_aqi")),
usaqiMax: coveredMax(v("us_aqi")),
uvIndexMax: uvIndexMax(v("uv_index")),
dustMax: coveredMax(v("dust")),
aodMax: coveredMax(v("aerosol_optical_depth")),
pollenAlderMax: coveredMax(v("alder_pollen")),
pollenBirchMax: coveredMax(v("birch_pollen")),
pollenGrassMax: coveredMax(v("grass_pollen")),
pollenMugwortMax: coveredMax(v("mugwort_pollen")),
pollenOliveMax: coveredMax(v("olive_pollen")),
pollenRagweedMax: coveredMax(v("ragweed_pollen")),
aqDomain: domain,
aqHours: present(pm25).length,
});
}
return out;
}
/** A day the feed cannot serve: every value null, no hours, no domain. */
export function uncoverableDay(date: string): DailyAirQualityObservation {
return {
date,
pm25Mean: null,
pm25Max: null,
pm10Mean: null,
no2Mean: null,
so2Mean: null,
coMean: null,
o3Max8h: null,
eaqiMax: null,
usaqiMax: null,
uvIndexMax: null,
dustMax: null,
aodMax: null,
pollenAlderMax: null,
pollenBirchMax: null,
pollenGrassMax: null,
pollenMugwortMax: null,
pollenOliveMax: null,
pollenRagweedMax: null,
aqDomain: null,
aqHours: 0,
};
}
/** Split an inclusive day range into chunks of at most `size` days. */
export function chunkDays(
startDate: string,
endDate: string,
size: number = AIR_QUALITY_CHUNK_DAYS,
): Array<{ startDate: string; endDate: string }> {
const chunks: Array<{ startDate: string; endDate: string }> = [];
let cursor = startDate;
while (cursor <= endDate) {
const last = shiftDateKey(cursor, size - 1);
const chunkEnd = last < endDate ? last : endDate;
chunks.push({ startDate: cursor, endDate: chunkEnd });
cursor = shiftDateKey(chunkEnd, 1);
}
return chunks;
}
/** The variables a chunk asks for: the pollutant set alone before the full era. */
export function variablesForChunk(endDate: string): readonly HourlyVariable[] {
return endDate < AIR_QUALITY_FULL_FEED_FROM
? POLLUTANT_VARIABLES
: AIR_QUALITY_VARIABLES;
}
/** Why a fetch stopped before its last chunk. */
export type AirQualityStop = "budget" | "error" | null;
export interface AirQualityFetchResult {
/** One observation per day the feed answered (and per uncoverable day). */
days: DailyAirQualityObservation[];
/** Null when every chunk was fetched. */
stopped: AirQualityStop;
}
/**
* Fetch and fold the air quality for a coarse location over an inclusive day
* range, newest chunk first, so a run the budget stops part-way has filled
* the recent days the correlation window reads. Never throws for a feed
* problem: the caller gets what was fetched and why it stopped, and the days
* not returned stay unfetched for the next run.
*/
export async function fetchDailyAirQuality(args: {
lat: number;
lon: number;
timezone: string;
startDate: string;
endDate: string;
}): Promise<AirQualityFetchResult> {
const days: DailyAirQualityObservation[] = [];
// Days before the feed's first day are answered here, without a request.
for (
let day = args.startDate;
day <= args.endDate && day < AIR_QUALITY_EARLIEST_DAY;
day = shiftDateKey(day, 1)
) {
days.push(uncoverableDay(day));
}
const start =
args.startDate < AIR_QUALITY_EARLIEST_DAY
? AIR_QUALITY_EARLIEST_DAY
: args.startDate;
if (start > args.endDate) return { days, stopped: null };
const chunks = chunkDays(start, args.endDate).reverse();
for (const chunk of chunks) {
const variables = variablesForChunk(chunk.endDate);
const weight = openMeteoCallWeight(
variables.length,
enumerateDayCount(chunk.startDate, chunk.endDate),
);
if (!(await reserveOpenMeteoCalls(weight))) {
return { days, stopped: "budget" };
}
try {
const hourly = await requestHourly({
lat: args.lat,
lon: args.lon,
timezone: args.timezone,
startDate: chunk.startDate,
endDate: chunk.endDate,
variables,
});
const wanted = (d: string) => d >= chunk.startDate && d <= chunk.endDate;
days.push(
...aggregateAirQuality(hourly, args).filter((d) => wanted(d.date)),
);
} catch {
// A refused, timed-out or unreadable response: stop here and keep what
// came back. The weather row is unaffected, and the days not returned
// stay unfetched for the gap fill.
return { days, stopped: "error" };
}
}
return { days, stopped: null };
}
async function requestHourly(args: {
lat: number;
lon: number;
timezone: string;
startDate: string;
endDate: string;
variables: readonly HourlyVariable[];
}): Promise<AirQualityHourly> {
const url = new URL(`${AIR_QUALITY_BASE_URL}/v1/air-quality`);
url.searchParams.set("latitude", String(args.lat));
url.searchParams.set("longitude", String(args.lon));
url.searchParams.set("hourly", args.variables.join(","));
url.searchParams.set("start_date", args.startDate);
url.searchParams.set("end_date", args.endDate);
url.searchParams.set("timezone", args.timezone);
url.searchParams.set("domains", "auto");
const res = await safeFetch(
url,
{ headers: { accept: "application/json" } },
{ timeoutMs: FETCH_TIMEOUT_MS },
);
if (!res.ok) {
throw new SafeFetchError(
`open-meteo air quality returned HTTP ${res.status}`,
"network",
);
}
const body = (await res.json()) as { hourly?: AirQualityHourly };
return body.hourly ?? {};
}
+39 -5
View File
@@ -19,7 +19,14 @@
* only" — see the nightly job and the backfill route.
*/
import { envOr } from "@/lib/env";
import { annotate } from "@/lib/logging/context";
import { safeFetch, SafeFetchError } from "@/lib/safe-fetch";
import { enumerateDayCount } from "@/lib/environment/day-span";
import {
OpenMeteoBudgetExhaustedError,
openMeteoCallWeight,
reserveOpenMeteoCalls,
} from "@/lib/environment/request-budget";
/** Hosted archive default; override with `OPENMETEO_BASE_URL` (self-host). */
const ARCHIVE_BASE_URL = envOr(
@@ -58,6 +65,8 @@ export interface DailyEnvironmentObservation {
tempMax: number | null;
tempMean: number | null;
apparentMean: number | null;
/** v1.42 — daily apparent ("feels-like") maximum, °C. */
apparentMax: number | null;
sunshineSec: number | null;
daylightSec: number | null;
precipSum: number | null;
@@ -97,6 +106,7 @@ interface ArchiveDaily {
temperature_2m_max?: (number | null)[];
temperature_2m_mean?: (number | null)[];
apparent_temperature_mean?: (number | null)[];
apparent_temperature_max?: (number | null)[];
sunshine_duration?: (number | null)[];
daylight_duration?: (number | null)[];
precipitation_sum?: (number | null)[];
@@ -115,22 +125,29 @@ interface ArchiveResponse {
hourly?: ArchiveHourly;
}
const DAILY_FIELDS = [
const DAILY_FIELD_LIST = [
"temperature_2m_min",
"temperature_2m_max",
"temperature_2m_mean",
"apparent_temperature_mean",
"apparent_temperature_max",
"sunshine_duration",
"daylight_duration",
"precipitation_sum",
"weather_code",
].join(",");
];
const DAILY_FIELDS = DAILY_FIELD_LIST.join(",");
const HOURLY_FIELDS = [
const HOURLY_FIELD_LIST = [
"surface_pressure",
"relative_humidity_2m",
"cloud_cover",
].join(",");
];
const HOURLY_FIELDS = HOURLY_FIELD_LIST.join(",");
/** Variables one archive request asks for, for the request budget. */
const ARCHIVE_VARIABLE_COUNT =
DAILY_FIELD_LIST.length + HOURLY_FIELD_LIST.length;
/** Per-day aggregation accumulator for the hourly-only fields. */
interface HourlyDayAgg {
@@ -174,6 +191,13 @@ export async function geocodeLocation(
const trimmed = query.trim();
if (trimmed.length === 0) return [];
// A geocoder call weighs one; a refused one answers "no match" rather than
// reaching the host, and says why on the request's event.
if (!(await reserveOpenMeteoCalls(1))) {
annotate({ action: { name: "environment.geocode.budget_exhausted" } });
return [];
}
const url = new URL(`${GEOCODING_BASE_URL}/v1/search`);
url.searchParams.set("name", trimmed);
url.searchParams.set("count", String(Math.min(Math.max(count, 1), 10)));
@@ -230,7 +254,9 @@ export async function geocodeLocation(
* Fetch the daily environment observations for a coarse location over a date
* range (inclusive YYYY-MM-DD). The hourly-only fields (pressure / humidity /
* cloud) are requested alongside and aggregated to per-day values here. Throws
* a {@link SafeFetchError} on egress failure so the caller can classify it.
* a {@link SafeFetchError} on egress failure so the caller can classify it,
* and an {@link OpenMeteoBudgetExhaustedError} when the instance-wide request
* budget refuses the call (nothing is sent then).
*/
export async function fetchDailyEnvironment(args: {
lat: number;
@@ -239,6 +265,13 @@ export async function fetchDailyEnvironment(args: {
startDate: string;
endDate: string;
}): Promise<DailyEnvironmentObservation[]> {
const weight = openMeteoCallWeight(
ARCHIVE_VARIABLE_COUNT,
enumerateDayCount(args.startDate, args.endDate),
);
if (!(await reserveOpenMeteoCalls(weight))) {
throw new OpenMeteoBudgetExhaustedError();
}
const url = new URL(`${ARCHIVE_BASE_URL}/v1/archive`);
url.searchParams.set("latitude", String(args.lat));
url.searchParams.set("longitude", String(args.lon));
@@ -285,6 +318,7 @@ export async function fetchDailyEnvironment(args: {
tempMax: firstNumber(daily.temperature_2m_max?.[i]),
tempMean: firstNumber(daily.temperature_2m_mean?.[i]),
apparentMean: firstNumber(daily.apparent_temperature_mean?.[i]),
apparentMax: firstNumber(daily.apparent_temperature_max?.[i]),
sunshineSec: firstNumber(daily.sunshine_duration?.[i]),
daylightSec: firstNumber(daily.daylight_duration?.[i]),
precipSum: firstNumber(daily.precipitation_sum?.[i]),
+146
View File
@@ -0,0 +1,146 @@
/**
* The instance-wide Open-Meteo request budget (v1.42, #615).
*
* The hosted Open-Meteo APIs are free for non-commercial use within 600 calls
* a minute, 5 000 an hour and 10 000 a day, counted per client address, so
* per instance. A "call" is weighted: a request for more than 10 variables
* or more than two weeks counts as several, `max(1, variables / 10) ×
* max(1, days / 14)` (open-meteo.com/en/pricing). One account's full
* backfill of weather and air quality is around 150 calls; a large instance
* filling hundreds of accounts in one night would run into the daily limit
* and every request after it would fail.
*
* So every request to the archive, the geocoder and the air-quality API asks
* this budget first, with its weight, and is not sent when the budget says
* no. The limits sit below the free ones (500 / 4 000 / 8 000) to leave room
* for clock skew between our windows and theirs. A refused request is not an
* error: the nightly job stops, reports `budget_blocked`, and the next night
* continues where it stopped, because what is missing is exactly what the
* gap fill looks for.
*
* The counter lives in Postgres (`rate_limits`, the table every rate limit
* here uses), so the web process and the worker share it, and the three
* windows are checked and charged together under one advisory lock: a
* request is admitted only when all three have room, and then charged to
* all three. Weights are stored in hundredths of a call, because the column
* is an integer and a 90-day air-quality chunk weighs about 10.9 calls.
*
* An operator running their own Open-Meteo instance (`OPENMETEO_*_URL`) is
* not bound by the hosted limits, but the budget still applies: it is cheap,
* and it keeps a misconfigured instance from being hammered.
*/
import { prisma } from "@/lib/db";
/** One budget window. */
export interface BudgetWindow {
/** Bucket key suffix, also the window's name. */
name: "minute" | "hour" | "day";
windowMs: number;
/** Calls admitted per window (whole calls). */
limit: number;
}
export const OPEN_METEO_BUDGET_WINDOWS: readonly BudgetWindow[] = [
{ name: "minute", windowMs: 60_000, limit: 500 },
{ name: "hour", windowMs: 3_600_000, limit: 4_000 },
{ name: "day", windowMs: 86_400_000, limit: 8_000 },
];
const BUCKET_PREFIX = "open-meteo-budget";
const LOCK_KEY = "open-meteo-budget";
/** Hundredths of a call: the unit the buckets count in. */
const CENTI = 100;
/**
* The weight of one request in calls, by the hosted pricing rule: more than
* 10 variables or more than 14 days counts proportionally more.
*/
export function openMeteoCallWeight(variables: number, days: number): number {
return Math.max(1, variables / 10) * Math.max(1, days / 14);
}
/** A bucket as stored: count in hundredths, and when its window resets. */
export interface BudgetBucket {
count: number;
resetAt: Date;
}
/**
* Pure admission decision: given the stored buckets (keyed by window name,
* absent when never written), whether `weightCenti` fits in every window, and
* the buckets as they must be written when it does. An expired bucket starts
* over at the request's weight. Exported for tests.
*/
export function admitWeight(
buckets: ReadonlyMap<BudgetWindow["name"], BudgetBucket>,
weightCenti: number,
now: Date,
windows: readonly BudgetWindow[] = OPEN_METEO_BUDGET_WINDOWS,
):
| { admitted: true; next: Map<BudgetWindow["name"], BudgetBucket> }
| { admitted: false; window: BudgetWindow["name"] } {
const next = new Map<BudgetWindow["name"], BudgetBucket>();
for (const w of windows) {
const current = buckets.get(w.name);
const live = current && current.resetAt.getTime() > now.getTime();
const count = (live ? current.count : 0) + weightCenti;
if (count > w.limit * CENTI) return { admitted: false, window: w.name };
next.set(w.name, {
count,
resetAt: live ? current.resetAt : new Date(now.getTime() + w.windowMs),
});
}
return { admitted: true, next };
}
/**
* Ask the budget for one request of `weight` calls. Returns true and charges
* every window when it fits, false (charging nothing) when any window is
* full.
*/
export async function reserveOpenMeteoCalls(weight: number): Promise<boolean> {
const weightCenti = Math.max(1, Math.ceil(weight * CENTI));
const keys = OPEN_METEO_BUDGET_WINDOWS.map(
(w) => `${BUCKET_PREFIX}:${w.name}`,
);
return prisma.$transaction(async (tx) => {
// `pg_advisory_xact_lock` returns void, which the client cannot
// deserialize as a column; selecting FROM it yields a plain row.
await tx.$queryRaw`
SELECT 1 AS locked
FROM pg_advisory_xact_lock(hashtextextended(${LOCK_KEY}, 0))
`;
const rows = await tx.rateLimit.findMany({
where: { key: { in: keys } },
select: { key: true, count: true, resetAt: true },
});
const buckets = new Map<BudgetWindow["name"], BudgetBucket>();
for (const row of rows) {
const name = row.key.slice(BUCKET_PREFIX.length + 1) as
BudgetWindow["name"] | string;
if (name === "minute" || name === "hour" || name === "day") {
buckets.set(name, { count: row.count, resetAt: row.resetAt });
}
}
const decision = admitWeight(buckets, weightCenti, new Date());
if (!decision.admitted) return false;
for (const [name, bucket] of decision.next) {
const key = `${BUCKET_PREFIX}:${name}`;
await tx.rateLimit.upsert({
where: { key },
create: { key, count: bucket.count, resetAt: bucket.resetAt },
update: { count: bucket.count, resetAt: bucket.resetAt },
});
}
return true;
});
}
/** Thrown by a client whose request the budget refused. Not a failure. */
export class OpenMeteoBudgetExhaustedError extends Error {
constructor() {
super("open-meteo request budget exhausted");
this.name = "OpenMeteoBudgetExhaustedError";
}
}
+335 -52
View File
@@ -20,13 +20,31 @@
*
* Coarse location only — the resolver never sees finer than the rounded city
* coordinates stored on the user / override.
*
* v1.42 (#615) — the locations are sealed at rest (`location-cipher.ts`): the
* home and the periods are opened here, the rows are written with the sealed
* copy only, and the requests still carry nothing but the coarse coordinates.
* The same run fetches the day's air quality, pollen and UV when the account
* and the operator allow it (`isAirQualityActive`). An air-quality failure
* never costs the weather row: the weather is stored, the air-quality part
* stays unfetched (`aqFetchedAt` null), and the nightly gap fill
* ({@link fillAirQualityGaps}) asks for it again.
*/
import { prisma } from "@/lib/db";
import {
fetchDailyEnvironment,
roundCoarse,
type DailyEnvironmentObservation,
} from "@/lib/environment/open-meteo";
import {
fetchDailyAirQuality,
isAirQualityActive,
type AirQualityStop,
type DailyAirQualityObservation,
} from "@/lib/environment/open-meteo-air-quality";
import { readLocation, sealLocation } from "@/lib/environment/location-cipher";
export { isAirQualityActive };
import { OpenMeteoBudgetExhaustedError } from "@/lib/environment/request-budget";
import type { EnvironmentLocationSource } from "@/generated/prisma/client";
import { shiftDateKey } from "@/lib/tz/format";
@@ -134,6 +152,81 @@ export interface FetchAndStoreResult {
skipped: number;
/** Distinct upstream fetches made. */
fetches: number;
/**
* True when the instance-wide request budget refused a request, so some
* days were left for the next run.
*/
budgetBlocked: boolean;
}
/** The user columns every environment read resolves the home from. */
const HOME_SELECT = {
homeLat: true,
homeLon: true,
homeLabel: true,
homeLocationEncrypted: true,
homeTimezone: true,
homeSince: true,
timezone: true,
environmentAirQualityEnabled: true,
} as const;
/**
* The account's home as the resolver uses it: the sealed copy opened, or the
* readable columns for a row the encryption backfill has not reached. Null
* when no home is set (or the sealed value does not open).
*/
export function resolveHome(user: {
homeLat: number | null;
homeLon: number | null;
homeLabel: string | null;
homeLocationEncrypted: Uint8Array | null;
homeTimezone: string | null;
homeSince: Date | null;
timezone: string;
}): HomeLocation | null {
const location = readLocation({
sealed: user.homeLocationEncrypted,
lat: user.homeLat,
lon: user.homeLon,
label: user.homeLabel ?? "",
});
if (!location) return null;
return {
...location,
// A home stored without a label (allowed since v1.25) carries an empty
// one; the stored day says "Home", as it always has.
label: location.label === "" ? "Home" : location.label,
timezone: user.homeTimezone ?? user.timezone,
// Effective-from day-key; null ⇒ home resolves no day (conservative).
since: user.homeSince ? utcDayKey(user.homeSince) : null,
};
}
/** Every dated location period of an account, opened. */
async function readTravelOverrides(userId: string): Promise<TravelOverride[]> {
const records = await prisma.environmentTravelLocation.findMany({
where: { userId },
select: {
startDate: true,
endDate: true,
lat: true,
lon: true,
label: true,
locationEncrypted: true,
},
});
return records.flatMap((row) => {
const location = readLocation({
sealed: row.locationEncrypted,
lat: row.lat,
lon: row.lon,
label: row.label,
});
return location
? [{ startDate: row.startDate, endDate: row.endDate, ...location }]
: [];
});
}
/**
@@ -151,55 +244,21 @@ export async function fetchAndStoreEnvironment(args: {
const user = await prisma.user.findUnique({
where: { id: userId },
select: {
homeLat: true,
homeLon: true,
homeLabel: true,
homeTimezone: true,
homeSince: true,
timezone: true,
},
select: HOME_SELECT,
});
if (!user) return { stored: 0, skipped: 0, fetches: 0 };
if (!user) {
return { stored: 0, skipped: 0, fetches: 0, budgetBlocked: false };
}
const home: HomeLocation | null =
user.homeLat != null && user.homeLon != null
? {
lat: user.homeLat,
lon: user.homeLon,
label: user.homeLabel ?? "Home",
timezone: user.homeTimezone ?? user.timezone,
// Effective-from day-key; null ⇒ home resolves no day (conservative).
since: user.homeSince ? utcDayKey(user.homeSince) : null,
}
: null;
const travelRecords = await prisma.environmentTravelLocation.findMany({
where: { userId },
select: {
startDate: true,
endDate: true,
lat: true,
lon: true,
label: true,
},
});
// Since v1.42 the plaintext location is nullable (migration 0382): a period
// whose plaintext the encryption backfill has already cleared resolves only
// through its sealed copy, which this reader does not open yet.
const travelRows: TravelOverride[] = travelRecords.flatMap((row) =>
row.lat != null && row.lon != null && row.label != null
? [{ ...row, lat: row.lat, lon: row.lon, label: row.label }]
: [],
);
const home = resolveHome(user);
const travelRows = await readTravelOverrides(userId);
const airQuality = isAirQualityActive(user.environmentAirQualityEnabled);
// The timezone used to enumerate AND fetch days — one tz keeps the stored
// `date` consistent with the user's day-keys elsewhere.
const timezone = home?.timezone ?? user.timezone;
const days = enumerateDays(startDate, endDate);
const resolvedByDay = new Map<string, ResolvedLocation>();
const groups = new Map<string, { loc: ResolvedLocation; days: string[] }>();
let skipped = 0;
for (const day of days) {
@@ -208,7 +267,6 @@ export async function fetchAndStoreEnvironment(args: {
skipped += 1;
continue;
}
resolvedByDay.set(day, loc);
const gk = groupKey(loc);
const group = groups.get(gk) ?? { loc, days: [] };
group.days.push(day);
@@ -217,6 +275,7 @@ export async function fetchAndStoreEnvironment(args: {
let stored = 0;
let fetches = 0;
let budgetBlocked = false;
for (const { loc, days: groupDays } of groups.values()) {
const min = groupDays[0];
const max = groupDays[groupDays.length - 1];
@@ -229,14 +288,38 @@ export async function fetchAndStoreEnvironment(args: {
startDate: min,
endDate: max,
});
} catch {
} catch (err) {
if (err instanceof OpenMeteoBudgetExhaustedError) {
// Nothing was sent. The remaining groups wait for the next run.
budgetBlocked = true;
break;
}
// A failed fetch for one location leaves its days un-stored; the nightly
// lookback re-attempts them. Other locations still get their rows.
continue;
}
fetches += 1;
// The air-quality part of the same days, at the same coarse location. A
// stop (budget or feed error) leaves the days it did not reach without
// air quality; the weather is stored regardless.
let airByDay: Map<string, DailyAirQualityObservation> | null = null;
if (airQuality) {
const result = await fetchDailyAirQuality({
lat: loc.lat,
lon: loc.lon,
timezone,
startDate: min,
endDate: max,
});
if (result.stopped === "budget") budgetBlocked = true;
airByDay = new Map(result.days.map((d) => [d.date, d]));
fetches += result.days.length > 0 ? 1 : 0;
}
const wanted = new Set(groupDays);
const sealed = sealLocation(loc);
const now = new Date();
// Batch the group's day upserts into a single transaction round-trip
// rather than awaiting one upsert per day in series (a 730-day backfill
// would otherwise issue 730 sequential statements). One transaction per
@@ -244,34 +327,111 @@ export async function fetchAndStoreEnvironment(args: {
// preserving the idempotent upsert semantics.
const ops = observations
.filter((obs) => wanted.has(obs.date))
.map((obs) => buildUpsert(userId, loc, obs));
.map((obs) =>
buildUpsert(userId, loc, sealed, obs, {
active: airQuality,
day: airByDay?.get(obs.date) ?? null,
now,
}),
);
if (ops.length > 0) {
await prisma.$transaction(ops);
stored += ops.length;
}
// Days the feed did not return (e.g. beyond the settling lag) stay absent.
if (budgetBlocked) break;
}
return { stored, skipped, fetches };
return { stored, skipped, fetches, budgetBlocked };
}
/** The air-quality columns of a stored day, from one observation. */
function airQualityColumns(day: DailyAirQualityObservation, fetchedAt: Date) {
return {
pm25Mean: day.pm25Mean,
pm25Max: day.pm25Max,
pm10Mean: day.pm10Mean,
no2Mean: day.no2Mean,
so2Mean: day.so2Mean,
coMean: day.coMean,
o3Max8h: day.o3Max8h,
eaqiMax: day.eaqiMax,
usaqiMax: day.usaqiMax,
uvIndexMax: day.uvIndexMax,
dustMax: day.dustMax,
aodMax: day.aodMax,
pollenAlderMax: day.pollenAlderMax,
pollenBirchMax: day.pollenBirchMax,
pollenGrassMax: day.pollenGrassMax,
pollenMugwortMax: day.pollenMugwortMax,
pollenOliveMax: day.pollenOliveMax,
pollenRagweedMax: day.pollenRagweedMax,
aqDomain: day.aqDomain,
aqHours: day.aqHours,
aqFetchedAt: fetchedAt,
};
}
/** The air-quality columns of a day whose air quality is not (yet) known. */
const AIR_QUALITY_UNFETCHED = {
pm25Mean: null,
pm25Max: null,
pm10Mean: null,
no2Mean: null,
so2Mean: null,
coMean: null,
o3Max8h: null,
eaqiMax: null,
usaqiMax: null,
uvIndexMax: null,
dustMax: null,
aodMax: null,
pollenAlderMax: null,
pollenBirchMax: null,
pollenGrassMax: null,
pollenMugwortMax: null,
pollenOliveMax: null,
pollenRagweedMax: null,
aqDomain: null,
aqHours: null,
aqFetchedAt: null,
} as const;
/** Build the idempotent per-day upsert operation (un-awaited, for a batch). */
function buildUpsert(
userId: string,
loc: ResolvedLocation,
sealed: Uint8Array<ArrayBuffer>,
obs: DailyEnvironmentObservation,
air: {
active: boolean;
day: DailyAirQualityObservation | null;
now: Date;
},
) {
// With air quality on, the row's air-quality part is rewritten with the
// weather: the fetched values, or nothing at all when the fetch did not
// reach the day, so a value measured for the location the day resolved to
// before can never stand beside the weather of the one it resolves to now.
// With it off, the stored part is left as it is.
const airColumns = !air.active
? {}
: air.day
? airQualityColumns(air.day, air.now)
: AIR_QUALITY_UNFETCHED;
const data = {
// Already coarse at the source; rounded again so a per-day row never holds
// more precision than the privacy floor, whatever resolved it.
lat: roundCoarse(loc.lat),
lon: roundCoarse(loc.lon),
locationLabel: loc.label,
// The coarse location rides sealed only; the readable columns are
// written empty (they drop in a later release).
locationEncrypted: sealed,
lat: null,
lon: null,
locationLabel: null,
source: loc.source,
tempMin: obs.tempMin,
tempMax: obs.tempMax,
tempMean: obs.tempMean,
apparentMean: obs.apparentMean,
apparentMax: obs.apparentMax,
sunshineSec: obs.sunshineSec != null ? Math.round(obs.sunshineSec) : null,
daylightSec: obs.daylightSec != null ? Math.round(obs.daylightSec) : null,
precipSum: obs.precipSum,
@@ -280,7 +440,8 @@ function buildUpsert(
humidityMean: obs.humidityMean,
cloudMean: obs.cloudMean,
weatherCode: obs.weatherCode,
fetchedAt: new Date(),
fetchedAt: air.now,
...airColumns,
};
return prisma.environmentContext.upsert({
where: { userId_date: { userId, date: obs.date } },
@@ -288,3 +449,125 @@ function buildUpsert(
update: data,
});
}
/** Days per gap-fill run: about 11 weighted calls per account per night. */
export const AIR_QUALITY_GAP_FILL_DAYS = 90;
/** Two gap days further apart than this are fetched as separate ranges. */
const GAP_RANGE_JOIN_DAYS = 7;
export interface AirQualityGapFillResult {
/** Stored days whose air-quality part was filled (or marked uncoverable). */
filled: number;
/** Why the run stopped early, or null. */
stopped: AirQualityStop;
}
/**
* The nightly gap fill: up to {@link AIR_QUALITY_GAP_FILL_DAYS} stored days
* of an account whose air-quality part was never fetched (`aqFetchedAt`
* null), newest first, so the correlation window fills first and an account
* with two years of weather catches up within about nine nights. Each day is
* fetched at the location it was stored for (opened from its sealed copy),
* never re-resolved. A day the feed cannot serve (before 2013) is marked
* fetched with every value null, so it is not asked for again. A no-op when
* the air-quality part is off for the account or the instance.
*/
export async function fillAirQualityGaps(
userId: string,
maxDays: number = AIR_QUALITY_GAP_FILL_DAYS,
): Promise<AirQualityGapFillResult> {
const user = await prisma.user.findUnique({
where: { id: userId },
select: HOME_SELECT,
});
if (!user || !isAirQualityActive(user.environmentAirQualityEnabled)) {
return { filled: 0, stopped: null };
}
const timezone = user.homeTimezone ?? user.timezone;
const rows = await prisma.environmentContext.findMany({
where: { userId, aqFetchedAt: null },
orderBy: { date: "desc" },
take: maxDays,
select: {
date: true,
lat: true,
lon: true,
locationLabel: true,
locationEncrypted: true,
},
});
// Group the days by location, then cut each group into ranges of nearby
// days, so a sparse group does not fetch the empty stretch between.
const byLocation = new Map<
string,
{ lat: number; lon: number; days: string[] }
>();
for (const row of rows) {
const location = readLocation({
sealed: row.locationEncrypted,
lat: row.lat,
lon: row.lon,
label: row.locationLabel,
});
if (!location) continue;
const key = `${location.lat},${location.lon}`;
const group = byLocation.get(key) ?? {
lat: location.lat,
lon: location.lon,
days: [],
};
group.days.push(row.date);
byLocation.set(key, group);
}
let filled = 0;
for (const group of byLocation.values()) {
for (const range of nearbyRanges(group.days)) {
const result = await fetchDailyAirQuality({
lat: group.lat,
lon: group.lon,
timezone,
startDate: range[0],
endDate: range[range.length - 1],
});
const wanted = new Set(range);
const now = new Date();
const ops = result.days
.filter((day) => wanted.has(day.date))
.map((day) =>
prisma.environmentContext.updateMany({
// Still unfetched, so a weather rewrite that landed meanwhile
// (with its own air quality) is not overwritten.
where: { userId, date: day.date, aqFetchedAt: null },
data: airQualityColumns(day, now),
}),
);
if (ops.length > 0) {
const done = await prisma.$transaction(ops);
filled += done.reduce((sum, r) => sum + r.count, 0);
}
if (result.stopped) return { filled, stopped: result.stopped };
}
}
return { filled, stopped: null };
}
/** Sorted runs of day keys, a new run wherever two days lie far apart. */
export function nearbyRanges(days: readonly string[]): string[][] {
const sorted = [...new Set(days)].sort();
const ranges: string[][] = [];
for (const day of sorted) {
const current = ranges[ranges.length - 1];
if (
current &&
shiftDateKey(current[current.length - 1], GAP_RANGE_JOIN_DAYS) >= day
) {
current.push(day);
} else {
ranges.push([day]);
}
}
return ranges;
}