mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
Merge the air quality history hardening into release/v1.42.0
This commit is contained in:
commit
ecf27cd9bc
8 files changed
+592
-303
No files matched your search
@@ -1,19 +1,19 @@
|
||||
/**
|
||||
* v1.42 — the air-quality history backfill's pure parts: how pending days
|
||||
* are cut into request ranges, what a range weighs, the ceilings that keep
|
||||
* the history from taking the nightly fetch's room, where a day without a
|
||||
* row is placed, and when an account is offered a run again.
|
||||
* the history from taking the nightly fetch's room, the entry-day snapshot a
|
||||
* chain hands on, and when an account is offered a run again.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS,
|
||||
historyRangeWeight,
|
||||
historyRequestFits,
|
||||
AIR_QUALITY_HISTORY_CEILING,
|
||||
planHistoryRanges,
|
||||
readAirQualityHistoryState,
|
||||
resolveHistoryLocation,
|
||||
readEntryDaysSnapshot,
|
||||
} from "../air-quality-history";
|
||||
import { airQualityChunkWeight } from "../open-meteo-air-quality";
|
||||
import { historyDue } from "@/lib/jobs/environment-air-quality-history";
|
||||
import { shiftDateKey } from "@/lib/tz/format";
|
||||
|
||||
@@ -64,100 +64,35 @@ describe("planHistoryRanges", () => {
|
||||
const days = daysFrom("2023-01-01", 122, 3);
|
||||
const ranges = planHistoryRanges(days);
|
||||
const ranged = ranges.reduce(
|
||||
(sum, r) => sum + historyRangeWeight(r, []),
|
||||
(sum, r) => sum + airQualityChunkWeight(r.startDate, r.endDate),
|
||||
0,
|
||||
);
|
||||
const perDay =
|
||||
days.length *
|
||||
historyRangeWeight(
|
||||
{ startDate: days[0], endDate: days[0], days: [days[0]] },
|
||||
[],
|
||||
);
|
||||
const perDay = days.length * airQualityChunkWeight(days[0], days[0]);
|
||||
expect(ranged).toBeLessThan(perDay / 4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("historyRangeWeight", () => {
|
||||
it("adds the weather only over the span of the days without a row", () => {
|
||||
const range = {
|
||||
startDate: "2023-01-01",
|
||||
endDate: "2023-03-31",
|
||||
days: ["2023-01-01", "2023-03-31"],
|
||||
};
|
||||
const airOnly = historyRangeWeight(range, []);
|
||||
expect(airOnly).toBeCloseTo((1.7 * 90) / 14);
|
||||
// One missing day: one weather request at its minimum weight (12 vars).
|
||||
expect(historyRangeWeight(range, ["2023-03-31"])).toBeCloseTo(
|
||||
airOnly + 1.2,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("historyRequestFits", () => {
|
||||
it("admits a request under every ceiling", () => {
|
||||
expect(historyRequestFits({}, 10)).toBe(true);
|
||||
});
|
||||
|
||||
it("holds the account to three quarters of its daily share", () => {
|
||||
describe("the history's ceiling", () => {
|
||||
it("holds the history to three quarters of the account's share and half of each instance window", () => {
|
||||
expect(AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS).toBe(300);
|
||||
expect(historyRequestFits({ "account-day": 295 }, 5)).toBe(true);
|
||||
expect(historyRequestFits({ "account-day": 295 }, 5.1)).toBe(false);
|
||||
});
|
||||
|
||||
it("holds the history to half of each instance window", () => {
|
||||
expect(historyRequestFits({ minute: 245 }, 5)).toBe(true);
|
||||
expect(historyRequestFits({ minute: 246 }, 5)).toBe(false);
|
||||
expect(historyRequestFits({ hour: 1999 }, 2)).toBe(false);
|
||||
expect(historyRequestFits({ day: 3995 }, 10)).toBe(false);
|
||||
expect(AIR_QUALITY_HISTORY_CEILING).toEqual({
|
||||
instanceShare: 0.5,
|
||||
accountDayCalls: 300,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveHistoryLocation", () => {
|
||||
const home = {
|
||||
lat: 51.5,
|
||||
lon: 7.2,
|
||||
label: "Home",
|
||||
timezone: "Europe/Berlin",
|
||||
since: "2026-09-01",
|
||||
};
|
||||
const trip = {
|
||||
startDate: "2024-07-01",
|
||||
endDate: "2024-07-14",
|
||||
lat: 38.7,
|
||||
lon: -9.1,
|
||||
label: "Lisbon",
|
||||
};
|
||||
|
||||
it("places a day of a trip at the trip", () => {
|
||||
expect(resolveHistoryLocation("2024-07-05", home, [trip])).toMatchObject({
|
||||
lat: 38.7,
|
||||
source: "TRAVEL",
|
||||
});
|
||||
});
|
||||
|
||||
it("places a day from the home's effective date on at the home", () => {
|
||||
expect(resolveHistoryLocation("2026-09-01", home, [trip])).toMatchObject({
|
||||
lat: 51.5,
|
||||
source: "HOME",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves a day before the home was set without a place, unless a period covers it", () => {
|
||||
expect(resolveHistoryLocation("2026-08-31", home, [trip])).toBeNull();
|
||||
expect(resolveHistoryLocation("2020-03-01", home, [trip])).toBeNull();
|
||||
expect(resolveHistoryLocation("2024-07-14", home, [trip])).toMatchObject({
|
||||
source: "TRAVEL",
|
||||
});
|
||||
});
|
||||
|
||||
it("places every day at a home stored without an effective date", () => {
|
||||
expect(
|
||||
resolveHistoryLocation("2015-01-01", { ...home, since: null }, []),
|
||||
).toMatchObject({ lat: 51.5, source: "HOME" });
|
||||
});
|
||||
|
||||
it("places nothing without a home or a period", () => {
|
||||
expect(resolveHistoryLocation("2020-03-01", null, [trip])).toBeNull();
|
||||
describe("entry-day snapshot", () => {
|
||||
it("reads a snapshot from a payload and refuses anything else", () => {
|
||||
const snapshot = {
|
||||
timezone: "Europe/Berlin",
|
||||
cutoff: "2026-10-01",
|
||||
days: ["2024-01-01"],
|
||||
};
|
||||
expect(readEntryDaysSnapshot(snapshot)).toEqual(snapshot);
|
||||
expect(readEntryDaysSnapshot(undefined)).toBeNull();
|
||||
expect(readEntryDaysSnapshot({ ...snapshot, days: [1] })).toBeNull();
|
||||
expect(readEntryDaysSnapshot({ ...snapshot, cutoff: 1 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ import {
|
||||
admitWeight,
|
||||
openMeteoCallWeight,
|
||||
reserveOpenMeteoCalls,
|
||||
windowsUnderCeiling,
|
||||
type BudgetBucket,
|
||||
} from "../request-budget";
|
||||
|
||||
@@ -134,3 +135,44 @@ describe("the per-account share", () => {
|
||||
expect(await reserveOpenMeteoCalls(1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a caller's own ceiling", () => {
|
||||
const ceiling = { instanceShare: 0.5, accountDayCalls: 300 };
|
||||
|
||||
it("lowers each instance window to its share and the account's day to its calls", () => {
|
||||
const windows = windowsUnderCeiling(
|
||||
[...OPEN_METEO_BUDGET_WINDOWS, OPEN_METEO_ACCOUNT_DAY_WINDOW],
|
||||
ceiling,
|
||||
);
|
||||
expect(windows.map((w) => [w.name, w.limit, w.windowMs])).toEqual([
|
||||
["minute", 250, 60_000],
|
||||
["hour", 2_000, 3_600_000],
|
||||
["day", 4_000, 86_400_000],
|
||||
["account-day", 300, 86_400_000],
|
||||
]);
|
||||
expect(windowsUnderCeiling(OPEN_METEO_BUDGET_WINDOWS, undefined)).toBe(
|
||||
OPEN_METEO_BUDGET_WINDOWS,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses past the ceiling what the budget alone would admit, and charges nothing", async () => {
|
||||
store.clear();
|
||||
const later = new Date(Date.now() + 3_600_000);
|
||||
store.set("open-meteo-budget:account:acct-c:day", {
|
||||
count: 295 * 100,
|
||||
resetAt: later,
|
||||
});
|
||||
expect(await reserveOpenMeteoCalls(5, "acct-c", ceiling)).toBe(true);
|
||||
expect(await reserveOpenMeteoCalls(0.1, "acct-c", ceiling)).toBe(false);
|
||||
expect(store.get("open-meteo-budget:account:acct-c:day")!.count).toBe(
|
||||
300 * 100,
|
||||
);
|
||||
// The same request without the ceiling is still inside the budget.
|
||||
expect(await reserveOpenMeteoCalls(0.1, "acct-c")).toBe(true);
|
||||
|
||||
store.clear();
|
||||
store.set("open-meteo-budget:minute", { count: 246 * 100, resetAt: later });
|
||||
expect(await reserveOpenMeteoCalls(5, "acct-c", ceiling)).toBe(false);
|
||||
expect(await reserveOpenMeteoCalls(4, "acct-c", ceiling)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -15,13 +15,15 @@
|
||||
* (only the air-quality part is fetched then, as the nightly gap fill
|
||||
* does);
|
||||
* - otherwise in the dated location period that covers it, else at the
|
||||
* home, but the home only from `homeSince` on (or on every day when no
|
||||
* `homeSince` is stored). A day before the home was set is never placed
|
||||
* there: the weather and air of a place the person was not at would be
|
||||
* wrong data in the correlations, the Coach and the day view, and an
|
||||
* empty day is honest where a wrong one is not. Such a day is filled
|
||||
* only when a period covers it. A day without a row gets its weather row
|
||||
* with the air quality, because the air quality is stored on it.
|
||||
* home, but the home only from `homeSince` on, by the same rule the
|
||||
* nightly fetch places a day with (`resolveLocationForDay`). A home
|
||||
* without a `homeSince` places no day at all, there as here. A day
|
||||
* before the home was set is never placed there: the weather and air of
|
||||
* a place the person was not at would be wrong data in the
|
||||
* correlations, the Coach and the day view, and an empty day is honest
|
||||
* where a wrong one is not. Such a day is filled only when a period
|
||||
* covers it. A day without a row gets its weather row with the air
|
||||
* quality, because the air quality is stored on it.
|
||||
*
|
||||
* A day inside a period whose sealed location does not open is skipped (fail
|
||||
* closed, as in the service), and so is a day with no place at all; neither
|
||||
@@ -40,7 +42,19 @@
|
||||
* over 14 (`openMeteoCallWeight`), so a lone day costs as much as two weeks:
|
||||
* joining short gaps is free, splitting long ones saves the gap. The newest
|
||||
* ranges go first, so the window the correlations read fills first. The
|
||||
* weather of a range is asked only over the span of its days without a row.
|
||||
* weather of a range is asked only over the span of its days without a row,
|
||||
* and before its air quality: a weather request the feed or the budget
|
||||
* refuses then costs no air-quality call, and when the air quality fails
|
||||
* after the weather came back, the weather rows are stored without it, so
|
||||
* the next run asks for the air quality alone and nothing paid is asked for
|
||||
* twice. A day the archive answer leaves out gets no row: it stays pending
|
||||
* rather than carrying an empty weather row that nothing would fill again.
|
||||
*
|
||||
* The first request the feed refuses (a 429, a 5xx, a network error) ends
|
||||
* the run with status `error` and no follow-up; the next nightly discovery
|
||||
* tries again. Before every range the run re-reads the module, the
|
||||
* account's air-quality switch and `ENVIRONMENT_AIR_QUALITY_DISABLED`, so
|
||||
* switching any of them off stops a run between two ranges.
|
||||
*
|
||||
* ## Budget
|
||||
*
|
||||
@@ -48,7 +62,11 @@
|
||||
* share (`request-budget.ts`). On top of that the history holds itself to
|
||||
* {@link AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS} of the account's 400 daily
|
||||
* calls and to half of each instance window, so the nightly fetch, the gap
|
||||
* fill and a backfill someone asked for keep their room. A run that reaches a
|
||||
* fill and a backfill someone asked for keep their room. The ceiling is
|
||||
* checked in the budget's own locked transaction, together with the charge
|
||||
* ({@link AIR_QUALITY_HISTORY_CEILING}), so two worker processes, or the
|
||||
* history beside the nightly fetch, cannot both read room only one of them
|
||||
* gets. A run that reaches a
|
||||
* ceiling stops; the nightly discovery picks the account up again, so a long
|
||||
* history spreads over several days by construction.
|
||||
*
|
||||
@@ -56,14 +74,16 @@
|
||||
*
|
||||
* A day is done once its row carries `aqFetchedAt`; done days are never asked
|
||||
* for again, and the writes only touch rows still without it (or create the
|
||||
* row). Each run stores `{ total, done, checkedAt, complete }` on
|
||||
* row). The days with entries are read once per chain and handed from run
|
||||
* to run in the follow-up's payload ({@link EntryDaysSnapshot}), so a chain
|
||||
* of fifty runs scans the entry tables once, not fifty times. Each run
|
||||
* stores `{ total, done, checkedAt, complete }` on
|
||||
* `User.environmentAqHistoryJson`, which the settings card shows as
|
||||
* "Backfilling air quality: x of y days".
|
||||
*/
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
import {
|
||||
airQualityChunkWeight,
|
||||
earliestAirQualityDay,
|
||||
fetchDailyAirQuality,
|
||||
isAirQualityActive,
|
||||
@@ -73,16 +93,13 @@ import {
|
||||
type DailyAirQualityObservation,
|
||||
} from "@/lib/environment/open-meteo-air-quality";
|
||||
import {
|
||||
archiveRequestWeight,
|
||||
fetchDailyEnvironment,
|
||||
type DailyEnvironmentObservation,
|
||||
} from "@/lib/environment/open-meteo";
|
||||
import {
|
||||
OPEN_METEO_ACCOUNT_DAY_WINDOW,
|
||||
OPEN_METEO_BUDGET_WINDOWS,
|
||||
OpenMeteoBudgetExhaustedError,
|
||||
readOpenMeteoBudgetUsage,
|
||||
type BudgetUsage,
|
||||
type BudgetCeiling,
|
||||
} from "@/lib/environment/request-budget";
|
||||
import { readLocation, sealLocation } from "@/lib/environment/location-cipher";
|
||||
import {
|
||||
@@ -93,11 +110,11 @@ import {
|
||||
inAnyPeriod,
|
||||
readTravelOverrides,
|
||||
resolveHome,
|
||||
resolveLocationForDay,
|
||||
utcDayKey,
|
||||
type HomeLocation,
|
||||
type ResolvedLocation,
|
||||
type TravelOverride,
|
||||
} from "@/lib/environment/service";
|
||||
import { isModuleEnabled } from "@/lib/modules/gate";
|
||||
import { shiftDateKey } from "@/lib/tz/format";
|
||||
import { startOfLocalDayKey } from "@/lib/tz/local-day";
|
||||
|
||||
@@ -115,6 +132,11 @@ export const AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS = Math.floor(
|
||||
);
|
||||
/** The history's share of each instance-wide window. */
|
||||
export const AIR_QUALITY_HISTORY_INSTANCE_SHARE = 0.5;
|
||||
/** Both ceilings, as every history request hands them to the budget. */
|
||||
export const AIR_QUALITY_HISTORY_CEILING: BudgetCeiling = {
|
||||
instanceShare: AIR_QUALITY_HISTORY_INSTANCE_SHARE,
|
||||
accountDayCalls: AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS,
|
||||
};
|
||||
|
||||
/** One pending day and where it is fetched. */
|
||||
export interface HistoryDay {
|
||||
@@ -159,60 +181,6 @@ export function planHistoryRanges(
|
||||
return ranges;
|
||||
}
|
||||
|
||||
/**
|
||||
* The weight of one range in calls: its air-quality request, plus the
|
||||
* weather request over the span of its days without a row, when it has any.
|
||||
*/
|
||||
export function historyRangeWeight(
|
||||
range: HistoryRange,
|
||||
missingRowDays: readonly string[],
|
||||
): number {
|
||||
const air = airQualityChunkWeight(range.startDate, range.endDate);
|
||||
if (missingRowDays.length === 0) return air;
|
||||
const sorted = [...missingRowDays].sort();
|
||||
return air + archiveRequestWeight(sorted[0], sorted[sorted.length - 1]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a request of `weight` calls fits under the history's ceilings, given
|
||||
* what each window already holds. Pure; exported for tests.
|
||||
*/
|
||||
export function historyRequestFits(
|
||||
usage: BudgetUsage,
|
||||
weight: number,
|
||||
): boolean {
|
||||
for (const w of OPEN_METEO_BUDGET_WINDOWS) {
|
||||
const ceiling = w.limit * AIR_QUALITY_HISTORY_INSTANCE_SHARE;
|
||||
if ((usage[w.name] ?? 0) + weight > ceiling) return false;
|
||||
}
|
||||
return (
|
||||
(usage["account-day"] ?? 0) + weight <=
|
||||
AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a day without a row is placed: the period that covers it, else the
|
||||
* home from `homeSince` on, or on every day when no `homeSince` is stored
|
||||
* (see the header). Null for a day before the home with no period. Pure;
|
||||
* exported for tests.
|
||||
*/
|
||||
export function resolveHistoryLocation(
|
||||
day: string,
|
||||
home: HomeLocation | null,
|
||||
travels: readonly TravelOverride[],
|
||||
): ResolvedLocation | null {
|
||||
for (const t of travels) {
|
||||
if (day >= t.startDate && day <= t.endDate) {
|
||||
return { lat: t.lat, lon: t.lon, label: t.label, source: "TRAVEL" };
|
||||
}
|
||||
}
|
||||
if (home && (home.since === null || day >= home.since)) {
|
||||
return { lat: home.lat, lon: home.lon, label: home.label, source: "HOME" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The stored progress, as the settings card reads it. */
|
||||
export interface AirQualityHistoryState {
|
||||
version: 1;
|
||||
@@ -258,6 +226,13 @@ export function readAirQualityHistoryState(
|
||||
* mood entry, a taken dose, a symptom or a workout. Distinct in SQL, so a
|
||||
* dense record costs a scan, not a transfer. `tz` is a zone the account
|
||||
* stored (home or profile), bound as a parameter.
|
||||
*
|
||||
* Measured on Postgres 16 with 4 000 days of readings: 4 000 rows answer in
|
||||
* about 5 ms, 400 000 rows (100 a day) in about 75 ms, almost all of it the
|
||||
* measurement scan and its hash aggregate (`EXPLAIN ANALYZE`: a parallel
|
||||
* sequential scan feeding a HashAggregate, 6.4 MB per worker). The cost
|
||||
* grows with the readings, not with the days, so it is read once per chain
|
||||
* and handed on ({@link EntryDaysSnapshot}), not once per run.
|
||||
*/
|
||||
export async function readEntryDays(
|
||||
userId: string,
|
||||
@@ -301,21 +276,53 @@ export async function readEntryDays(
|
||||
.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* The days with entries as one run read them, handed to the next run of the
|
||||
* same chain so the entry tables are scanned once per chain. Valid only for
|
||||
* the zone and the cutoff it was read under; a run under another one (the
|
||||
* home moved, or the chain crossed midnight) reads afresh.
|
||||
*/
|
||||
export interface EntryDaysSnapshot {
|
||||
timezone: string;
|
||||
cutoff: string;
|
||||
days: string[];
|
||||
}
|
||||
|
||||
/** Tolerant read of a snapshot from a job payload; null for anything else. */
|
||||
export function readEntryDaysSnapshot(raw: unknown): EntryDaysSnapshot | null {
|
||||
if (raw === null || typeof raw !== "object") return null;
|
||||
const r = raw as Record<string, unknown>;
|
||||
if (
|
||||
typeof r.timezone !== "string" ||
|
||||
typeof r.cutoff !== "string" ||
|
||||
!Array.isArray(r.days) ||
|
||||
!r.days.every((d) => typeof d === "string")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return { timezone: r.timezone, cutoff: r.cutoff, days: r.days as string[] };
|
||||
}
|
||||
|
||||
/** What the history still has to do for an account, and how far it is. */
|
||||
export interface HistoryWork {
|
||||
pending: HistoryDay[];
|
||||
total: number;
|
||||
done: number;
|
||||
timezone: string;
|
||||
/** The entry days this collection used, for the chain's next run. */
|
||||
entryDays: EntryDaysSnapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect the pending days and the counts. Null when the account is gone or
|
||||
* its air quality is off (for the account or the instance).
|
||||
* its air quality is off (for the account or the instance). With `entryDays`
|
||||
* from the chain's previous run under the same zone and cutoff, the entry
|
||||
* tables are not read again.
|
||||
*/
|
||||
export async function collectAirQualityHistory(
|
||||
userId: string,
|
||||
now: Date = new Date(),
|
||||
entryDays: EntryDaysSnapshot | null = null,
|
||||
): Promise<HistoryWork | null> {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
@@ -329,11 +336,23 @@ export async function collectAirQualityHistory(
|
||||
const timezone = home?.timezone ?? user.timezone;
|
||||
const cutoff = shiftDateKey(utcDayKey(now), -AIR_QUALITY_HISTORY_SETTLE_DAYS);
|
||||
if (cutoff < AIR_QUALITY_EARLIEST_DAY) {
|
||||
return { pending: [], total: 0, done: 0, timezone };
|
||||
return {
|
||||
pending: [],
|
||||
total: 0,
|
||||
done: 0,
|
||||
timezone,
|
||||
entryDays: { timezone, cutoff, days: [] },
|
||||
};
|
||||
}
|
||||
|
||||
const [entryDays, rows] = await Promise.all([
|
||||
readEntryDays(userId, timezone, AIR_QUALITY_EARLIEST_DAY, cutoff),
|
||||
const reuse =
|
||||
entryDays !== null &&
|
||||
entryDays.timezone === timezone &&
|
||||
entryDays.cutoff === cutoff;
|
||||
const [days, rows] = await Promise.all([
|
||||
reuse
|
||||
? entryDays.days
|
||||
: readEntryDays(userId, timezone, AIR_QUALITY_EARLIEST_DAY, cutoff),
|
||||
prisma.environmentContext.findMany({
|
||||
where: { userId, date: { gte: AIR_QUALITY_EARLIEST_DAY, lte: cutoff } },
|
||||
select: {
|
||||
@@ -352,7 +371,7 @@ export async function collectAirQualityHistory(
|
||||
const pending: HistoryDay[] = [];
|
||||
let total = 0;
|
||||
let done = 0;
|
||||
for (const date of entryDays) {
|
||||
for (const date of days) {
|
||||
const row = rowByDate.get(date);
|
||||
let location: ResolvedLocation | null;
|
||||
if (row) {
|
||||
@@ -366,7 +385,7 @@ export async function collectAirQualityHistory(
|
||||
} else if (inAnyPeriod(date, travel.unreadable)) {
|
||||
location = null;
|
||||
} else {
|
||||
location = resolveHistoryLocation(date, home, travel.overrides);
|
||||
location = resolveLocationForDay(date, home, travel.overrides);
|
||||
}
|
||||
if (!location) continue;
|
||||
// Outside what the source serves at this place: left empty, not counted.
|
||||
@@ -378,31 +397,38 @@ export async function collectAirQualityHistory(
|
||||
pending.push({ date, location, hasRow: row != null });
|
||||
}
|
||||
}
|
||||
return { pending, total, done, timezone };
|
||||
}
|
||||
|
||||
/** A weather observation with nothing in it, for a day the archive skipped. */
|
||||
function emptyWeather(date: string): DailyEnvironmentObservation {
|
||||
return {
|
||||
date,
|
||||
tempMin: null,
|
||||
tempMax: null,
|
||||
tempMean: null,
|
||||
apparentMean: null,
|
||||
apparentMax: null,
|
||||
sunshineSec: null,
|
||||
daylightSec: null,
|
||||
precipSum: null,
|
||||
pressureMean: null,
|
||||
pressureDelta: null,
|
||||
humidityMean: null,
|
||||
cloudMean: null,
|
||||
weatherCode: null,
|
||||
pending,
|
||||
total,
|
||||
done,
|
||||
timezone,
|
||||
entryDays: { timezone, cutoff, days },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the history may still send for the account: the account exists,
|
||||
* its environment module is on, and its air quality is on for the account
|
||||
* and the instance. Read before every range, so switching any of them off
|
||||
* stops a run between two ranges, not only at the next run.
|
||||
*/
|
||||
async function historyStillActive(userId: string): Promise<boolean> {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { environmentAirQualityEnabled: true },
|
||||
});
|
||||
if (!user || !isAirQualityActive(user.environmentAirQualityEnabled)) {
|
||||
return false;
|
||||
}
|
||||
return isModuleEnabled(userId, "environment");
|
||||
}
|
||||
|
||||
export interface AirQualityHistoryRunResult {
|
||||
/** `inactive`: the account is gone or its air quality is off. */
|
||||
/**
|
||||
* `inactive`: the account is gone, or its module or air quality is off
|
||||
* (at the start or between two ranges). `error`: the feed refused a
|
||||
* request, which ends the run and its chain.
|
||||
*/
|
||||
status: "inactive" | "complete" | "progress" | "budget" | "error";
|
||||
/** Days whose air quality this run stored. */
|
||||
filled: number;
|
||||
@@ -412,20 +438,30 @@ export interface AirQualityHistoryRunResult {
|
||||
done: number;
|
||||
/** Days still pending after this run. */
|
||||
remaining: number;
|
||||
/** The entry days this run used, for its follow-up; null when inactive. */
|
||||
entryDays: EntryDaysSnapshot | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* One run of the history for an account: up to `maxRanges` ranges, newest
|
||||
* first, inside the history's budget ceilings. Stores the progress and says
|
||||
* why it stopped.
|
||||
* why it stopped. `entryDays` is the previous run's snapshot in a chain.
|
||||
*/
|
||||
export async function runAirQualityHistory(
|
||||
userId: string,
|
||||
opts: { now?: Date; maxRanges?: number } = {},
|
||||
opts: {
|
||||
now?: Date;
|
||||
maxRanges?: number;
|
||||
entryDays?: EntryDaysSnapshot | null;
|
||||
} = {},
|
||||
): Promise<AirQualityHistoryRunResult> {
|
||||
const now = opts.now ?? new Date();
|
||||
const maxRanges = opts.maxRanges ?? AIR_QUALITY_HISTORY_RANGES_PER_RUN;
|
||||
const work = await collectAirQualityHistory(userId, now);
|
||||
const work = await collectAirQualityHistory(
|
||||
userId,
|
||||
now,
|
||||
opts.entryDays ?? null,
|
||||
);
|
||||
if (!work) {
|
||||
return {
|
||||
status: "inactive",
|
||||
@@ -434,6 +470,7 @@ export async function runAirQualityHistory(
|
||||
total: 0,
|
||||
done: 0,
|
||||
remaining: 0,
|
||||
entryDays: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -461,46 +498,23 @@ export async function runAirQualityHistory(
|
||||
)
|
||||
.sort((a, b) => b.range.endDate.localeCompare(a.range.endDate));
|
||||
|
||||
const budget = { accountId: userId, ceiling: AIR_QUALITY_HISTORY_CEILING };
|
||||
let filled = 0;
|
||||
let fetches = 0;
|
||||
let stop: "budget" | "error" | null = null;
|
||||
let errors = 0;
|
||||
let stop: "budget" | "error" | "inactive" | null = null;
|
||||
let attempted = 0;
|
||||
for (const { group, range } of planned) {
|
||||
if (attempted >= maxRanges) break;
|
||||
if (!(await historyStillActive(userId))) {
|
||||
stop = "inactive";
|
||||
break;
|
||||
}
|
||||
attempted += 1;
|
||||
const days = range.days.map((d) => group.byDate.get(d)!);
|
||||
const missing = days.filter((d) => !d.hasRow).map((d) => d.date);
|
||||
const usage = await readOpenMeteoBudgetUsage(userId, new Date());
|
||||
if (!historyRequestFits(usage, historyRangeWeight(range, missing))) {
|
||||
stop = "budget";
|
||||
break;
|
||||
}
|
||||
|
||||
const air = await fetchDailyAirQuality(
|
||||
{
|
||||
lat: group.lat,
|
||||
lon: group.lon,
|
||||
timezone: work.timezone,
|
||||
startDate: range.startDate,
|
||||
endDate: range.endDate,
|
||||
},
|
||||
{ accountId: userId },
|
||||
);
|
||||
if (air.stopped === "budget") {
|
||||
stop = "budget";
|
||||
break;
|
||||
}
|
||||
if (air.stopped === "error") {
|
||||
errors += 1;
|
||||
continue;
|
||||
}
|
||||
fetches += 1;
|
||||
const airByDay = new Map<string, DailyAirQualityObservation>(
|
||||
air.days.map((d) => [d.date, d]),
|
||||
);
|
||||
const airFor = (date: string) => airByDay.get(date) ?? uncoverableDay(date);
|
||||
|
||||
// The weather first, so a refused or failed archive request costs no
|
||||
// air-quality call.
|
||||
let weather: Map<string, DailyEnvironmentObservation> | null = null;
|
||||
if (missing.length > 0) {
|
||||
try {
|
||||
@@ -512,55 +526,88 @@ export async function runAirQualityHistory(
|
||||
startDate: missing[0],
|
||||
endDate: missing[missing.length - 1],
|
||||
},
|
||||
{ accountId: userId },
|
||||
budget,
|
||||
);
|
||||
fetches += 1;
|
||||
weather = new Map(observations.map((o) => [o.date, o]));
|
||||
} catch (err) {
|
||||
if (err instanceof OpenMeteoBudgetExhaustedError) {
|
||||
stop = "budget";
|
||||
} else {
|
||||
errors += 1;
|
||||
}
|
||||
// The days with a row are stored below; those without one stay
|
||||
// pending until the weather can be fetched with them.
|
||||
stop =
|
||||
err instanceof OpenMeteoBudgetExhaustedError ? "budget" : "error";
|
||||
if (stop === "error") fetches += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const air = await fetchDailyAirQuality(
|
||||
{
|
||||
lat: group.lat,
|
||||
lon: group.lon,
|
||||
timezone: work.timezone,
|
||||
startDate: range.startDate,
|
||||
endDate: range.endDate,
|
||||
},
|
||||
budget,
|
||||
);
|
||||
if (air.stopped !== "budget") fetches += 1;
|
||||
const airByDay = new Map<string, DailyAirQualityObservation>(
|
||||
air.days.map((d) => [d.date, d]),
|
||||
);
|
||||
// A day the feed did not answer after a full fetch is one it cannot
|
||||
// cover; after a stopped fetch it is simply not fetched yet.
|
||||
const airFor = (date: string): DailyAirQualityObservation | null =>
|
||||
airByDay.get(date) ??
|
||||
(air.stopped === null ? uncoverableDay(date) : null);
|
||||
|
||||
const fetchedAt = new Date();
|
||||
const ops: Prisma.PrismaPromise<unknown>[] = [];
|
||||
const fillsAir: boolean[] = [];
|
||||
for (const day of days) {
|
||||
const dayAir = airFor(day.date);
|
||||
if (day.hasRow) {
|
||||
if (!dayAir) continue;
|
||||
ops.push(
|
||||
prisma.environmentContext.updateMany({
|
||||
// Still unfetched, so a weather rewrite that landed meanwhile
|
||||
// (with its own air quality) is not overwritten.
|
||||
where: { userId, date: day.date, aqFetchedAt: null },
|
||||
data: airQualityColumns(airFor(day.date), fetchedAt),
|
||||
data: airQualityColumns(dayAir, fetchedAt),
|
||||
}),
|
||||
);
|
||||
} else if (weather) {
|
||||
ops.push(
|
||||
buildUpsert(
|
||||
userId,
|
||||
day.location,
|
||||
sealLocation(day.location),
|
||||
weather.get(day.date) ?? emptyWeather(day.date),
|
||||
{ active: true, day: airFor(day.date), now: fetchedAt },
|
||||
),
|
||||
);
|
||||
fillsAir.push(true);
|
||||
continue;
|
||||
}
|
||||
// A day the archive answer left out gets no row: it stays pending
|
||||
// instead of carrying empty weather nothing would fetch again.
|
||||
const observation = weather?.get(day.date);
|
||||
if (!observation) continue;
|
||||
// Without its air quality (the air request stopped), the weather row
|
||||
// is still stored: it was paid for, and the next run then asks for
|
||||
// the air quality alone.
|
||||
ops.push(
|
||||
buildUpsert(
|
||||
userId,
|
||||
day.location,
|
||||
sealLocation(day.location),
|
||||
observation,
|
||||
{ active: true, day: dayAir, now: fetchedAt },
|
||||
),
|
||||
);
|
||||
fillsAir.push(dayAir !== null);
|
||||
}
|
||||
if (ops.length > 0) {
|
||||
const results = await prisma.$transaction(ops);
|
||||
for (const r of results) {
|
||||
results.forEach((r, i) => {
|
||||
if (!fillsAir[i]) return;
|
||||
filled +=
|
||||
r && typeof r === "object" && "count" in r
|
||||
? Number((r as { count: number }).count)
|
||||
: 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
if (air.stopped) {
|
||||
stop = air.stopped;
|
||||
break;
|
||||
}
|
||||
if (stop) break;
|
||||
}
|
||||
|
||||
const done = Math.min(work.total, work.done + filled);
|
||||
@@ -578,12 +625,14 @@ export async function runAirQualityHistory(
|
||||
});
|
||||
|
||||
const status: AirQualityHistoryRunResult["status"] =
|
||||
remaining === 0
|
||||
? "complete"
|
||||
: stop === "budget"
|
||||
? "budget"
|
||||
: errors > 0 && filled === 0
|
||||
? "error"
|
||||
: "progress";
|
||||
return { status, filled, fetches, total: work.total, done, remaining };
|
||||
remaining === 0 ? "complete" : (stop ?? "progress");
|
||||
return {
|
||||
status,
|
||||
filled,
|
||||
fetches,
|
||||
total: work.total,
|
||||
done,
|
||||
remaining,
|
||||
entryDays: work.entryDays,
|
||||
};
|
||||
}
|
||||
@@ -72,6 +72,7 @@ import { enumerateDayCount } from "@/lib/environment/day-span";
|
||||
import {
|
||||
openMeteoCallWeight,
|
||||
reserveOpenMeteoCalls,
|
||||
type BudgetCeiling,
|
||||
} from "@/lib/environment/request-budget";
|
||||
|
||||
/** Hosted default; override with `OPENMETEO_AIR_QUALITY_URL` (self-host). */
|
||||
@@ -395,7 +396,7 @@ export async function fetchDailyAirQuality(
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
},
|
||||
budget: { accountId?: string } = {},
|
||||
budget: { accountId?: string; ceiling?: BudgetCeiling } = {},
|
||||
): Promise<AirQualityFetchResult> {
|
||||
const days: DailyAirQualityObservation[] = [];
|
||||
// Days before the feed's first day at this location are answered here,
|
||||
@@ -415,7 +416,9 @@ export async function fetchDailyAirQuality(
|
||||
for (const chunk of chunks) {
|
||||
const variables = variablesForChunk(chunk.endDate);
|
||||
const weight = airQualityChunkWeight(chunk.startDate, chunk.endDate);
|
||||
if (!(await reserveOpenMeteoCalls(weight, budget.accountId))) {
|
||||
if (
|
||||
!(await reserveOpenMeteoCalls(weight, budget.accountId, budget.ceiling))
|
||||
) {
|
||||
return { days, stopped: "budget" };
|
||||
}
|
||||
try {
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
OpenMeteoBudgetExhaustedError,
|
||||
openMeteoCallWeight,
|
||||
reserveOpenMeteoCalls,
|
||||
type BudgetCeiling,
|
||||
} from "@/lib/environment/request-budget";
|
||||
|
||||
/** Hosted archive default; override with `OPENMETEO_BASE_URL` (self-host). */
|
||||
@@ -278,10 +279,12 @@ export async function fetchDailyEnvironment(
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
},
|
||||
budget: { accountId?: string } = {},
|
||||
budget: { accountId?: string; ceiling?: BudgetCeiling } = {},
|
||||
): Promise<DailyEnvironmentObservation[]> {
|
||||
const weight = archiveRequestWeight(args.startDate, args.endDate);
|
||||
if (!(await reserveOpenMeteoCalls(weight, budget.accountId))) {
|
||||
if (
|
||||
!(await reserveOpenMeteoCalls(weight, budget.accountId, budget.ceiling))
|
||||
) {
|
||||
throw new OpenMeteoBudgetExhaustedError();
|
||||
}
|
||||
const url = new URL(`${ARCHIVE_BASE_URL}/v1/archive`);
|
||||
|
||||
@@ -114,16 +114,54 @@ export function admitWeight(
|
||||
return { admitted: true, next };
|
||||
}
|
||||
|
||||
/**
|
||||
* A lower ceiling a caller holds itself to inside the budget: a share of each
|
||||
* instance window, and a number of calls inside the account's daily share.
|
||||
* The air-quality history backfill uses one so the nightly fetch and
|
||||
* on-demand requests keep their room. The ceiling is checked against
|
||||
* everything the windows hold, not only against the caller's own calls, and
|
||||
* the request is charged to the windows as usual.
|
||||
*/
|
||||
export interface BudgetCeiling {
|
||||
/** Share of each instance window the request may fill up to (0..1). */
|
||||
instanceShare: number;
|
||||
/** Calls the account's daily window may hold after the request. */
|
||||
accountDayCalls: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The windows with their limits lowered to `ceiling`. Only the limits
|
||||
* change, so the buckets written are the same as without a ceiling. Pure;
|
||||
* exported for tests.
|
||||
*/
|
||||
export function windowsUnderCeiling(
|
||||
windows: readonly BudgetWindow[],
|
||||
ceiling: BudgetCeiling | undefined,
|
||||
): readonly BudgetWindow[] {
|
||||
if (!ceiling) return windows;
|
||||
return windows.map((w) => ({
|
||||
...w,
|
||||
limit:
|
||||
w.name === "account-day"
|
||||
? Math.min(w.limit, ceiling.accountDayCalls)
|
||||
: w.limit * ceiling.instanceShare,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the budget for one request of `weight` calls. Returns true and charges
|
||||
* every window when it fits, false (charging nothing) when any window is
|
||||
* full. With `accountId`, the account's own daily share is checked and
|
||||
* charged as well (`OPEN_METEO_ACCOUNT_DAY_SHARE`); requests made for no
|
||||
* particular account (the geocoder) are bound by the instance windows only.
|
||||
* With `ceiling`, the request is admitted only under that lower ceiling, and
|
||||
* the check runs under the same lock as the charge, so two processes cannot
|
||||
* both read room that only one of them gets.
|
||||
*/
|
||||
export async function reserveOpenMeteoCalls(
|
||||
weight: number,
|
||||
accountId?: string,
|
||||
ceiling?: BudgetCeiling,
|
||||
): Promise<boolean> {
|
||||
const weightCenti = Math.max(1, Math.ceil(weight * CENTI));
|
||||
const windows: readonly BudgetWindow[] = accountId
|
||||
@@ -150,7 +188,12 @@ export async function reserveOpenMeteoCalls(
|
||||
const name = nameOfKey.get(row.key);
|
||||
if (name) buckets.set(name, { count: row.count, resetAt: row.resetAt });
|
||||
}
|
||||
const decision = admitWeight(buckets, weightCenti, new Date(), windows);
|
||||
const decision = admitWeight(
|
||||
buckets,
|
||||
weightCenti,
|
||||
new Date(),
|
||||
windowsUnderCeiling(windows, ceiling),
|
||||
);
|
||||
if (!decision.admitted) return false;
|
||||
for (const [name, bucket] of decision.next) {
|
||||
const key = keyOf(name);
|
||||
@@ -171,39 +214,3 @@ export class OpenMeteoBudgetExhaustedError extends Error {
|
||||
this.name = "OpenMeteoBudgetExhaustedError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Calls already charged to each live window, by window name. */
|
||||
export type BudgetUsage = Partial<Record<BudgetWindow["name"], number>>;
|
||||
|
||||
/**
|
||||
* Read how much of each window is in use, in calls, without charging
|
||||
* anything. A window never written or already reset reads as absent (zero).
|
||||
* With `accountId` the account's daily share is read as well. Used by the
|
||||
* air-quality history backfill, which holds itself under a ceiling of its own
|
||||
* so the nightly fetch and on-demand requests keep their room.
|
||||
*/
|
||||
export async function readOpenMeteoBudgetUsage(
|
||||
accountId?: string,
|
||||
now: Date = new Date(),
|
||||
): Promise<BudgetUsage> {
|
||||
const keys = new Map<string, BudgetWindow["name"]>(
|
||||
OPEN_METEO_BUDGET_WINDOWS.map((w) => [
|
||||
`${BUCKET_PREFIX}:${w.name}`,
|
||||
w.name,
|
||||
]),
|
||||
);
|
||||
if (accountId) {
|
||||
keys.set(`${BUCKET_PREFIX}:account:${accountId}:day`, "account-day");
|
||||
}
|
||||
const rows = await prisma.rateLimit.findMany({
|
||||
where: { key: { in: [...keys.keys()] } },
|
||||
select: { key: true, count: true, resetAt: true },
|
||||
});
|
||||
const usage: BudgetUsage = {};
|
||||
for (const row of rows) {
|
||||
const name = keys.get(row.key);
|
||||
if (!name || row.resetAt.getTime() <= now.getTime()) continue;
|
||||
usage[name] = row.count / CENTI;
|
||||
}
|
||||
return usage;
|
||||
}
|
||||
@@ -16,8 +16,16 @@
|
||||
* ceiling stopped it, it sends its own follow-up half a minute later. A run
|
||||
* the budget stopped sends none: the next nightly discovery resumes it, which
|
||||
* is what spreads a long history over several days. The module gate and the
|
||||
* air-quality switches are re-read on every run, so switching either off
|
||||
* ends the chain.
|
||||
* air-quality switches are re-read before every range, so switching either
|
||||
* off ends the run and the chain.
|
||||
*
|
||||
* A run the feed refused a request in (429, 5xx, network) ends with status
|
||||
* `error`: it reports a failed job and sends no follow-up, and the sends
|
||||
* carry `retryLimit: 0`, so pg-boss does not retry it straight into the
|
||||
* same refusal either. The next nightly discovery tries again.
|
||||
*
|
||||
* The follow-up carries the days with entries the run read
|
||||
* (`entryDays`), so a chain scans the entry tables once.
|
||||
*
|
||||
* The queue name MUST be registered in `allQueues` in
|
||||
* `src/lib/jobs/reminder/register-maintenance.ts`, or the sends silently
|
||||
@@ -37,7 +45,9 @@ import {
|
||||
import { isAirQualityOperatorDisabled } from "@/lib/environment/open-meteo-air-quality";
|
||||
import {
|
||||
readAirQualityHistoryState,
|
||||
readEntryDaysSnapshot,
|
||||
runAirQualityHistory,
|
||||
type EntryDaysSnapshot,
|
||||
} from "@/lib/environment/air-quality-history";
|
||||
import { workerLog } from "./reminder/shared";
|
||||
|
||||
@@ -59,8 +69,16 @@ export interface EnvironmentAqHistoryPayload {
|
||||
/** Absent on the discovery tick. */
|
||||
userId?: string;
|
||||
continuation?: number;
|
||||
/** The previous run's entry days, on a follow-up only. */
|
||||
entryDays?: EntryDaysSnapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* No pg-boss retry: a run fails when the feed refused it, and an immediate
|
||||
* retry would only ask the same feed again. The next discovery is the retry.
|
||||
*/
|
||||
const AQ_HISTORY_SEND_OPTIONS = { retryLimit: 0 } as const;
|
||||
|
||||
function singletonKey(userId: string): string {
|
||||
return `environment-aq-history:${userId}`;
|
||||
}
|
||||
@@ -80,6 +98,7 @@ export async function enqueueAirQualityHistory(
|
||||
ENVIRONMENT_AQ_HISTORY_QUEUE,
|
||||
{ userId } satisfies EnvironmentAqHistoryPayload,
|
||||
{
|
||||
...AQ_HISTORY_SEND_OPTIONS,
|
||||
singletonKey: singletonKey(userId),
|
||||
...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}),
|
||||
},
|
||||
@@ -149,6 +168,7 @@ export async function discoverAirQualityHistory(
|
||||
ENVIRONMENT_AQ_HISTORY_QUEUE,
|
||||
{ userId: candidate.id } satisfies EnvironmentAqHistoryPayload,
|
||||
{
|
||||
...AQ_HISTORY_SEND_OPTIONS,
|
||||
singletonKey: singletonKey(candidate.id),
|
||||
...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}),
|
||||
},
|
||||
@@ -203,7 +223,9 @@ export async function handleEnvironmentAqHistory(
|
||||
return jobDone({ skipped: "environment_module_disabled" });
|
||||
}
|
||||
|
||||
const result = await runAirQualityHistory(userId);
|
||||
const result = await runAirQualityHistory(userId, {
|
||||
entryDays: readEntryDaysSnapshot(payload.entryDays),
|
||||
});
|
||||
const continuation = payload.continuation ?? 0;
|
||||
let continued = false;
|
||||
if (
|
||||
@@ -216,8 +238,10 @@ export async function handleEnvironmentAqHistory(
|
||||
{
|
||||
userId,
|
||||
continuation: continuation + 1,
|
||||
...(result.entryDays ? { entryDays: result.entryDays } : {}),
|
||||
} satisfies EnvironmentAqHistoryPayload,
|
||||
{
|
||||
...AQ_HISTORY_SEND_OPTIONS,
|
||||
singletonKey: singletonKey(userId),
|
||||
startAfter: AQ_HISTORY_CONTINUATION_DELAY_SECONDS,
|
||||
},
|
||||
@@ -228,13 +252,18 @@ export async function handleEnvironmentAqHistory(
|
||||
"info",
|
||||
`[environment-aq-history] user=${userId} status=${result.status} filled=${result.filled} done=${result.done}/${result.total} fetches=${result.fetches}${continued ? " continued" : ""}`,
|
||||
);
|
||||
// Every range failed at the feed: say so, so the failing-jobs card sees a
|
||||
// The feed refused a request: say so, so the failing-jobs card sees a
|
||||
// feed that keeps refusing. A budget stop is not a failure: nothing was
|
||||
// sent past the ceiling, and the next discovery resumes where it stopped.
|
||||
if (result.status === "error") {
|
||||
return jobFailed("air-quality history: every range failed", undefined, {
|
||||
fetches: result.fetches,
|
||||
});
|
||||
return jobFailed(
|
||||
"air-quality history: the feed refused a request",
|
||||
undefined,
|
||||
{
|
||||
fetches: result.fetches,
|
||||
days_stored: result.filled,
|
||||
},
|
||||
);
|
||||
}
|
||||
return jobDone({
|
||||
outcome: result.status,
|
||||
|
||||
@@ -21,8 +21,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS,
|
||||
AIR_QUALITY_HISTORY_CEILING,
|
||||
runAirQualityHistory,
|
||||
} from "@/lib/environment/air-quality-history";
|
||||
import { reserveOpenMeteoCalls } from "@/lib/environment/request-budget";
|
||||
import { openLocation, sealLocation } from "@/lib/environment/location-cipher";
|
||||
import { handleEnvironmentAqHistory } from "@/lib/jobs/environment-air-quality-history";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
@@ -49,8 +51,19 @@ function daysBetween(start: string, end: string): string[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
interface StubOptions {
|
||||
/** HTTP status the archive answers with instead of data. */
|
||||
archiveStatus?: number;
|
||||
/** HTTP status the air-quality API answers with instead of data. */
|
||||
airStatus?: number;
|
||||
/** Days the archive leaves out of its answer. */
|
||||
omitArchiveDays?: readonly string[];
|
||||
/** Runs before every air-quality answer. */
|
||||
onAir?: () => Promise<void>;
|
||||
}
|
||||
|
||||
/** Answers both feeds with a full day of values for every requested day. */
|
||||
function openMeteoStub() {
|
||||
function openMeteoStub(opts: StubOptions = {}) {
|
||||
return vi.fn(async (input: unknown) => {
|
||||
const url = new URL(String(input));
|
||||
const days = daysBetween(
|
||||
@@ -58,6 +71,8 @@ function openMeteoStub() {
|
||||
url.searchParams.get("end_date")!,
|
||||
);
|
||||
if (url.pathname === "/v1/air-quality") {
|
||||
await opts.onAir?.();
|
||||
if (opts.airStatus) return new Response("", { status: opts.airStatus });
|
||||
const variables = url.searchParams.get("hourly")!.split(",");
|
||||
const time = days.flatMap((d) =>
|
||||
Array.from(
|
||||
@@ -69,11 +84,15 @@ function openMeteoStub() {
|
||||
for (const v of variables) hourly[v] = time.map(() => 12);
|
||||
return Response.json({ hourly });
|
||||
}
|
||||
const daily: Record<string, unknown> = { time: days };
|
||||
for (const field of url.searchParams.get("daily")!.split(",")) {
|
||||
daily[field] = days.map(() => 5);
|
||||
if (opts.archiveStatus) {
|
||||
return new Response("", { status: opts.archiveStatus });
|
||||
}
|
||||
const time = days.flatMap((d) =>
|
||||
const answered = days.filter((d) => !opts.omitArchiveDays?.includes(d));
|
||||
const daily: Record<string, unknown> = { time: answered };
|
||||
for (const field of url.searchParams.get("daily")!.split(",")) {
|
||||
daily[field] = answered.map(() => 5);
|
||||
}
|
||||
const time = answered.flatMap((d) =>
|
||||
Array.from(
|
||||
{ length: 24 },
|
||||
(_, h) => `${d}T${String(h).padStart(2, "0")}:00`,
|
||||
@@ -194,6 +213,22 @@ function requests() {
|
||||
return fetchSpy.mock.calls.map(([input]) => new URL(String(input)));
|
||||
}
|
||||
|
||||
function useStub(opts: StubOptions) {
|
||||
fetchSpy = openMeteoStub(opts);
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
}
|
||||
|
||||
/** `[path, start_date]` of every request sent, in order. */
|
||||
function sentRanges() {
|
||||
return requests().map((u) => [u.pathname, u.searchParams.get("start_date")]);
|
||||
}
|
||||
|
||||
async function rowOn(day: string) {
|
||||
return getPrismaClient().environmentContext.findUnique({
|
||||
where: { userId_date: { userId: OWNER, date: day } },
|
||||
});
|
||||
}
|
||||
|
||||
describe("air-quality history backfill", () => {
|
||||
it("fills every past day with entries the source reaches, once", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
@@ -366,3 +401,189 @@ describe("air-quality history backfill", () => {
|
||||
expect(sends).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("air-quality history: placing, failing and stopping", () => {
|
||||
it("places no day at a home stored without an effective date, as the nightly fetch does", async () => {
|
||||
await getPrismaClient().user.update({
|
||||
where: { id: OWNER },
|
||||
data: { homeSince: null },
|
||||
});
|
||||
const result = await runAirQualityHistory(OWNER, { now: NOW });
|
||||
// The two 2020 trip days, the 2024 trip day, the day with a row and the
|
||||
// day already done; the home day without a row is not placed anywhere.
|
||||
expect(result).toMatchObject({ status: "complete", total: 5, done: 5 });
|
||||
expect(await rowOn("2025-09-10")).toBeNull();
|
||||
expect(sentRanges().map(([, start]) => start)).not.toContain("2025-09-10");
|
||||
});
|
||||
|
||||
it("stops at the first refused archive request and sends no air-quality request for it", async () => {
|
||||
useStub({ archiveStatus: 503 });
|
||||
const result = await runAirQualityHistory(OWNER, { now: NOW });
|
||||
expect(result).toMatchObject({ status: "error", filled: 0, fetches: 1 });
|
||||
expect(sentRanges()).toEqual([["/v1/archive", "2025-09-10"]]);
|
||||
});
|
||||
|
||||
it("stops at the first refused air-quality request and keeps the weather it already paid for", async () => {
|
||||
useStub({ airStatus: 429 });
|
||||
const result = await runAirQualityHistory(OWNER, { now: NOW });
|
||||
expect(result).toMatchObject({ status: "error", filled: 0, fetches: 2 });
|
||||
expect(sentRanges()).toEqual([
|
||||
["/v1/archive", "2025-09-10"],
|
||||
["/v1/air-quality", "2025-09-10"],
|
||||
]);
|
||||
// The weather row is stored, without air quality.
|
||||
expect(await rowOn("2025-09-10")).toMatchObject({
|
||||
tempMax: 5,
|
||||
pm25Mean: null,
|
||||
aqFetchedAt: null,
|
||||
});
|
||||
|
||||
// The next run asks for that day's air quality only.
|
||||
useStub({});
|
||||
const next = await runAirQualityHistory(OWNER, { now: NOW });
|
||||
expect(next.status).toBe("complete");
|
||||
expect(sentRanges().filter(([, start]) => start === "2025-09-10")).toEqual([
|
||||
["/v1/air-quality", "2025-09-10"],
|
||||
]);
|
||||
expect(await rowOn("2025-09-10")).toMatchObject({
|
||||
tempMax: 5,
|
||||
pm25Mean: 12,
|
||||
});
|
||||
});
|
||||
|
||||
it("sends no follow-up and no retry after a refused request", async () => {
|
||||
useStub({ archiveStatus: 503 });
|
||||
const sends: Array<{ data: unknown; options: unknown }> = [];
|
||||
const boss = {
|
||||
send: vi.fn(async (_name: string, data: unknown, options: unknown) => {
|
||||
sends.push({ data, options });
|
||||
return "job-id";
|
||||
}),
|
||||
};
|
||||
const outcome = await handleEnvironmentAqHistory(boss as never, {
|
||||
userId: OWNER,
|
||||
});
|
||||
expect(outcome.ok).toBe(false);
|
||||
expect(sends).toEqual([]);
|
||||
});
|
||||
|
||||
it("creates no row for a day the archive answer leaves out", async () => {
|
||||
useStub({ omitArchiveDays: ["2020-03-05"] });
|
||||
const result = await runAirQualityHistory(OWNER, { now: NOW });
|
||||
expect(result).toMatchObject({ status: "progress", remaining: 1 });
|
||||
expect(await rowOn("2020-03-05")).toBeNull();
|
||||
expect(await rowOn("2020-03-01")).toMatchObject({ pm25Mean: 12 });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"the account's air quality",
|
||||
() =>
|
||||
getPrismaClient()
|
||||
.user.update({
|
||||
where: { id: OWNER },
|
||||
data: { environmentAirQualityEnabled: false },
|
||||
})
|
||||
.then(() => undefined),
|
||||
],
|
||||
[
|
||||
"the environment module",
|
||||
() =>
|
||||
getPrismaClient()
|
||||
.user.update({
|
||||
where: { id: OWNER },
|
||||
data: { modulePreferencesJson: { environment: false } },
|
||||
})
|
||||
.then(() => undefined),
|
||||
],
|
||||
[
|
||||
"the operator switch",
|
||||
async () => {
|
||||
vi.stubEnv("ENVIRONMENT_AIR_QUALITY_DISABLED", "true");
|
||||
},
|
||||
],
|
||||
])("stops between two ranges once %s is switched off", async (_, off) => {
|
||||
let first = true;
|
||||
useStub({
|
||||
onAir: async () => {
|
||||
if (first) await off();
|
||||
first = false;
|
||||
},
|
||||
});
|
||||
const result = await runAirQualityHistory(OWNER, { now: NOW });
|
||||
expect(result.status).toBe("inactive");
|
||||
expect(
|
||||
requests().filter((u) => u.pathname === "/v1/air-quality"),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("holds the history's ceiling under concurrent reservations", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const key = `open-meteo-budget:account:${OWNER}:day`;
|
||||
await prisma.rateLimit.create({
|
||||
data: {
|
||||
key,
|
||||
count: (AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS - 1) * 100,
|
||||
resetAt: new Date(Date.now() + 3_600_000),
|
||||
},
|
||||
});
|
||||
const admitted = await Promise.all(
|
||||
Array.from({ length: 6 }, () =>
|
||||
reserveOpenMeteoCalls(0.5, OWNER, AIR_QUALITY_HISTORY_CEILING),
|
||||
),
|
||||
);
|
||||
expect(admitted.filter(Boolean)).toHaveLength(2);
|
||||
const bucket = await prisma.rateLimit.findUniqueOrThrow({ where: { key } });
|
||||
expect(bucket.count).toBe(AIR_QUALITY_HISTORY_ACCOUNT_DAY_CALLS * 100);
|
||||
});
|
||||
|
||||
it("reads the days with entries once per chain and hands them to the follow-up", async () => {
|
||||
// Nine more home days, each more than two weeks apart: more ranges than
|
||||
// one run works through, so the run sends a follow-up.
|
||||
for (let i = 0; i < 9; i++) {
|
||||
await measurementOn(
|
||||
new Date(Date.parse("2025-01-10T00:00:00Z") + i * 20 * 86_400_000)
|
||||
.toISOString()
|
||||
.slice(0, 10),
|
||||
);
|
||||
}
|
||||
const sends: Array<{ data: Record<string, unknown>; options: unknown }> =
|
||||
[];
|
||||
const boss = {
|
||||
send: vi.fn(
|
||||
async (
|
||||
_name: string,
|
||||
data: Record<string, unknown>,
|
||||
options: unknown,
|
||||
) => {
|
||||
sends.push({ data, options });
|
||||
return "job-id";
|
||||
},
|
||||
),
|
||||
};
|
||||
const outcome = await handleEnvironmentAqHistory(boss as never, {
|
||||
userId: OWNER,
|
||||
});
|
||||
expect(outcome.ok).toBe(true);
|
||||
expect(sends).toHaveLength(1);
|
||||
const followUp = sends[0]!.data as {
|
||||
userId: string;
|
||||
continuation: number;
|
||||
entryDays: { timezone: string; cutoff: string; days: string[] };
|
||||
};
|
||||
expect(followUp.entryDays.timezone).toBe("Europe/Berlin");
|
||||
// A failed run is not retried straight into the same refusal.
|
||||
expect(sends[0]!.options).toMatchObject({ retryLimit: 0 });
|
||||
expect(followUp.entryDays.days).toContain("2025-01-10");
|
||||
|
||||
// An entry added after the chain read its days is left to the next
|
||||
// chain: the follow-up does not scan the entry tables again.
|
||||
await measurementOn("2025-11-20");
|
||||
fetchSpy.mockClear();
|
||||
await handleEnvironmentAqHistory(boss as never, followUp);
|
||||
expect(await rowOn("2025-11-20")).toBeNull();
|
||||
// A fresh chain reads it.
|
||||
const fresh = await runAirQualityHistory(OWNER);
|
||||
expect(fresh.entryDays!.days).toContain("2025-11-20");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user