diff --git a/.env.example b/.env.example index 449cffe..e1fc5e2 100644 --- a/.env.example +++ b/.env.example @@ -61,6 +61,16 @@ TRAINING_HUB_SITE_HOSTING_LOCATION=Ashburn, Virginia, USA MARKETGUARD_DB_DRIVER=mariadb MARKETGUARD_CACHE_TTL_SECONDS=60 MARKETGUARD_STALE_IF_ERROR_SECONDS=300 +# Refresh the Lowest BIN and Bazaar snapshots on a timer instead of inside client +# requests. Leave this on: with it off, the first caller after the cache expires +# pays for the full Hypixel fetch and every other request waits behind it. +MARKETGUARD_BACKGROUND_REFRESH_ENABLED=true +# 0 = follow MARKETGUARD_CACHE_TTL_SECONDS +MARKETGUARD_BACKGROUND_REFRESH_INTERVAL_SECONDS=0 +MARKETGUARD_BACKGROUND_REFRESH_RETRY_SECONDS=15 +# Extra snapshot age /api/v1/ready still reports as 200. Hypixel only regenerates +# these datasets about once a minute, so a slightly-over-TTL snapshot is normal. +MARKETGUARD_READINESS_FRESH_GRACE_SECONDS=120 MARKETGUARD_HISTORY_RETENTION_DAYS=45 MARKETGUARD_LOWESTBIN_RATE_LIMIT_PER_MINUTE=30 MARKETGUARD_HYPIXEL_API_KEY= diff --git a/DEPLOY.md b/DEPLOY.md index 421e5b8..89479c2 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -268,6 +268,7 @@ Important notes: - keep `TRAINING_HUB_TRUSTED_PROXIES=127.0.0.1` unless you know you need more - `/docs`, `/redoc`, and `/openapi.json` should stay disabled publicly unless you intentionally expose them - keep `WEB_CONCURRENCY=1` while `/api/v1/players` uses its process-local per-IP limiter; Redis shares cached responses but not rate-limit state +- keep `MARKETGUARD_BACKGROUND_REFRESH_ENABLED=true`; the single worker relies on snapshots being refreshed off the request path - use Admin Analytics > Metrics to inspect Player QUERY cache hit rate, response time, active upstream loads, and upstream failures before raising `MARKETGUARD_PLAYERS_MAX_UPSTREAM_CONCURRENCY` ## 9) Run Preflight @@ -491,6 +492,39 @@ bash scripts/preflight.sh Fix the exact value the script reports before retrying the deploy. +### Uptime monitor reports the API offline or flapping + +The public API keeps its Hypixel snapshots warm with a background refresher. If +uptime checks flap, work through this in order. + +1. Point the monitor at `https:///api/v1/ready` and accept `200` and `206` + as up. `206` means "serving slightly stale data", not "down"; only `503` + means no usable snapshot. Do not monitor `/api/v1/health` - it is + intentionally `403` from public networks. +2. Confirm the refresher started: + + ```bash + docker compose logs scamscreener-api | grep "background refresher" + ``` + + You should see `Started MarketGuard lowestbin background refresher` and the + same for `bazaar`. If instead you see `MarketGuard background refresh is + disabled`, `MARKETGUARD_BACKGROUND_REFRESH_ENABLED` is set to `false` - + with it off, refreshes fall back into client requests and a single Uvicorn + worker goes unresponsive on every cache expiry. +3. Check for repeated `background refresh failed` warnings. Those are upstream + Hypixel or storage errors: the last good snapshot is still served (marked + stale) and the refresher retries every + `MARKETGUARD_BACKGROUND_REFRESH_RETRY_SECONDS`. +4. If snapshots are consistently just past their TTL, raise + `MARKETGUARD_READINESS_FRESH_GRACE_SECONDS` rather than lowering + `MARKETGUARD_CACHE_TTL_SECONDS`; Hypixel only regenerates these datasets + about once per minute, so fetching more often buys nothing and burns quota. + +Behind Cloudflare, also make sure the monitor is not being served a cached +response: `/api/v1/ready` sends `Cache-Control: no-store`, so a cached `200` +there means a proxy rule is overriding origin cache headers. + ### Compose starts but the app is unhealthy Check: @@ -505,6 +539,9 @@ Typical causes: - missing SMTP values while admin MFA or password-reset mail is enabled - mismatched domain settings - missing external-auth provider credentials when that flow is enabled +- the API container failing its healthcheck because the event loop is busy; with + background refresh enabled this should not happen, so capture + `docker compose logs scamscreener-api` before restarting ## Final Expected State diff --git a/README.md b/README.md index d0a448e..ebd2e47 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,10 @@ The production topology is Compose-first. Running a single `docker run` containe - `MARKETGUARD_DB_REQUIRE_TLS` and `MARKETGUARD_DB_SSL_CA` enable verified external MariaDB TLS; the managed internal compose database defaults to plain internal transport - `MARKETGUARD_CACHE_TTL_SECONDS` default `60` - `MARKETGUARD_STALE_IF_ERROR_SECONDS` default `300` +- `MARKETGUARD_BACKGROUND_REFRESH_ENABLED` default `true`; refreshes the Lowest BIN and Bazaar snapshots on a timer so client requests never pay for a Hypixel fetch +- `MARKETGUARD_BACKGROUND_REFRESH_INTERVAL_SECONDS` default `0`, meaning "follow `MARKETGUARD_CACHE_TTL_SECONDS`" +- `MARKETGUARD_BACKGROUND_REFRESH_RETRY_SECONDS` default `15`; retry delay after a failed background refresh +- `MARKETGUARD_READINESS_FRESH_GRACE_SECONDS` default `120`; extra snapshot age `/api/v1/ready` still reports as `200` instead of `206` - `MARKETGUARD_HISTORY_RETENTION_DAYS` default `45` - `MARKETGUARD_LOWESTBIN_RATE_LIMIT_PER_MINUTE` default `30` - `MARKETGUARD_HYPIXEL_API_KEY` required for `/api/v1/players`; keep it only in the deployment secret environment diff --git a/app/main.py b/app/main.py index 6cf7c3e..3b82ded 100644 --- a/app/main.py +++ b/app/main.py @@ -1,14 +1,53 @@ from __future__ import annotations +import contextlib +import logging + from fastapi import FastAPI from .marketguard_api.config import MarketGuardSettings from .marketguard_api.player_service import PlayerService +from .marketguard_api.refresher import build_refresh_supervisor from .marketguard_api.routes import register_marketguard_routes from .marketguard_api.service import BazaarService, LowestBinService from .training_hub.config.settings import TrainingHubSettings from .training_hub.main import create_training_hub_app, install_filtered_openapi +logger = logging.getLogger(__name__) + + +def _install_marketguard_refreshers(app: FastAPI) -> None: + """Run the snapshot refreshers alongside the Training Hub lifespan. + + The combined entrypoint reuses the hub's lifespan, so the refreshers are + layered on top of it rather than replacing it. Without them the MarketGuard + routes would fall back to refreshing inside client requests. + """ + supervisor = build_refresh_supervisor( + getattr(app.state, "marketguard_settings", None), + lowestbin_service=getattr(app.state, "marketguard_service", None), + bazaar_service=getattr(app.state, "marketguard_bazaar_service", None), + ) + app.state.marketguard_refresh_supervisor = supervisor + if supervisor is None: + return + + hub_lifespan = app.router.lifespan_context + + @contextlib.asynccontextmanager + async def combined_lifespan(scoped_app: FastAPI): + async with hub_lifespan(scoped_app): + await supervisor.start() + try: + yield + finally: + try: + await supervisor.stop() + except Exception: + logger.exception("Could not stop the MarketGuard snapshot refreshers during shutdown.") + + app.router.lifespan_context = combined_lifespan + def create_app( training_hub_settings: TrainingHubSettings | None = None, @@ -25,6 +64,7 @@ def create_app( bazaar_service=marketguard_bazaar_service, player_service=marketguard_player_service, ) + _install_marketguard_refreshers(app) install_filtered_openapi( app, route_filter=lambda route: str(route.endpoint.__module__) == "app.marketguard_api.routes", diff --git a/app/marketguard_api/config.py b/app/marketguard_api/config.py index c48986c..2310450 100644 --- a/app/marketguard_api/config.py +++ b/app/marketguard_api/config.py @@ -108,6 +108,10 @@ class MarketGuardSettings: snapshot_retries: int = 3 cache_ttl_seconds: int = 60 stale_if_error_seconds: int = 300 + background_refresh_enabled: bool = True + background_refresh_interval_seconds: int = 0 + background_refresh_retry_seconds: int = 15 + readiness_fresh_grace_seconds: int = 120 history_retention_days: int = 45 lowestbin_rate_limit_per_minute: int = 30 players_rate_limit_per_minute: int = 3 @@ -123,6 +127,28 @@ class MarketGuardSettings: redis_key_prefix: str = "marketguard:response" api_docs_enabled: bool = True + @property + def effective_background_refresh_interval_seconds(self) -> int: + """Refresh cadence of the background snapshot refresher. + + Defaults to the cache TTL so a snapshot is replaced right about when it + would otherwise go stale, which keeps every request on a warm cache. + """ + if self.background_refresh_interval_seconds > 0: + return self.background_refresh_interval_seconds + return max(5, int(self.cache_ttl_seconds)) + + @property + def readiness_fresh_seconds(self) -> int: + """Snapshot age still reported as fully ready by ``/api/v1/ready``. + + Hypixel only regenerates the auction house roughly once per minute, so a + snapshot slightly older than the cache TTL is normal operation, not a + degraded service. Without the grace window every uptime probe that + requires HTTP 200 flaps between ready and degraded. + """ + return int(self.cache_ttl_seconds) + int(self.readiness_fresh_grace_seconds) + @classmethod def from_env(cls) -> "MarketGuardSettings": base_dir = Path(__file__).resolve().parents[2] @@ -167,6 +193,10 @@ def from_env(cls) -> "MarketGuardSettings": snapshot_retries=_env_int("MARKETGUARD_SNAPSHOT_RETRIES", 3, 1, 10), cache_ttl_seconds=_env_int("MARKETGUARD_CACHE_TTL_SECONDS", 60, 5, 900), stale_if_error_seconds=_env_int("MARKETGUARD_STALE_IF_ERROR_SECONDS", 300, 5, 3600), + background_refresh_enabled=_env_bool("MARKETGUARD_BACKGROUND_REFRESH_ENABLED", True), + background_refresh_interval_seconds=_env_int("MARKETGUARD_BACKGROUND_REFRESH_INTERVAL_SECONDS", 0, 0, 900), + background_refresh_retry_seconds=_env_int("MARKETGUARD_BACKGROUND_REFRESH_RETRY_SECONDS", 15, 5, 300), + readiness_fresh_grace_seconds=_env_int("MARKETGUARD_READINESS_FRESH_GRACE_SECONDS", 120, 0, 3600), history_retention_days=_env_int("MARKETGUARD_HISTORY_RETENTION_DAYS", 45, 31, 365), lowestbin_rate_limit_per_minute=_env_int("MARKETGUARD_LOWESTBIN_RATE_LIMIT_PER_MINUTE", 30, 0, 600), players_rate_limit_per_minute=_env_int("MARKETGUARD_PLAYERS_RATE_LIMIT_PER_MINUTE", 3, 0, 60), @@ -185,6 +215,10 @@ def from_env(cls) -> "MarketGuardSettings": ) if settings.stale_if_error_seconds < settings.cache_ttl_seconds: raise ValueError("MARKETGUARD_STALE_IF_ERROR_SECONDS must be greater than or equal to CACHE_TTL_SECONDS.") + if settings.readiness_fresh_grace_seconds > settings.stale_if_error_seconds: + raise ValueError( + "MARKETGUARD_READINESS_FRESH_GRACE_SECONDS must not exceed MARKETGUARD_STALE_IF_ERROR_SECONDS." + ) if settings.redis_enabled and not settings.redis_url: raise ValueError("MARKETGUARD_REDIS_ENABLED requires a usable Redis URL or host/port configuration.") return settings diff --git a/app/marketguard_api/main.py b/app/marketguard_api/main.py index f298625..af64d84 100644 --- a/app/marketguard_api/main.py +++ b/app/marketguard_api/main.py @@ -12,6 +12,7 @@ from .config import MarketGuardSettings from .player_service import PlayerService from .rate_limit import InMemoryRateLimiter +from .refresher import build_refresh_supervisor from .routes import register_marketguard_routes from .service import BazaarService, LowestBinService from .storage import MarketGuardStorage @@ -47,15 +48,27 @@ def create_marketguard_app( runtime_bazaar_service = bazaar_service or BazaarService(runtime_settings, storage=storage) runtime_player_service = player_service or PlayerService(runtime_settings) persistent_api_metrics = PersistentApiMetricsRecorder(runtime_settings.database_url) + refresh_supervisor = build_refresh_supervisor( + runtime_settings, + lowestbin_service=lowestbin_service, + bazaar_service=runtime_bazaar_service, + ) docs_url = "/docs" if runtime_settings.api_docs_enabled else None redoc_url = "/redoc" if runtime_settings.api_docs_enabled else None openapi_url = "/openapi.json" if runtime_settings.api_docs_enabled else None @asynccontextmanager async def app_lifespan(_: FastAPI): + if refresh_supervisor is not None: + await refresh_supervisor.start() try: yield finally: + if refresh_supervisor is not None: + try: + await refresh_supervisor.stop() + except Exception: + logger.exception("Could not stop the MarketGuard snapshot refreshers during shutdown.") try: await run_in_threadpool(persistent_api_metrics.flush) except Exception: @@ -85,6 +98,7 @@ async def app_lifespan(_: FastAPI): app.state.marketguard_response_cache = runtime_response_cache app.state.live_api_metrics = LiveApiRequestMetrics() app.state.persistent_api_metrics = persistent_api_metrics + app.state.marketguard_refresh_supervisor = refresh_supervisor def _require_internal_observability_access(request: Request) -> None: if _request_originates_from_internal_network(request, runtime_settings.trusted_proxies): diff --git a/app/marketguard_api/refresher.py b/app/marketguard_api/refresher.py new file mode 100644 index 0000000..9aae65b --- /dev/null +++ b/app/marketguard_api/refresher.py @@ -0,0 +1,189 @@ +"""Background refresh of the MarketGuard market snapshots. + +Without this, a Hypixel refresh only ever happened inside a client request: the +first caller after the cache TTL expired paid the full cost of paginating the +auction house and decoding every item payload, and - with a single Uvicorn +worker - every other caller, plus the container health probe and the public +readiness probe, waited behind it. That is what an uptime monitor sees as the +service going offline several times an hour. + +Refreshing on a timer instead keeps the stored snapshot inside its TTL, so the +request path is a plain MariaDB read and readiness stays green. +""" + +from __future__ import annotations + +import asyncio +import logging +import random +from typing import Protocol + +from .config import MarketGuardSettings +from .exceptions import HypixelUpstreamError, MarketGuardStorageError + +logger = logging.getLogger(__name__) + +# Spread the very first refresh of each dataset so a restart does not fire every +# upstream fetch in the same instant. +_INITIAL_JITTER_SECONDS = 2.0 +# Applied to every scheduled interval so multiple instances drift apart instead +# of hammering Hypixel in lockstep. +_INTERVAL_JITTER_RATIO = 0.1 + + +class RefreshableService(Protocol): + async def refresh(self) -> object: ... + + def disable_inline_refresh(self) -> None: ... + + def enable_inline_refresh(self) -> None: ... + + +class MarketSnapshotRefresher: + """Keeps one dataset's stored snapshot fresh on a fixed cadence.""" + + def __init__( + self, + name: str, + service: RefreshableService, + settings: MarketGuardSettings, + *, + sleep=asyncio.sleep, + ) -> None: + self._name = str(name) + self._service = service + self._settings = settings + self._sleep = sleep + self._task: asyncio.Task[None] | None = None + self._first_refresh_done = asyncio.Event() + + @property + def name(self) -> str: + return self._name + + @property + def is_running(self) -> bool: + return self._task is not None and not self._task.done() + + async def start(self) -> None: + if self.is_running: + return + self._first_refresh_done.clear() + self._task = asyncio.create_task(self._run(), name=f"marketguard-refresh-{self._name}") + logger.info( + "Started MarketGuard %s background refresher (interval %ss).", + self._name, + self._settings.effective_background_refresh_interval_seconds, + ) + + async def stop(self) -> None: + task = self._task + self._task = None + # Requests must be able to refresh again once nothing else does it. + self._service.enable_inline_refresh() + if task is None or task.done(): + return + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception: + logger.exception("MarketGuard %s background refresher failed during shutdown.", self._name) + logger.info("Stopped MarketGuard %s background refresher.", self._name) + + async def wait_for_first_refresh(self, timeout: float | None = None) -> bool: + """Block until the first refresh attempt settled. Test and startup hook.""" + if timeout is None: + await self._first_refresh_done.wait() + return True + try: + await asyncio.wait_for(self._first_refresh_done.wait(), timeout) + return True + except asyncio.TimeoutError: + return False + + async def _run(self) -> None: + await self._sleep(random.uniform(0, _INITIAL_JITTER_SECONDS)) + while True: + succeeded = await self._refresh_once() + self._first_refresh_done.set() + await self._sleep(self._next_delay_seconds(succeeded)) + + async def _refresh_once(self) -> bool: + try: + await self._service.refresh() + except asyncio.CancelledError: + raise + except (HypixelUpstreamError, MarketGuardStorageError) as exc: + # Expected upstream/storage trouble: the stored snapshot is still + # served (as stale) and the next tick retries sooner. + logger.warning("MarketGuard %s background refresh failed: %s", self._name, exc) + return False + except Exception: + logger.exception("Unexpected error during MarketGuard %s background refresh.", self._name) + return False + + # Only now is a request guaranteed to find a snapshot it can serve + # without going upstream itself. + self._service.disable_inline_refresh() + logger.debug("Refreshed MarketGuard %s snapshot.", self._name) + return True + + def _next_delay_seconds(self, succeeded: bool) -> float: + if not succeeded: + return float(self._settings.background_refresh_retry_seconds) + interval = float(self._settings.effective_background_refresh_interval_seconds) + return interval + random.uniform(0, interval * _INTERVAL_JITTER_RATIO) + + +class MarketSnapshotRefreshSupervisor: + """Owns the refreshers for every dataset served by the public API.""" + + def __init__(self, refreshers: list[MarketSnapshotRefresher]) -> None: + self._refreshers = list(refreshers) + + @property + def refreshers(self) -> list[MarketSnapshotRefresher]: + return list(self._refreshers) + + async def start(self) -> None: + for refresher in self._refreshers: + await refresher.start() + + async def stop(self) -> None: + for refresher in self._refreshers: + await refresher.stop() + + +def build_refresh_supervisor( + settings: MarketGuardSettings | None, + *, + lowestbin_service: RefreshableService | None, + bazaar_service: RefreshableService | None, +) -> MarketSnapshotRefreshSupervisor | None: + """Build the supervisor, or ``None`` when background refresh is disabled.""" + if settings is None: + return None + if not settings.background_refresh_enabled: + logger.warning( + "MarketGuard background refresh is disabled; snapshot refreshes fall back to the request path." + ) + return None + + refreshers: list[MarketSnapshotRefresher] = [] + for name, service in (("lowestbin", lowestbin_service), ("bazaar", bazaar_service)): + if service is None or not _is_refreshable(service): + continue + refreshers.append(MarketSnapshotRefresher(name, service, settings)) + + if not refreshers: + return None + return MarketSnapshotRefreshSupervisor(refreshers) + + +def _is_refreshable(service: object) -> bool: + return all( + callable(getattr(service, attribute, None)) + for attribute in ("refresh", "disable_inline_refresh", "enable_inline_refresh") + ) diff --git a/app/marketguard_api/routes.py b/app/marketguard_api/routes.py index a6c34ce..e095e8d 100644 --- a/app/marketguard_api/routes.py +++ b/app/marketguard_api/routes.py @@ -632,7 +632,10 @@ def _readiness_component_payload( if not isinstance(generated_at, datetime): return {"status": "down", "lastUpdated": None} snapshot_age_seconds = now_epoch_seconds - generated_at.timestamp() - if snapshot_age_seconds < int(settings.cache_ttl_seconds): + # Hypixel only regenerates these datasets about once a minute, so a snapshot + # marginally older than the cache TTL is healthy, not degraded. Reporting it + # as degraded made uptime probes that require HTTP 200 flap constantly. + if snapshot_age_seconds < int(settings.readiness_fresh_seconds): component_status = "ok" elif snapshot_age_seconds < int(settings.stale_if_error_seconds): component_status = "stale" diff --git a/app/marketguard_api/service.py b/app/marketguard_api/service.py index de33579..b0208c2 100644 --- a/app/marketguard_api/service.py +++ b/app/marketguard_api/service.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, replace from decimal import Decimal, InvalidOperation from datetime import datetime, timezone +from typing import Any from .client import HypixelAuctionClient, HypixelBazaarClient from .config import MarketGuardSettings @@ -17,6 +18,62 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True, slots=True) +class LowestBinSnapshotMetadata: + """Per-item metadata belonging to exactly one Lowest BIN snapshot.""" + + auctioneer_uuids: dict[str, str] + item_names: dict[str, str] + + +@dataclass(frozen=True, slots=True) +class LowestBinIndex: + lowest_bins: dict[str, float] + metadata: LowestBinSnapshotMetadata + total_bin_auctions: int + + +def build_lowestbin_index(auctions: list[dict[str, Any]]) -> LowestBinIndex: + """Decode every auction payload and index the lowest BIN price per item key. + + This is pure CPU work: a full Hypixel auction house is ~100k auctions, and + each one costs a base64 decode, a gzip inflate and an NBT parse. Measured at + 5-10 seconds for a full house, so it must never run on the event loop - + doing so stalls every other request, including the health and readiness + probes. Callers run it through ``asyncio.to_thread``. + """ + lowest_bins: dict[str, float] = {} + auctioneer_uuids: dict[str, str] = {} + item_names: dict[str, str] = {} + total_bin_auctions = 0 + + for auction in auctions: + if auction.get("bin") is not True: + continue + + resolved_item = resolve_auction_item(auction) + if resolved_item is None: + continue + + auctioneer_uuid = _parse_auctioneer_uuid(auction.get("auctioneer")) + if auctioneer_uuid is None: + continue + + total_bin_auctions += 1 + for item_key in resolved_item.keys: + current_lowest = lowest_bins.get(item_key) + if current_lowest is None or resolved_item.unit_price < current_lowest: + lowest_bins[item_key] = resolved_item.unit_price + auctioneer_uuids[item_key] = auctioneer_uuid + item_names[item_key] = _parse_item_name(auction.get("item_name"), item_key) + + return LowestBinIndex( + lowest_bins=dict(sorted(lowest_bins.items(), key=lambda item: item[0].lower())), + metadata=LowestBinSnapshotMetadata(auctioneer_uuids=auctioneer_uuids, item_names=item_names), + total_bin_auctions=total_bin_auctions, + ) + + class LowestBinService: def __init__( self, @@ -29,39 +86,32 @@ def __init__( self._client = client or HypixelAuctionClient(settings) self._clock = clock or _utc_epoch_seconds self._storage = storage or MarketGuardStorage(settings.database_url, settings.history_retention_days) - self._last_auctioneer_uuids: dict[str, str] = {} - self._last_item_names: dict[str, str] = {} self._lock = asyncio.Lock() + # Flipped on by the background refresher. While it owns the upstream + # fetch, a request must never trigger one itself: a Hypixel refresh + # costs tens of seconds, and paying it inside a request is what turned + # a slow refresh into an outage for every other caller. + self._inline_refresh_enabled = True + + @property + def inline_refresh_enabled(self) -> bool: + return self._inline_refresh_enabled + + def disable_inline_refresh(self) -> None: + self._inline_refresh_enabled = False + + def enable_inline_refresh(self) -> None: + self._inline_refresh_enabled = True async def aclose(self) -> None: await self._client.aclose() async def get_lowest_bins(self) -> LowestBinSnapshot: - async with self._lock: - now = self._clock() - stored = await asyncio.to_thread(self._storage.read_lowestbin_snapshot) - if stored is not None: - self._remember_snapshot_metadata(stored) - if _is_snapshot_fresh(stored.snapshot, now, self._settings.cache_ttl_seconds): - return stored.snapshot - - try: - snapshot = await self._refresh_snapshot() - except HypixelUpstreamError: - if stored is not None and _can_serve_snapshot_stale( - stored.snapshot, - now, - self._settings.stale_if_error_seconds, - ): - logger.warning("Serving stale MarketGuard Lowest BIN data from MariaDB after Hypixel refresh failure.") - return replace(stored.snapshot, is_stale=True) - raise - - await self._persist_snapshot(snapshot) - return snapshot + snapshot, _metadata = await self._load_snapshot() + return snapshot async def get_lowest_bins_v2(self) -> LowestBinV2Snapshot: - snapshot = await self.get_lowest_bins() + snapshot, metadata = await self._load_snapshot() anchor_day = snapshot_day_from_last_updated(snapshot.snapshot_last_updated) averages = await asyncio.to_thread( self._storage.get_averages, @@ -71,14 +121,14 @@ async def get_lowest_bins_v2(self) -> LowestBinV2Snapshot: items: dict[str, LowestBinV2Entry] = {} for item_key, price in snapshot.items.items(): - auctioneer_uuid = self._find_auctioneer_uuid_for_price(item_key, price) + auctioneer_uuid = _lookup_auctioneer_uuid(metadata, item_key, price) if auctioneer_uuid is None: continue average_window = averages.get(item_key) items[item_key] = LowestBinV2Entry( price=price, auctioneer_uuid=auctioneer_uuid, - item_name=self._find_item_name_for_key(item_key), + item_name=_lookup_item_name(metadata, item_key), avg_7d=None if average_window is None else average_window.avg_7d, avg_30d=None if average_window is None else average_window.avg_30d, ) @@ -93,53 +143,94 @@ async def get_lowest_bins_v2(self) -> LowestBinV2Snapshot: is_stale=snapshot.is_stale, ) - async def _refresh_snapshot(self) -> LowestBinSnapshot: - auction_snapshot = await self._client.fetch_snapshot() - lowest_bins: dict[str, float] = {} - auctioneer_uuids: dict[str, str] = {} - item_names: dict[str, str] = {} - total_bin_auctions = 0 + async def refresh(self) -> LowestBinSnapshot: + """Fetch a fresh snapshot from Hypixel and persist it. - for auction in auction_snapshot.auctions: - if auction.get("bin") is not True: - continue + Entry point for the background refresher. Serialized against inline + refreshes by the same lock so only one upstream fetch is ever in flight. + """ + async with self._lock: + snapshot, metadata = await self._refresh_snapshot() + await self._persist_snapshot(snapshot, metadata) + return snapshot - resolved_item = resolve_auction_item(auction) - if resolved_item is None: - continue + async def _load_snapshot(self) -> tuple[LowestBinSnapshot, LowestBinSnapshotMetadata]: + """Serve the current snapshot, refreshing inline only when unavoidable. + + The read path deliberately takes no lock: a stored snapshot is served + straight from MariaDB even while a refresh is running, so a slow Hypixel + fetch can never queue up client requests behind it. + """ + now = self._clock() + stored = await asyncio.to_thread(self._storage.read_lowestbin_snapshot) + if stored is not None: + metadata = _metadata_from_stored(stored) + if _is_snapshot_fresh(stored.snapshot, now, self._settings.cache_ttl_seconds): + return stored.snapshot, metadata + if not self._inline_refresh_enabled and _can_serve_snapshot_stale( + stored.snapshot, + now, + self._settings.stale_if_error_seconds, + ): + # The refresher is behind (Hypixel slow or down). Serving the + # last good snapshot beats making the caller wait for upstream. + return replace(stored.snapshot, is_stale=True), metadata - auctioneer_uuid = _parse_auctioneer_uuid(auction.get("auctioneer")) - if auctioneer_uuid is None: - continue + async with self._lock: + # Another caller may have refreshed while we waited for the lock. + recheck_now = self._clock() + recheck = await asyncio.to_thread(self._storage.read_lowestbin_snapshot) + if recheck is not None and _is_snapshot_fresh( + recheck.snapshot, + recheck_now, + self._settings.cache_ttl_seconds, + ): + return recheck.snapshot, _metadata_from_stored(recheck) + + fallback = recheck if recheck is not None else stored + try: + snapshot, metadata = await self._refresh_snapshot() + except HypixelUpstreamError: + if fallback is not None and _can_serve_snapshot_stale( + fallback.snapshot, + recheck_now, + self._settings.stale_if_error_seconds, + ): + logger.warning("Serving stale MarketGuard Lowest BIN data from MariaDB after Hypixel refresh failure.") + return replace(fallback.snapshot, is_stale=True), _metadata_from_stored(fallback) + raise - total_bin_auctions += 1 - for item_key in resolved_item.keys: - current_lowest = lowest_bins.get(item_key) - if current_lowest is None or resolved_item.unit_price < current_lowest: - lowest_bins[item_key] = resolved_item.unit_price - auctioneer_uuids[item_key] = auctioneer_uuid - item_names[item_key] = _parse_item_name(auction.get("item_name"), item_key) + await self._persist_snapshot(snapshot, metadata) + return snapshot, metadata + + async def _refresh_snapshot(self) -> tuple[LowestBinSnapshot, LowestBinSnapshotMetadata]: + auction_snapshot = await self._client.fetch_snapshot() + # Decoding ~100k NBT item payloads is seconds of CPU; keep it off the + # event loop so health, readiness and every other route stay responsive. + index = await asyncio.to_thread(build_lowestbin_index, auction_snapshot.auctions) snapshot = LowestBinSnapshot( generated_at=_clock_datetime(self._clock()), snapshot_last_updated=auction_snapshot.last_updated, total_pages=auction_snapshot.total_pages, total_auctions=len(auction_snapshot.auctions), - total_bin_auctions=total_bin_auctions, - items=dict(sorted(lowest_bins.items(), key=lambda item: item[0].lower())), + total_bin_auctions=index.total_bin_auctions, + items=index.lowest_bins, is_stale=False, ) - self._last_auctioneer_uuids = auctioneer_uuids - self._last_item_names = item_names - return snapshot + return snapshot, index.metadata - async def _persist_snapshot(self, snapshot: LowestBinSnapshot) -> None: + async def _persist_snapshot( + self, + snapshot: LowestBinSnapshot, + metadata: LowestBinSnapshotMetadata, + ) -> None: try: await asyncio.to_thread( self._storage.write_lowestbin_snapshot, snapshot, - auctioneer_uuids=self._last_auctioneer_uuids, - item_names=self._last_item_names, + auctioneer_uuids=metadata.auctioneer_uuids, + item_names=metadata.item_names, ) except MarketGuardStorageError: logger.exception( @@ -148,25 +239,28 @@ async def _persist_snapshot(self, snapshot: LowestBinSnapshot) -> None: ) raise - def _remember_snapshot_metadata(self, stored: StoredLowestBinSnapshot) -> None: - self._last_auctioneer_uuids = dict(stored.auctioneer_uuids) - self._last_item_names = dict(stored.item_names) - def _find_auctioneer_uuid_for_price(self, item_key: str, price: float) -> str | None: - auctioneer_uuids = getattr(self, "_last_auctioneer_uuids", {}) - auctioneer_uuid = auctioneer_uuids.get(item_key) - if not auctioneer_uuid: - logger.warning("Missing auctioneer UUID for Lowest BIN key %s at price %s.", item_key, price) - return None - return auctioneer_uuid +def _metadata_from_stored(stored: StoredLowestBinSnapshot) -> LowestBinSnapshotMetadata: + return LowestBinSnapshotMetadata( + auctioneer_uuids=dict(stored.auctioneer_uuids), + item_names=dict(stored.item_names), + ) - def _find_item_name_for_key(self, item_key: str) -> str: - item_names = getattr(self, "_last_item_names", {}) - item_name = item_names.get(item_key) - if item_name: - return item_name - logger.warning("Missing item_name for Lowest BIN key %s.", item_key) - return item_key + +def _lookup_auctioneer_uuid(metadata: LowestBinSnapshotMetadata, item_key: str, price: float) -> str | None: + auctioneer_uuid = metadata.auctioneer_uuids.get(item_key) + if not auctioneer_uuid: + logger.warning("Missing auctioneer UUID for Lowest BIN key %s at price %s.", item_key, price) + return None + return auctioneer_uuid + + +def _lookup_item_name(metadata: LowestBinSnapshotMetadata, item_key: str) -> str: + item_name = metadata.item_names.get(item_key) + if item_name: + return item_name + logger.warning("Missing item_name for Lowest BIN key %s.", item_key) + return item_key class BazaarService: @@ -182,32 +276,70 @@ def __init__( self._clock = clock or _utc_epoch_seconds self._storage = storage or MarketGuardStorage(settings.database_url, settings.history_retention_days) self._lock = asyncio.Lock() + self._inline_refresh_enabled = True + + @property + def inline_refresh_enabled(self) -> bool: + return self._inline_refresh_enabled + + def disable_inline_refresh(self) -> None: + self._inline_refresh_enabled = False + + def enable_inline_refresh(self) -> None: + self._inline_refresh_enabled = True async def aclose(self) -> None: await self._client.aclose() - async def get_bazaar(self) -> BazaarSnapshot: + async def refresh(self) -> BazaarSnapshot: + """Fetch fresh bazaar data from Hypixel and persist it.""" async with self._lock: - now = self._clock() - cached = await asyncio.to_thread(self._storage.read_bazaar_snapshot) - if cached is not None and _is_snapshot_fresh(cached, now, self._settings.cache_ttl_seconds): + snapshot = await self._refresh_snapshot() + await self._persist_snapshot(snapshot) + return snapshot + + async def get_bazaar(self) -> BazaarSnapshot: + now = self._clock() + cached = await asyncio.to_thread(self._storage.read_bazaar_snapshot) + if cached is not None: + if _is_snapshot_fresh(cached, now, self._settings.cache_ttl_seconds): return cached + if not self._inline_refresh_enabled and _can_serve_snapshot_stale( + cached, + now, + self._settings.stale_if_error_seconds, + ): + return replace(cached, is_stale=True) + + async with self._lock: + recheck_now = self._clock() + recheck = await asyncio.to_thread(self._storage.read_bazaar_snapshot) + if recheck is not None and _is_snapshot_fresh(recheck, recheck_now, self._settings.cache_ttl_seconds): + return recheck + fallback = recheck if recheck is not None else cached try: snapshot = await self._refresh_snapshot() except HypixelUpstreamError: - if cached is not None and _can_serve_snapshot_stale(cached, now, self._settings.stale_if_error_seconds): + if fallback is not None and _can_serve_snapshot_stale( + fallback, + recheck_now, + self._settings.stale_if_error_seconds, + ): logger.warning("Serving stale MarketGuard bazaar data from MariaDB after Hypixel refresh failure.") - return replace(cached, is_stale=True) + return replace(fallback, is_stale=True) raise - try: - await asyncio.to_thread(self._storage.write_bazaar_snapshot, snapshot) - except MarketGuardStorageError: - logger.exception("Failed to persist MarketGuard bazaar snapshot %s.", snapshot.snapshot_last_updated) - raise + await self._persist_snapshot(snapshot) return snapshot + async def _persist_snapshot(self, snapshot: BazaarSnapshot) -> None: + try: + await asyncio.to_thread(self._storage.write_bazaar_snapshot, snapshot) + except MarketGuardStorageError: + logger.exception("Failed to persist MarketGuard bazaar snapshot %s.", snapshot.snapshot_last_updated) + raise + async def _refresh_snapshot(self) -> BazaarSnapshot: bazaar_snapshot = await self._client.fetch_snapshot() products: dict[str, dict[str, float | int | str]] = {} diff --git a/requirements.txt b/requirements.txt index feabdcd..0ed721b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ python-dotenv==1.2.2 python-multipart==0.0.31 jinja2==3.1.6 PyMySQL==1.1.2 -cryptography==48.0.1 +cryptography==50.0.0 webauthn==2.7.1 qrcode==8.2 pytest==9.0.3 diff --git a/tests/test_marketguard_api.py b/tests/test_marketguard_api.py index 30e457e..026e8c3 100644 --- a/tests/test_marketguard_api.py +++ b/tests/test_marketguard_api.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any +import threading + import httpx from fastapi.testclient import TestClient @@ -24,6 +26,8 @@ from app.marketguard_api.models import BazaarSnapshot, LowestBinSnapshot from app.marketguard_api.nbt import parse_inventory_nbt from app.marketguard_api.player_service import PlayerService +from app.marketguard_api.refresher import MarketSnapshotRefresher, build_refresh_supervisor +from app.marketguard_api import service as service_module from app.marketguard_api.service import BazaarService, LowestBinService from app.marketguard_api.storage import LowestBinAverageWindow, StoredLowestBinSnapshot, snapshot_day_from_last_updated from app.training_hub.config.settings import TrainingHubSettings @@ -1042,9 +1046,12 @@ async def _hypixel_handler(_request: httpx.Request) -> httpx.Response: return httpx.Response(403, json={"success": False, "cause": "Invalid API key"}) settings = _marketguard_settings(hypixel_api_key="invalid-hypixel-key") + marketguard_service, marketguard_bazaar_service = _noop_marketguard_services(settings) app = create_app( training_hub_settings=_training_hub_settings(tmp_path), marketguard_settings=settings, + marketguard_service=marketguard_service, + marketguard_bazaar_service=marketguard_bazaar_service, marketguard_player_service=_marketguard_player_service( settings, _hypixel_handler, @@ -1137,7 +1144,7 @@ async def get_players(self, queries) -> dict[str, object]: async def aclose(self) -> None: return None - settings = _marketguard_settings(players_rate_limit_per_minute=3) + settings = _marketguard_settings(players_rate_limit_per_minute=3, hypixel_api_key="test-key") marketguard_service, marketguard_bazaar_service = _noop_marketguard_services(settings) counting_player_service = _CountingPlayerService() app = create_marketguard_app( @@ -2523,13 +2530,23 @@ def _marketguard_settings( redis_enabled: bool = False, redis_url: str = "", trusted_proxies: set[str] | None = None, + background_refresh_enabled: bool = False, + background_refresh_interval_seconds: int = 0, + background_refresh_retry_seconds: int = 15, + readiness_fresh_grace_seconds: int = 0, ) -> MarketGuardSettings: + # Background refresh and the readiness grace window default to off here so + # tests drive the refresh path explicitly instead of racing a timer. return MarketGuardSettings( hypixel_api_base_url="https://api.hypixel.net/v2", database_url="mariadb://scamscreener:test@127.0.0.1:3306/scamscreener_hub", hypixel_api_key=hypixel_api_key, cache_ttl_seconds=cache_ttl_seconds, stale_if_error_seconds=stale_if_error_seconds, + background_refresh_enabled=background_refresh_enabled, + background_refresh_interval_seconds=background_refresh_interval_seconds, + background_refresh_retry_seconds=background_refresh_retry_seconds, + readiness_fresh_grace_seconds=readiness_fresh_grace_seconds, history_retention_days=history_retention_days, lowestbin_rate_limit_per_minute=lowestbin_rate_limit_per_minute, players_rate_limit_per_minute=players_rate_limit_per_minute, @@ -2868,3 +2885,430 @@ def _compound_payload(*children: bytes) -> bytes: def _string_payload(value: str) -> bytes: encoded = value.encode("utf-8") return struct.pack(">H", len(encoded)) + encoded + + +# --- Background snapshot refresh ------------------------------------------------- +# +# Regression cover for the availability incident: the Hypixel refresh used to run +# inside client requests and decoded every auction payload on the event loop, so +# a single Uvicorn worker went unresponsive for seconds at a time on every cache +# expiry - which the uptime monitor reported as the API being offline. + + +def test_lowestbin_refresh_decodes_auctions_off_the_event_loop(monkeypatch) -> None: + """The auction decode must run on a worker thread, not the event loop. + + Decoding a full auction house is seconds of base64 + gzip + NBT work. Running + it on the loop froze every other request, the container health probe and the + public readiness probe for the duration - the direct cause of the uptime + monitor reporting the API offline. Asserting on thread identity keeps this + deterministic instead of timing-dependent. + """ + loop_thread_id = threading.get_ident() + observed: dict[str, Any] = {"decode_thread_id": None, "ticks_during_decode": 0} + decode_started = threading.Event() + decode_may_finish = threading.Event() + real_build_index = service_module.build_lowestbin_index + + def _instrumented_build_index(auctions): + observed["decode_thread_id"] = threading.get_ident() + decode_started.set() + # Hold the decode open so the test can prove the loop still runs. + decode_may_finish.wait(timeout=5) + return real_build_index(auctions) + + monkeypatch.setattr(service_module, "build_lowestbin_index", _instrumented_build_index) + + async def _handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "success": True, + "totalPages": 1, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0), + "auctions": [_auction("HYPERION", 100_000_000, item_name="Hyperion")], + }, + ) + + settings = _marketguard_settings() + service = _marketguard_service(settings, _handler) + + async def _exercise() -> None: + refresh_task = asyncio.create_task(service.refresh()) + # Tick the loop while the decode is blocked in its worker thread. + while not decode_started.is_set(): + await asyncio.sleep(0.001) + for _ in range(10): + await asyncio.sleep(0.001) + observed["ticks_during_decode"] += 1 + decode_may_finish.set() + await refresh_task + await service.aclose() + + asyncio.run(_exercise()) + + assert observed["decode_thread_id"] is not None + assert observed["decode_thread_id"] != loop_thread_id + # The loop kept being scheduled the whole time the decode was running. + assert observed["ticks_during_decode"] == 10 + + +def test_background_refresher_keeps_requests_off_the_upstream_path() -> None: + """Once the refresher owns the refresh, a stale snapshot is served, not refetched.""" + clock = [0.0] + request_count = 0 + + async def _handler(_request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response( + 200, + json={ + "success": True, + "totalPages": 1, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0) + request_count, + "auctions": [_auction("HYPERION", 100_000_000, item_name="Hyperion")], + }, + ) + + settings = _marketguard_settings(cache_ttl_seconds=5, stale_if_error_seconds=300) + service = _marketguard_service(settings, _handler, clock=lambda: clock[0]) + + async def _exercise() -> tuple[Any, int, int]: + refresher = MarketSnapshotRefresher("lowestbin", service, settings, sleep=_single_cycle_sleep()) + await refresher.start() + assert await refresher.wait_for_first_refresh(timeout=5) + after_first_refresh = request_count + + # Snapshot is now past its TTL, but the refresher owns refreshing it. + clock[0] = 60.0 + snapshot = await service.get_lowest_bins() + served_without_upstream = request_count + + await refresher.stop() + await service.aclose() + return snapshot, after_first_refresh, served_without_upstream + + snapshot, after_first_refresh, served_without_upstream = asyncio.run(_exercise()) + + assert after_first_refresh >= 1 + assert snapshot.items == {"HYPERION": 100_000_000.0} + assert snapshot.is_stale is True + # The request served the stored snapshot instead of paying for a fetch. + assert served_without_upstream == after_first_refresh + + +def test_background_refresher_restores_inline_refresh_after_stop() -> None: + request_count = 0 + + async def _handler(_request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response( + 200, + json={ + "success": True, + "totalPages": 1, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0) + request_count, + "auctions": [_auction("HYPERION", 100_000_000, item_name="Hyperion")], + }, + ) + + settings = _marketguard_settings(cache_ttl_seconds=5, stale_if_error_seconds=300) + service = _marketguard_service(settings, _handler, clock=lambda: 0.0) + + async def _exercise() -> tuple[bool, bool]: + refresher = MarketSnapshotRefresher("lowestbin", service, settings, sleep=_single_cycle_sleep()) + await refresher.start() + assert await refresher.wait_for_first_refresh(timeout=5) + disabled_while_running = not service.inline_refresh_enabled + await refresher.stop() + enabled_after_stop = service.inline_refresh_enabled + await service.aclose() + return disabled_while_running, enabled_after_stop + + disabled_while_running, enabled_after_stop = asyncio.run(_exercise()) + + assert disabled_while_running is True + # Nothing else refreshes after shutdown, so requests must be able to again. + assert enabled_after_stop is True + + +def test_background_refresher_survives_upstream_failure_and_retries_sooner() -> None: + attempts = 0 + + async def _handler(_request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + return httpx.Response(503, json={"success": False, "cause": "maintenance"}) + + settings = _marketguard_settings( + cache_ttl_seconds=60, + stale_if_error_seconds=300, + background_refresh_retry_seconds=15, + ) + service = _marketguard_service(settings, _handler, clock=lambda: 0.0) + + async def _exercise() -> tuple[bool, float]: + refresher = MarketSnapshotRefresher("lowestbin", service, settings, sleep=_single_cycle_sleep()) + await refresher.start() + assert await refresher.wait_for_first_refresh(timeout=5) + # A failed refresh must not hand the request path off to a snapshot that + # does not exist yet. + inline_still_enabled = service.inline_refresh_enabled + retry_delay = refresher._next_delay_seconds(False) + await refresher.stop() + await service.aclose() + return inline_still_enabled, retry_delay + + inline_still_enabled, retry_delay = asyncio.run(_exercise()) + + assert attempts >= 1 + assert inline_still_enabled is True + assert retry_delay == 15.0 + + +def test_background_refresher_interval_defaults_to_cache_ttl_with_jitter() -> None: + settings = _marketguard_settings(cache_ttl_seconds=45) + assert settings.effective_background_refresh_interval_seconds == 45 + + pinned = _marketguard_settings(cache_ttl_seconds=45, background_refresh_interval_seconds=20) + assert pinned.effective_background_refresh_interval_seconds == 20 + + service = _NoopRefreshableService() + refresher = MarketSnapshotRefresher("lowestbin", service, settings, sleep=_single_cycle_sleep()) + delay = refresher._next_delay_seconds(True) + # Jitter keeps multiple instances from hitting Hypixel in lockstep. + assert 45.0 <= delay <= 45.0 * 1.1 + + +def test_build_refresh_supervisor_respects_the_disable_switch() -> None: + disabled = _marketguard_settings(background_refresh_enabled=False) + assert ( + build_refresh_supervisor( + disabled, + lowestbin_service=_NoopRefreshableService(), + bazaar_service=_NoopRefreshableService(), + ) + is None + ) + + enabled = _marketguard_settings(background_refresh_enabled=True) + supervisor = build_refresh_supervisor( + enabled, + lowestbin_service=_NoopRefreshableService(), + bazaar_service=_NoopRefreshableService(), + ) + assert supervisor is not None + assert [refresher.name for refresher in supervisor.refreshers] == ["lowestbin", "bazaar"] + + +def test_bazaar_background_refresh_serves_stale_without_upstream_call() -> None: + clock = [0.0] + request_count = 0 + + async def _handler(_request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response( + 200, + json={ + "success": True, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0) + request_count, + "products": { + "ENCHANTED_DIAMOND": { + "quick_status": { + "buyPrice": 100.0, + "sellPrice": 90.0, + "buyVolume": 10, + "sellVolume": 20, + "buyMovingWeek": 30, + "sellMovingWeek": 40, + } + } + }, + }, + ) + + settings = _marketguard_settings(cache_ttl_seconds=5, stale_if_error_seconds=300) + service = _marketguard_bazaar_service(settings, _handler, clock=lambda: clock[0]) + + async def _exercise() -> tuple[Any, int, int]: + refresher = MarketSnapshotRefresher("bazaar", service, settings, sleep=_single_cycle_sleep()) + await refresher.start() + assert await refresher.wait_for_first_refresh(timeout=5) + after_first_refresh = request_count + + clock[0] = 60.0 + snapshot = await service.get_bazaar() + served_without_upstream = request_count + + await refresher.stop() + await service.aclose() + return snapshot, after_first_refresh, served_without_upstream + + snapshot, after_first_refresh, served_without_upstream = asyncio.run(_exercise()) + + assert after_first_refresh >= 1 + assert snapshot.is_stale is True + assert served_without_upstream == after_first_refresh + + +def test_concurrent_lowestbin_requests_trigger_a_single_upstream_refresh() -> None: + """A cold cache under load must not fan out into one Hypixel fetch per caller.""" + request_count = 0 + + async def _handler(_request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + await asyncio.sleep(0.02) + return httpx.Response( + 200, + json={ + "success": True, + "totalPages": 1, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0), + "auctions": [_auction("HYPERION", 100_000_000, item_name="Hyperion")], + }, + ) + + settings = _marketguard_settings(cache_ttl_seconds=60) + service = _marketguard_service(settings, _handler, clock=lambda: 0.0) + + async def _exercise() -> list[Any]: + snapshots = await asyncio.gather(*(service.get_lowest_bins() for _ in range(8))) + await service.aclose() + return snapshots + + snapshots = asyncio.run(_exercise()) + + assert all(snapshot.items == {"HYPERION": 100_000_000.0} for snapshot in snapshots) + assert request_count == 1 + + +def test_readiness_grace_window_keeps_a_normally_aged_snapshot_ready() -> None: + """Hypixel regenerates roughly once a minute, so TTL+15s is healthy, not degraded.""" + snapshot_age_seconds = 75 + + def _build_app(settings: MarketGuardSettings): + store = _MemoryMarketGuardStorage(retention_days=settings.history_retention_days) + _seed_lowestbin_snapshot( + store, + snapshot_last_updated=1_700_000_000_000, + item_prices={"HYPERION": 98_000_000.0}, + generated_at=_relative_snapshot_time(snapshot_age_seconds), + ) + _seed_bazaar_snapshot( + store, + snapshot_last_updated=1_700_000_100_000, + generated_at=_relative_snapshot_time(snapshot_age_seconds), + ) + return create_marketguard_app( + settings=settings, + service=_ReadinessLowestBinService(store), + bazaar_service=_ReadinessBazaarService(store), + ) + + graced_app = _build_app(_marketguard_settings(cache_ttl_seconds=60, readiness_fresh_grace_seconds=120)) + with TestClient(graced_app) as client: + graced = client.get("/api/v1/ready") + + strict_app = _build_app(_marketguard_settings(cache_ttl_seconds=60, readiness_fresh_grace_seconds=0)) + with TestClient(strict_app) as strict_client: + strict = strict_client.get("/api/v1/ready") + + assert graced.status_code == 200 + assert graced.json()["status"] == "ok" + assert graced.headers["x-readiness-status"] == "ok" + # Without the grace window the same snapshot reads as degraded, which is what + # made uptime probes requiring HTTP 200 flap between ready and degraded. + assert strict.status_code == 206 + assert strict.json()["status"] == "degraded" + + +def _single_cycle_sleep(): + """Sleep stub that lets the refresher run exactly one refresh. + + The initial jitter returns immediately; every later delay parks until the + task is cancelled by stop(). Without the park the refresher would spin on a + zero-length sleep and race the assertions with extra upstream calls. + """ + + calls = {"count": 0} + + async def _sleep(_delay: float) -> None: + calls["count"] += 1 + if calls["count"] <= 1: + return + await asyncio.Event().wait() + + return _sleep + + +class _NoopRefreshableService: + def __init__(self) -> None: + self.refresh_count = 0 + self.inline_refresh_enabled = True + + async def refresh(self) -> None: + self.refresh_count += 1 + + def disable_inline_refresh(self) -> None: + self.inline_refresh_enabled = False + + def enable_inline_refresh(self) -> None: + self.inline_refresh_enabled = True + + +def test_marketguard_app_starts_and_stops_the_background_refreshers() -> None: + """The app lifespan must actually own the refreshers, not just build them.""" + settings = _marketguard_settings(background_refresh_enabled=True, cache_ttl_seconds=60) + store = _MemoryMarketGuardStorage(retention_days=settings.history_retention_days) + + async def _lowestbin_handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "success": True, + "totalPages": 1, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0), + "auctions": [_auction("HYPERION", 100_000_000, item_name="Hyperion")], + }, + ) + + async def _bazaar_handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "success": True, + "lastUpdated": _epoch_millis(2025, 3, 1, 12, 0), + "products": {}, + }, + ) + + app = create_marketguard_app( + settings=settings, + service=_marketguard_service(settings, _lowestbin_handler, storage=store), + bazaar_service=_marketguard_bazaar_service(settings, _bazaar_handler, storage=store), + player_service=_marketguard_player_service( + settings, + _unexpected_hypixel_handler, + _unexpected_mojang_handler, + ), + response_cache=None, + ) + + supervisor = app.state.marketguard_refresh_supervisor + assert supervisor is not None + assert [refresher.name for refresher in supervisor.refreshers] == ["lowestbin", "bazaar"] + + with TestClient(app): + assert all(refresher.is_running for refresher in supervisor.refreshers) + + # Shutdown must stop every refresher; a leaked task would keep hitting + # Hypixel after the app is gone. + assert not any(refresher.is_running for refresher in supervisor.refreshers) + + +async def _unexpected_hypixel_handler(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"Unexpected Hypixel request: {request.url}")