admin / Bid-Sentinel
publicBid Scrape and Tracking Application with AI Capability
Bid-Sentinel / bid-sentinel-v2 / backend / app / scraper / scheduler.py
4645 B · main
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | """Background scheduler supporting multiple day/time slots. If one or more enabled ScheduleSlots exist, the scraper runs at each matching day-of-week + time (interpreted in ``SCHEDULE_TIMEZONE``). If no slots are defined, it falls back to a fixed interval (``SCRAPER_INTERVAL_MINUTES``). """ from __future__ import annotations import asyncio import logging from datetime import datetime, timedelta from sqlalchemy import select from app.config import settings from app.database import AsyncSessionLocal from app.models import ScheduleSlot from app.scraper.runner import run_all_scrapers logger = logging.getLogger("scraper.scheduler") try: from zoneinfo import ZoneInfo def _tz(): try: return ZoneInfo(settings.schedule_timezone) except Exception: # noqa: BLE001 - missing tzdata etc. return None except ImportError: # pragma: no cover def _tz(): return None def _now() -> datetime: tz = _tz() return datetime.now(tz) if tz else datetime.now() async def get_enabled_slots() -> list[ScheduleSlot]: async with AsyncSessionLocal() as db: result = await db.execute( select(ScheduleSlot).where(ScheduleSlot.enabled.is_(True)) ) return list(result.scalars().all()) def compute_next_run(slots: list[ScheduleSlot], *, after: datetime | None = None) -> datetime | None: """Return the next datetime any enabled slot fires, searching 7 days ahead.""" if not slots: return None now = after or _now() best: datetime | None = None for day_offset in range(0, 8): day = now + timedelta(days=day_offset) for slot in slots: if slot.day_of_week != day.weekday(): continue candidate = day.replace( hour=slot.hour, minute=slot.minute, second=0, microsecond=0 ) if candidate <= now: continue if best is None or candidate < best: best = candidate return best class ScraperScheduler: def __init__(self) -> None: self._task: asyncio.Task | None = None self._stop = asyncio.Event() self._last_fired_key: str | None = None # "YYYY-MM-DD-HH-MM" dedup guard self._last_interval_run: datetime | None = None async def _run_once(self) -> None: async with AsyncSessionLocal() as db: await run_all_scrapers(db) async def _loop(self) -> None: await asyncio.sleep(10) # let startup settle while not self._stop.is_set(): try: await self._tick() except Exception as exc: # noqa: BLE001 - loop must survive failures logger.error("Scheduler tick failed: %s", exc) try: # Wake every 30s so minute-precise slots fire promptly. await asyncio.wait_for(self._stop.wait(), timeout=30) except asyncio.TimeoutError: pass async def _tick(self) -> None: slots = await get_enabled_slots() now = _now() if slots: key = now.strftime("%Y-%m-%d-%H-%M") for slot in slots: if ( slot.day_of_week == now.weekday() and slot.hour == now.hour and slot.minute == now.minute and self._last_fired_key != key ): logger.info("Schedule slot matched (%s); running scrapers", key) self._last_fired_key = key await self._run_once() return return # No slots -> interval fallback. interval = timedelta(minutes=max(1, settings.scraper_interval_minutes)) if self._last_interval_run is None or (now - self._last_interval_run) >= interval: logger.info("Interval fallback elapsed; running scrapers") self._last_interval_run = now await self._run_once() def start(self) -> None: if not settings.scraper_enabled: logger.info("Scraper scheduler disabled via SCRAPER_ENABLED=false") return if self._task is None: self._task = asyncio.create_task(self._loop()) logger.info("Scraper scheduler started (tz=%s)", settings.schedule_timezone) async def stop(self) -> None: self._stop.set() if self._task: self._task.cancel() try: await self._task except (asyncio.CancelledError, Exception): # noqa: BLE001 pass scheduler = ScraperScheduler() |