Catalyst / admin/Bid-Sentinel 14.8 GB / 57.8 GB 40.0 GB free
Help Sign in

admin / Bid-Sentinel

public

Bid Scrape and Tracking Application with AI Capability

Code Issues Pull requests Pipelines Packages Security Insights Wiki Settings
Bid-Sentinel / bid-sentinel-v2 / backend / app / scraper / scheduler.py 4645 B · main
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
"""Background scheduler supporting multiple day/time slots.

If one or more enabled ScheduleSlots exist, the scraper runs at each matching
day-of-week + time (interpreted in ``SCHEDULE_TIMEZONE``). If no slots are
defined, it falls back to a fixed interval (``SCRAPER_INTERVAL_MINUTES``).
"""
from __future__ import annotations

import asyncio
import logging
from datetime import datetime, timedelta

from sqlalchemy import select

from app.config import settings
from app.database import AsyncSessionLocal
from app.models import ScheduleSlot
from app.scraper.runner import run_all_scrapers

logger = logging.getLogger("scraper.scheduler")

try:
    from zoneinfo import ZoneInfo

    def _tz():
        try:
            return ZoneInfo(settings.schedule_timezone)
        except Exception:  # noqa: BLE001 - missing tzdata etc.
            return None
except ImportError:  # pragma: no cover
    def _tz():
        return None


def _now() -> datetime:
    tz = _tz()
    return datetime.now(tz) if tz else datetime.now()


async def get_enabled_slots() -> list[ScheduleSlot]:
    async with AsyncSessionLocal() as db:
        result = await db.execute(
            select(ScheduleSlot).where(ScheduleSlot.enabled.is_(True))
        )
        return list(result.scalars().all())


def compute_next_run(slots: list[ScheduleSlot], *, after: datetime | None = None) -> datetime | None:
    """Return the next datetime any enabled slot fires, searching 7 days ahead."""
    if not slots:
        return None
    now = after or _now()
    best: datetime | None = None
    for day_offset in range(0, 8):
        day = now + timedelta(days=day_offset)
        for slot in slots:
            if slot.day_of_week != day.weekday():
                continue
            candidate = day.replace(
                hour=slot.hour, minute=slot.minute, second=0, microsecond=0
            )
            if candidate <= now:
                continue
            if best is None or candidate < best:
                best = candidate
    return best


class ScraperScheduler:
    def __init__(self) -> None:
        self._task: asyncio.Task | None = None
        self._stop = asyncio.Event()
        self._last_fired_key: str | None = None  # "YYYY-MM-DD-HH-MM" dedup guard
        self._last_interval_run: datetime | None = None

    async def _run_once(self) -> None:
        async with AsyncSessionLocal() as db:
            await run_all_scrapers(db)

    async def _loop(self) -> None:
        await asyncio.sleep(10)  # let startup settle
        while not self._stop.is_set():
            try:
                await self._tick()
            except Exception as exc:  # noqa: BLE001 - loop must survive failures
                logger.error("Scheduler tick failed: %s", exc)
            try:
                # Wake every 30s so minute-precise slots fire promptly.
                await asyncio.wait_for(self._stop.wait(), timeout=30)
            except asyncio.TimeoutError:
                pass

    async def _tick(self) -> None:
        slots = await get_enabled_slots()
        now = _now()

        if slots:
            key = now.strftime("%Y-%m-%d-%H-%M")
            for slot in slots:
                if (
                    slot.day_of_week == now.weekday()
                    and slot.hour == now.hour
                    and slot.minute == now.minute
                    and self._last_fired_key != key
                ):
                    logger.info("Schedule slot matched (%s); running scrapers", key)
                    self._last_fired_key = key
                    await self._run_once()
                    return
            return

        # No slots -> interval fallback.
        interval = timedelta(minutes=max(1, settings.scraper_interval_minutes))
        if self._last_interval_run is None or (now - self._last_interval_run) >= interval:
            logger.info("Interval fallback elapsed; running scrapers")
            self._last_interval_run = now
            await self._run_once()

    def start(self) -> None:
        if not settings.scraper_enabled:
            logger.info("Scraper scheduler disabled via SCRAPER_ENABLED=false")
            return
        if self._task is None:
            self._task = asyncio.create_task(self._loop())
            logger.info("Scraper scheduler started (tz=%s)", settings.schedule_timezone)

    async def stop(self) -> None:
        self._stop.set()
        if self._task:
            self._task.cancel()
            try:
                await self._task
            except (asyncio.CancelledError, Exception):  # noqa: BLE001
                pass


scheduler = ScraperScheduler()