admin / Bid-Sentinel

"""Accurate best-effort HTML scraper for portals without a JSON/OCDS API.

Strategy: fetch the portal's search/listing page once (polite), find the result
cards, and extract structured fields from each — title, link, buyer, value, and
published/closing dates — rather than naively harvesting links. Only results
whose text matches the cyber keyword list are kept. Degrades gracefully (returns
[] on any failure) and never aborts the run.

Where a portal exposes a JSON/OCDS API, prefer ``ocds_api`` instead — it's far
more reliable. This module is the fallback for HTML-only sites; the result-card
selectors and ``search_url`` may still need per-site tuning, and auth-gated
portals need credentials before they return anything.
"""
from __future__ import annotations

import logging
import re
from urllib.parse import urljoin

import httpx
from bs4 import BeautifulSoup

from app.scraper.base import TenderRecord, parse_datetime, parse_value
from app.scraper.keywords import match_keyword

logger = logging.getLogger("scraper.generic")

HEADERS = {
    "User-Agent": "CyberTenderTracker/1.0 (+https://example.com)",
    "Accept": "text/html,application/xhtml+xml",
}

_MAX_RESULTS = 60

# Selectors that commonly wrap one search result / notice on procurement portals.
_RESULT_SELECTORS = [
    "[class*=search-result]",
    "[class*=result-item]",
    "[class*=resultitem]",
    "[class*=notice-item]",
    "[class*=opportunity]",
    "[class*=tender-item]",
    "li[class*=result]",
    "article",
]

_DATE = r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{1,2}\s+[A-Za-z]{3,9}\s+\d{4}"
_MONEY_RE = re.compile(r"£\s?[\d,]+(?:\.\d+)?\s?(?:million|billion|bn|m|k)?", re.IGNORECASE)


def _norm(text: str) -> str:
    return " ".join((text or "").split())


def _labelled(text: str, labels: list[str], width: int = 80) -> str | None:
    # Capture up to the next field/sentence boundary (period, pipe, newline, bullet).
    for label in labels:
        m = re.search(rf"{label}\s*[:\-]\s*([^.|\n•;]{{1,{width}}})", text, re.IGNORECASE)
        if m:
            value = _norm(m.group(1)).strip(" .;,-")
            if value:
                return value
    return None


def _labelled_date(text: str, labels: list[str]):
    for label in labels:
        m = re.search(rf"{label}[^0-9]{{0,25}}({_DATE})", text, re.IGNORECASE)
        if m:
            return parse_datetime(m.group(1))
    return None


def _money(text: str):
    m = _MONEY_RE.search(text)
    return parse_value(m.group(0)) if m else (None, None)


def _parse_card(card, base: str, source: str, keywords: list[str] | None) -> TenderRecord | None:
    link = card.find("a", href=True)
    if not link:
        return None
    heading = card.find(["h1", "h2", "h3", "h4"])
    title = _norm((heading.get_text(" ") if heading else "") or link.get_text(" "))
    if len(title) < 10:
        return None

    text = _norm(card.get_text(" "))
    matched = match_keyword(title, text, keywords=keywords)
    if matched is None:
        return None

    url = urljoin(base, link["href"])
    if not url.startswith("http"):
        return None

    amount, value_text = _money(text)
    return TenderRecord(
        url=url,
        title=title[:1024],
        source=source,
        buyer=_labelled(text, ["buyer", "contracting authority", "authority", "organisation", "organization"]),
        reference=_labelled(text, ["reference", "notice id", "ref no", "ref"], width=40),
        value_amount=amount,
        value_text=value_text,
        published_date=_labelled_date(text, ["published", "date published", "publication"]),
        closing_date=_labelled_date(text, ["closing", "deadline", "closes", "closing date"]),
        matched_keyword=matched,
    )


def make_scraper(source_name: str, search_url: str, query: dict | None = None):
    """Build an ``async fetch(keywords)`` for a portal's HTML search/listing page."""

    async def fetch(keywords: list[str] | None = None) -> list[TenderRecord]:
        out: list[TenderRecord] = []
        seen: set[str] = set()
        try:
            async with httpx.AsyncClient(timeout=30.0, headers=HEADERS, follow_redirects=True) as client:
                resp = await client.get(search_url, params=query or None)
                resp.raise_for_status()
                soup = BeautifulSoup(resp.text, "lxml")
                base = str(resp.url)

                # 1. Try to find structured result cards.
                cards = []
                for selector in _RESULT_SELECTORS:
                    cards = soup.select(selector)
                    if cards:
                        break

                def add(rec: TenderRecord | None) -> None:
                    if rec and rec.url not in seen:
                        seen.add(rec.url)
                        out.append(rec)

                if cards:
                    for card in cards:
                        if len(out) >= _MAX_RESULTS:
                            break
                        add(_parse_card(card, base, source_name, keywords))

                # 2. Fallback: harvest keyword-matching links (still records source).
                if not out:
                    for anchor in soup.find_all("a", href=True):
                        if len(out) >= _MAX_RESULTS:
                            break
                        title = _norm(anchor.get_text(" "))
                        if len(title) < 12:
                            continue
                        matched = match_keyword(title, keywords=keywords)
                        if not matched:
                            continue
                        url = urljoin(base, anchor["href"])
                        if not url.startswith("http"):
                            continue
                        add(TenderRecord(url=url, title=title[:1024], source=source_name, matched_keyword=matched))
        except Exception as exc:  # noqa: BLE001 - a portal must never abort the run
            logger.info("%s: generic scrape returned no results (%s)", source_name, exc)
        logger.info("%s: HTML scraper found %d candidate(s)", source_name, len(out))
        return out

    return fetch