admin / Bid-Sentinel

"""Scraper for Contracts Finder (https://www.contractsfinder.service.gov.uk).

Primary path uses the official OCDS Search API (JSON), which is far more stable
than scraping HTML. A BeautifulSoup-based HTML fallback is included to satisfy
the parsing requirement and to demonstrate graceful degradation.
"""
from __future__ import annotations

import logging

import httpx
from bs4 import BeautifulSoup

from app.scraper.base import (
    TenderRecord,
    cpv_text,
    is_award_or_closed,
    parse_datetime,
    parse_value,
)
from app.scraper.keywords import CYBER_KEYWORDS, match_keyword

logger = logging.getLogger("scraper.contracts_finder")

SOURCE = "Contracts Finder"
API_URL = "https://www.contractsfinder.service.gov.uk/Published/Notices/OCDS/Search"
HEADERS = {"User-Agent": "CyberTenderTracker/1.0 (+https://example.com)", "Accept": "application/json"}

# High-signal umbrella terms issued as server-side queries. Returned notices are
# still filtered locally against the FULL keyword list (built-in + custom), so a
# handful of broad queries gives the same coverage with far fewer HTTP calls than
# searching all ~37 keywords individually. Cyber tenders almost always contain at
# least one of these; the NCSC/specific terms are included so niche notices that
# only mention e.g. "GovAssure" are still surfaced.
PRIMARY_SEARCH_TERMS: list[str] = [
    "Cyber Security",
    "Managed Detection and Response",
    "Security Operations Centre",
    "Incident Response",
    "Vulnerability Management",
    "Cyber Threat Intelligence",
    "Threat Hunting",
    "Penetration Testing",
    "Cyber Essentials Plus",
    "Cyber Assessment Framework",
    "GovAssure",
    "SIEM",
]


def _build_search_terms(full_keywords: list[str]) -> list[str]:
    """Umbrella query terms plus any user-supplied custom keywords.

    Custom keywords are searched server-side too (they won't be covered by the
    umbrella terms), while the built-in long-tail is left to local filtering.
    """
    builtin_lower = {k.lower() for k in CYBER_KEYWORDS}
    terms = list(PRIMARY_SEARCH_TERMS)
    seen = {t.lower() for t in terms}
    for kw in full_keywords:
        if kw.lower() not in builtin_lower and kw.lower() not in seen:
            terms.append(kw)
            seen.add(kw.lower())
    return terms


async def fetch(keywords: list[str] | None = None) -> list[TenderRecord]:
    """Fetch cyber-security notices from Contracts Finder. Never raises."""
    full_keywords = keywords or CYBER_KEYWORDS
    search_terms = _build_search_terms(full_keywords)
    logger.info(
        "Contracts Finder: %d server-side queries (filtering against %d keywords)",
        len(search_terms),
        len(full_keywords),
    )
    records: list[TenderRecord] = []
    async with httpx.AsyncClient(timeout=30.0, headers=HEADERS) as client:
        for term in search_terms:
            try:
                # Results are filtered locally against the FULL keyword list.
                records.extend(await _search_keyword(client, term, full_keywords))
            except Exception as exc:  # noqa: BLE001 - keep scraping other terms
                logger.warning("Contracts Finder search failed for %r: %s", term, exc)
    return records


async def _search_keyword(
    client: httpx.AsyncClient, keyword: str, keywords: list[str]
) -> list[TenderRecord]:
    params = {"keyword": keyword, "size": "50"}
    resp = await client.get(API_URL, params=params)
    resp.raise_for_status()

    try:
        data = resp.json()
    except ValueError:
        logger.warning("Non-JSON response for %r; attempting HTML fallback", keyword)
        return _parse_html(resp.text, keywords)

    results = data.get("results") or data.get("releases") or []
    out: list[TenderRecord] = []
    for item in results:
        rec = _parse_ocds_release(item, keywords)
        if rec:
            out.append(rec)
    return out


def _parse_ocds_release(item: dict, keywords: list[str] | None) -> TenderRecord | None:
    """Map one OCDS release dict to a TenderRecord. Returns None if unusable."""
    try:
        release = item.get("releases", [item])
        release = release[0] if isinstance(release, list) and release else item

        tender = release.get("tender", {}) or {}
        ocid = release.get("ocid") or item.get("ocid")
        title = tender.get("title") or release.get("title")
        description = tender.get("description")

        if not title:
            return None

        # Only keep opportunities currently open for tender — skip award/contract
        # notices and closed/cancelled tenders (closing date is checked later).
        if is_award_or_closed(release, tender):
            return None

        # Keyword relevance is always required; the CPV AND-filter is applied
        # later in the runner using the classification captured below.
        keyword = match_keyword(title, description, keywords=keywords)
        if keyword is None:
            return None  # relevance gate

        notice_id = ocid or release.get("id")
        url = (
            f"https://www.contractsfinder.service.gov.uk/Notice/{notice_id}"
            if notice_id
            else release.get("uri", "")
        )
        if not url:
            return None

        value = (tender.get("value") or {}).get("amount")
        amount, value_text = parse_value(value)
        period = tender.get("contractPeriod") or {}
        duration = _format_period(period)

        buyer = (release.get("buyer") or {}).get("name")
        tender_period = tender.get("tenderPeriod") or {}

        return TenderRecord(
            url=url,
            title=title.strip(),
            source=SOURCE,
            reference=str(notice_id) if notice_id else None,
            description=(description or "").strip() or None,
            published_date=parse_datetime(release.get("date")),
            closing_date=parse_datetime(tender_period.get("endDate")),
            value_amount=amount,
            value_text=value_text,
            currency=(tender.get("value") or {}).get("currency", "GBP"),
            duration=duration,
            buyer=buyer,
            matched_keyword=keyword,
            cpv=cpv_text(tender),
        )
    except Exception as exc:  # noqa: BLE001
        logger.warning("Failed to parse Contracts Finder release: %s", exc)
        return None


def _format_period(period: dict) -> str | None:
    if not period:
        return None
    months = period.get("durationInDays")
    if months:
        return f"{round(months / 30)} months (approx.)"
    start, end = period.get("startDate"), period.get("endDate")
    if start and end:
        return f"{start[:10]} to {end[:10]}"
    return None


def _parse_html(html: str, keywords: list[str] | None = None) -> list[TenderRecord]:
    """BeautifulSoup fallback parser for the public search results page."""
    out: list[TenderRecord] = []
    try:
        soup = BeautifulSoup(html, "lxml")
        for card in soup.select("div.search-result"):
            link = card.find("a", href=True)
            if not link:
                continue
            title = link.get_text(strip=True)
            keyword = match_keyword(title, keywords=keywords)
            if not keyword:
                continue
            href = link["href"]
            url = href if href.startswith("http") else f"https://www.contractsfinder.service.gov.uk{href}"
            out.append(TenderRecord(url=url, title=title, source=SOURCE, matched_keyword=keyword))
    except Exception as exc:  # noqa: BLE001
        logger.warning("HTML fallback parse failed: %s", exc)
    return out