admin / Bid-Sentinel

"""Canonical registry of UK public and private sector Bid Portals checked by Bid Sentinel.

Each portal maps to a ``fetch`` coroutine with the signature::

    async def fetch(keywords: list[str]) -> list[TenderRecord]

Every portal ships with an active scraper (``live=True``). Mechanism per portal,
**JSON/OCDS API preferred over HTML**:

* Find a Tender and Contracts Finder use the official OCDS JSON APIs — robust.
* Any other portal that exposes an OCDS feed can use ``ocds_api.make_scraper(
  name, api_url)`` — just set its endpoint here.
* Portals with no JSON API use the accurate HTML scraper (``generic_html``),
  which parses structured result cards (title/buyer/value/dates). Because each
  site's markup differs (and several require an authenticated session), the
  ``search_url`` / selectors may need per-site tuning before they return results
  reliably. Switch any of these to ``ocds_api`` the moment a JSON endpoint is
  confirmed.
"""
from __future__ import annotations

import logging
from dataclasses import dataclass
from typing import Awaitable, Callable

from app.scraper import contracts_finder, find_a_tender, generic_html, ocds_api  # noqa: F401
from app.scraper.base import TenderRecord

logger = logging.getLogger("scraper.portals")

FetchFn = Callable[[list[str]], Awaitable[list[TenderRecord]]]


@dataclass(frozen=True)
class PortalDef:
    key: str
    name: str
    url: str
    scope: str
    fetch: FetchFn
    live: bool = False


# Order here is the display order in the UI.
PORTALS: list[PortalDef] = [
    PortalDef(
        key="find_a_tender",
        name="Find a Tender Service (FTS)",
        url="https://www.find-tender.service.gov.uk/",
        scope="Central digital platform for all above-threshold UK public sector procurement.",
        fetch=find_a_tender.fetch,
        live=True,
    ),
    PortalDef(
        key="contracts_finder",
        name="Contracts Finder",
        url="https://www.contractsfinder.service.gov.uk/Search/Results",
        scope="Mandatory portal for below-threshold contracts in England and wider public sector.",
        fetch=contracts_finder.fetch,
        live=True,
    ),
    PortalDef(
        key="sell2wales",
        name="Sell2Wales",
        url="https://www.sell2wales.gov.wales/",
        scope="Primary procurement portal for all Welsh public bodies.",
        fetch=generic_html.make_scraper(
            "Sell2Wales", "https://www.sell2wales.gov.wales/search/Search.aspx"
        ),
        live=True,
    ),
    PortalDef(
        key="public_contracts_scotland",
        name="Public Contracts Scotland (PCS)",
        url="https://www.publiccontractsscotland.gov.uk/",
        scope="Primary source for all Scottish public sector tenders.",
        fetch=generic_html.make_scraper(
            "Public Contracts Scotland", "https://www.publiccontractsscotland.gov.uk/search/Search.aspx"
        ),
        live=True,
    ),
    PortalDef(
        key="etendersni",
        name="eTendersNI",
        url="https://etendersni.gov.uk/",
        scope="Primary eProcurement platform for Northern Ireland.",
        fetch=generic_html.make_scraper("eTendersNI", "https://etendersni.gov.uk/epps/quickSearchAction.do"),
        live=True,
    ),
    PortalDef(
        key="procontract",
        name="ProContract (Due North)",
        url="https://procontract.due-north.com/",
        scope="Used by regional councils and local authorities across the UK.",
        fetch=generic_html.make_scraper(
            "ProContract (Due North)", "https://procontract.due-north.com/Opportunities/Index"
        ),
        live=True,
    ),
    PortalDef(
        key="delta_esourcing",
        name="Delta eSourcing",
        url="https://www.delta-esourcing.com/",
        scope="Common platform used by various contracting authorities to manage bids.",
        fetch=generic_html.make_scraper(
            "Delta eSourcing", "https://www.delta-esourcing.com/tenders/UK-UK-Opportunities"
        ),
        live=True,
    ),
    PortalDef(
        key="in_tend",
        name="In-Tend",
        url="https://in-tendhost.co.uk/",
        scope="Widely used across the public sector via localized portal instances.",
        fetch=generic_html.make_scraper("In-Tend", "https://in-tendhost.co.uk/"),
        live=True,
    ),
    PortalDef(
        key="achilles",
        name="Achilles",
        url="https://www.achilles.com/",
        scope="Supply chain risk/procurement network for utilities, transport and infrastructure.",
        fetch=generic_html.make_scraper("Achilles", "https://www.achilles.com/find-tenders/"),
        live=True,
    ),
    PortalDef(
        key="ccs_gca",
        name="Crown Commercial Service / Government Commercial Agency",
        url="https://www.gca.gov.uk/",
        scope="Largest UK public procurement organization; major central government frameworks.",
        fetch=generic_html.make_scraper("CCS / Government Commercial Agency", "https://www.gca.gov.uk/"),
        live=True,
    ),
    PortalDef(
        key="digital_marketplace",
        name="Digital Marketplace (G-Cloud)",
        url="https://www.applytosupply.digitalmarketplace.service.gov.uk/",
        scope="Cloud and digital services frameworks (G-Cloud / Digital Outcomes).",
        fetch=generic_html.make_scraper(
            "Digital Marketplace (G-Cloud)",
            "https://www.applytosupply.digitalmarketplace.service.gov.uk/g-cloud/search",
        ),
        live=True,
    ),
]

PORTALS_BY_KEY: dict[str, PortalDef] = {p.key: p for p in PORTALS}