admin / Bid-Sentinel

"""Orchestrates the enabled portals and persists results with dedup."""
from __future__ import annotations

import logging
from datetime import datetime, timezone

from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession

from app import ai, analysis
from app.appsettings import get_ai_enabled, get_cpv_strict
from app.models import (
    CpvCode,
    CustomKeyword,
    Portal,
    SuppressedOpportunity,
    Tender,
    TuneOutTerm,
)
from app.scraper.base import TenderRecord, normalize_cpv
from app.scraper.keywords import CYBER_KEYWORDS, matches_tuneout, passes_cpv_filter
from app.scraper.portals import PORTALS_BY_KEY
from app.scraper.sample_data import sample_records

logger = logging.getLogger("scraper.runner")


def _is_closed(closing_date, now: datetime) -> bool:
    """True if a closing date is in the past (opportunity no longer open).

    Records with no closing date are treated as open (kept) — we can't tell, and
    dropping them would lose opportunities from sources that omit the date.
    """
    if closing_date is None:
        return False
    dt = closing_date
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt < now


async def get_active_keywords(db: AsyncSession) -> list[str]:
    """Built-in cyber keywords plus any user-supplied custom keywords."""
    result = await db.execute(select(CustomKeyword.keyword).order_by(CustomKeyword.id))
    custom = [k for (k,) in result.all()]
    # Preserve order, drop case-insensitive duplicates.
    seen, combined = set(), []
    for kw in [*CYBER_KEYWORDS, *custom]:
        if kw.lower() not in seen:
            seen.add(kw.lower())
            combined.append(kw)
    return combined


async def get_cpv_terms(db: AsyncSession) -> list[str]:
    """User CPV codes and their descriptions, as extra match terms.

    Each code (normalised) and its description are strong relevance signals: a
    notice classified with — or mentioning — one is treated as a match.
    """
    result = await db.execute(
        select(CpvCode.code, CpvCode.description)
        .where(CpvCode.enabled.is_(True))
        .order_by(CpvCode.id)
    )
    terms: list[str] = []
    seen: set[str] = set()
    for code, description in result.all():
        for term in (normalize_cpv(code), (description or "").strip()):
            key = term.lower()
            if term and key not in seen:
                seen.add(key)
                terms.append(term)
    return terms


async def get_tuneout_terms(db: AsyncSession) -> list[str]:
    """User 'tune-out' terms — opportunities matching one are suppressed (unless
    they are cyber-security related, which are always kept)."""
    result = await db.execute(select(TuneOutTerm.term).order_by(TuneOutTerm.id))
    return [t for (t,) in result.all()]


async def get_suppressed_keys(db: AsyncSession) -> tuple[set[str], set[str]]:
    """URLs and references of opportunities the user bulk-deleted. These must
    never be scraped back in. Returns (urls, references)."""
    result = await db.execute(select(SuppressedOpportunity.url, SuppressedOpportunity.reference))
    urls: set[str] = set()
    refs: set[str] = set()
    for url, ref in result.all():
        if url:
            urls.add(url)
        if ref:
            refs.add(ref)
    return urls, refs


async def _enabled_portals(db: AsyncSession) -> list[Portal]:
    result = await db.execute(select(Portal).where(Portal.enabled.is_(True)).order_by(Portal.id))
    return list(result.scalars().all())


async def run_all_scrapers(db: AsyncSession, *, use_sample_fallback: bool = True) -> dict:
    """Run every *enabled* portal, dedupe, and insert new tenders.

    Returns a summary dict: {inserted, skipped, sources, keywords, message}.
    """
    keywords = await get_active_keywords(db)
    cpv_terms = await get_cpv_terms(db)
    portals = await _enabled_portals(db)

    cpv_strict = await get_cpv_strict(db)

    all_records: list[TenderRecord] = []
    sources: list[str] = []
    # Per-portal diagnostics: candidates fetched, or an error string.
    portal_stats: list[dict] = []

    for portal in portals:
        portal_def = PORTALS_BY_KEY.get(portal.key)
        if portal_def is None:
            # Custom (user-added) portal: registered and selectable, but no
            # automated scraper. Surface it so it's clear why it returns nothing.
            portal_stats.append({"name": portal.name, "candidates": 0, "note": "no scraper (custom)"})
            continue
        try:
            # Scrapers filter on keywords (cyber relevance). CPV codes are applied
            # as an AND-filter afterwards (see _persist).
            recs = await portal_def.fetch(keywords)
            logger.info("%s returned %d candidate records", portal.name, len(recs))
            all_records.extend(recs)
            sources.append(portal.name)
            portal_stats.append({"name": portal.name, "candidates": len(recs), "note": ""})
        except Exception as exc:  # noqa: BLE001 - one portal must not abort the run
            logger.error("Portal %s crashed: %s", portal.name, exc)
            portal_stats.append({"name": portal.name, "candidates": 0, "note": f"error: {exc}"})

    # If no live portal returned anything (e.g. offline demo), seed sample data.
    has_live_enabled = any(PORTALS_BY_KEY.get(p.key, None) and PORTALS_BY_KEY[p.key].live for p in portals)
    used_sample = False
    if not all_records and use_sample_fallback and has_live_enabled:
        logger.warning("No live records fetched; using sample data fallback")
        all_records = sample_records()
        sources.append("Sample Data")
        used_sample = True

    capability_terms = await analysis.get_capability_terms(db)
    use_ai = await get_ai_enabled(db)
    tuneout_terms = await get_tuneout_terms(db)
    suppressed_urls, suppressed_refs = await get_suppressed_keys(db)
    breakdown = await _persist(
        db,
        all_records,
        capability_terms,
        use_ai,
        tuneout_terms,
        cpv_terms,
        suppressed_urls,
        suppressed_refs,
        cpv_strict,
    )
    inserted = breakdown["inserted"]
    skipped = sum(v for k, v in breakdown.items() if k != "inserted")
    candidates = len(all_records)

    # Human-readable multi-line summary with the diagnostics.
    portal_line = ", ".join(
        f"{p['name']}: {p['candidates']}" + (f" ({p['note']})" if p["note"] else "")
        for p in portal_stats
    )
    skip_bits = [
        f"{breakdown['duplicate']} already tracked",
        f"{breakdown['closed']} closed/expired",
        f"{breakdown['cpv_filtered']} CPV-filtered",
        f"{breakdown['tuned_out']} tuned-out",
        f"{breakdown['suppressed']} suppressed",
    ]
    if breakdown["error"]:
        skip_bits.append(f"{breakdown['error']} errors")
    cpv_note = ""
    if cpv_terms:
        cpv_note = f" · CPV filter {'strict' if cpv_strict else 'lenient'} ({len(cpv_terms)} code(s))"
    msg = (
        f"Scrape complete: {inserted} new, {skipped} skipped of {candidates} candidate(s). "
        f"Skipped — {', '.join(skip_bits)}.{cpv_note}"
        + (" · showing sample data (no live results)" if used_sample else "")
        + (f" · Per portal — {portal_line}" if portal_line else "")
    )
    logger.info(msg)
    return {
        "inserted": inserted,
        "skipped": skipped,
        "sources": sources,
        "keywords": keywords,
        "message": msg,
        "candidates": candidates,
        "used_sample": used_sample,
        "portals": portal_stats,
        "skipped_breakdown": breakdown,
    }


async def _persist(
    db: AsyncSession,
    records: list[TenderRecord],
    capability_terms: list[str],
    use_ai: bool,
    tuneout_terms: list[str] | None = None,
    cpv_terms: list[str] | None = None,
    suppressed_urls: set[str] | None = None,
    suppressed_refs: set[str] | None = None,
    cpv_strict: bool = False,
) -> dict:
    """Insert new records; returns a breakdown of what happened to each candidate:
    {inserted, duplicate, cpv_filtered, tuned_out, suppressed, error}."""
    tuneout_terms = tuneout_terms or []
    cpv_terms = cpv_terms or []
    suppressed_urls = suppressed_urls or set()
    suppressed_refs = suppressed_refs or set()
    b = {
        "inserted": 0, "duplicate": 0, "closed": 0,
        "cpv_filtered": 0, "tuned_out": 0, "suppressed": 0, "error": 0,
    }
    now = datetime.now(timezone.utc)

    # In-batch dedup on url/reference before touching the DB.
    seen_urls: set[str] = set()
    seen_refs: set[str] = set()

    for rec in records:
        if not rec.url or rec.url in seen_urls:
            b["duplicate"] += 1
            continue
        if rec.reference and rec.reference in seen_refs:
            b["duplicate"] += 1
            continue

        # Only keep opportunities still open — drop any whose closing date has passed.
        if _is_closed(rec.closing_date, now):
            b["closed"] += 1
            continue

        # Suppression: never re-scrape opportunities the user bulk-deleted.
        if rec.url in suppressed_urls or (rec.reference and rec.reference in suppressed_refs):
            b["suppressed"] += 1
            continue

        # CPV AND-filter: when CPV codes are configured, keep only notices that also
        # match a CPV code. In lenient mode, notices that declare no CPV data are
        # kept (keyword relevance already applied); strict mode requires a match.
        if not passes_cpv_filter(
            rec.title, rec.description, rec.cpv, cpv_terms=cpv_terms, strict=cpv_strict
        ):
            b["cpv_filtered"] += 1
            continue

        # Tune-out filter: drop non-cyber noise the user has chosen to hide.
        # Cyber-security-related opportunities are never dropped (guardrail).
        if matches_tuneout(rec.title, rec.description, tuneout_terms):
            b["tuned_out"] += 1
            continue

        # DB-level dedup: skip if url or reference already exists.
        stmt = select(Tender.id).where(Tender.url == rec.url)
        if rec.reference:
            stmt = select(Tender.id).where(
                (Tender.url == rec.url) | (Tender.reference == rec.reference)
            )
        existing = (await db.execute(stmt)).first()
        if existing:
            b["duplicate"] += 1
            continue

        fields = await ai.analyze_opportunity(
            f"{rec.title}\n{rec.description or ''}", capability_terms, use_ai
        )
        tender = Tender(
            reference=rec.reference,
            url=rec.url,
            title=rec.title[:1024],
            description=rec.description,
            published_date=rec.published_date,
            closing_date=rec.closing_date,
            value_amount=rec.value_amount,
            value_text=rec.value_text,
            currency=rec.currency or "GBP",
            duration=rec.duration,
            buyer=rec.buyer,
            source=rec.source,
            matched_keyword=rec.matched_keyword,
            required_certs=fields["required_certs"],
            fit_score=fields["fit_score"],
            fit_matched=fields["fit_matched"],
            tech_requirements=fields["tech_requirements"],
            ai_summary=fields.get("ai_summary", ""),
        )
        db.add(tender)
        try:
            await db.flush()  # surface unique violations per-row
        except IntegrityError:
            await db.rollback()
            b["error"] += 1
            continue

        seen_urls.add(rec.url)
        if rec.reference:
            seen_refs.add(rec.reference)
        b["inserted"] += 1

    await db.commit()
    return b