admin / Bid-Sentinel

"""Deterministic (non-AI) analysis: certification detection + capability fit.

No external services, no API keys — just curated lexicons and word-boundary
matching. Fully explainable and offline.
"""
from __future__ import annotations

import io
import logging
import re

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.models import CapabilityTerm
from app.scraper.keywords import CYBER_KEYWORDS

logger = logging.getLogger("analysis")

# ---------------------------------------------------------------------------
# Certification / accreditation lexicon
# Each entry: (canonical name, [lowercased aliases matched with word boundaries])
# Extend this list to widen coverage — same idea as the keyword list.
# ---------------------------------------------------------------------------
CERT_LEXICON: list[tuple[str, list[str]]] = [
    ("Cyber Essentials Plus", ["cyber essentials plus", "cyber essentials+", "ce plus"]),
    ("Cyber Essentials", ["cyber essentials"]),
    ("ISO 27001", ["iso 27001", "iso/iec 27001", "iso27001", "iso 27001:2022"]),
    ("ISO 9001", ["iso 9001"]),
    ("ISO 22301", ["iso 22301"]),
    ("SOC 2", ["soc 2", "soc2", "soc ii"]),
    ("PCI DSS", ["pci dss", "pci-dss"]),
    ("IASME", ["iasme"]),
    ("CREST", ["crest"]),
    ("CHECK (NCSC)", ["check provider", "ncsc check", "check penetration", "check-certified"]),
    ("NCSC", ["ncsc"]),
    ("Cyber Assessment Framework (CAF)", ["cyber assessment framework", "caf"]),
    ("GovAssure", ["govassure"]),
    ("NIST", ["nist"]),
    ("SC Clearance", ["sc clearance", "sc cleared", "sc-cleared", "security check clearance"]),
    ("DV Clearance", ["dv clearance", "developed vetting", "dv cleared"]),
    ("BPSS", ["bpss", "baseline personnel security standard"]),
    ("NPPV", ["nppv"]),
    ("List X", ["list x"]),
    ("JOSCAR", ["joscar"]),
    ("Constructionline", ["constructionline"]),
]

# Drop the base cert when a more specific one is present (avoids double-listing).
_SUPPRESS = {"Cyber Essentials": "Cyber Essentials Plus"}

CERT_NAMES = [name for name, _ in CERT_LEXICON]


def _present(alias: str, haystack: str) -> bool:
    return re.search(rf"\b{re.escape(alias)}\b", haystack) is not None


def detect_certs(text: str | None) -> list[str]:
    """Return the certifications/accreditations mentioned in the text."""
    haystack = (text or "").lower()
    if not haystack:
        return []
    found: list[str] = []
    for canonical, aliases in CERT_LEXICON:
        if any(_present(a, haystack) for a in aliases):
            found.append(canonical)
    return [c for c in found if not (c in _SUPPRESS and _SUPPRESS[c] in found)]


def find_terms(text: str | None, terms: list[str]) -> list[str]:
    """Return every term that appears in the text (word-boundary matched)."""
    haystack = (text or "").lower()
    if not haystack:
        return []
    out: list[str] = []
    for term in terms:
        t = term.lower().strip()
        if t and re.search(rf"\b{re.escape(t)}\b", haystack):
            out.append(term)
    return out


def score_fit(text: str | None, capability_terms: list[str]) -> tuple[int | None, list[str]]:
    """Deterministic fit: share of your capability terms found in the text.

    Returns (score 0-100 or None if no profile defined, matched terms).
    """
    if not capability_terms:
        return None, []
    matched = find_terms(text, capability_terms)
    score = round(100 * len(matched) / len(capability_terms))
    return min(100, score), matched


# Phrases that signal a requirement, and technical terms to look for.
_REQ_CUES = [
    "must", "shall", "required", "require", "essential", "mandatory", "minimum of",
    "experience", "expertise", "knowledge of", "proficien", "demonstrable",
    "responsible for", "ability to", "capable of", "deliver", "provide", "maintain",
]
_TECH_HINTS = [t.lower() for t in CYBER_KEYWORDS] + [
    "siem", "edr", "xdr", "ndr", "firewall", "cloud", "azure", "aws", "gcp",
    "network", "endpoint", "vulnerabilit", "penetration", "encryption", "identity",
    "iam", "zero trust", "threat", "malware", "monitoring", "logging", "incident",
    "forensic", "compliance", "iso 27001", "patch", "backup", "resilience",
    "architecture", "siem/soar", "soar", "dlp", "active directory",
]


def extract_tech_requirements(text: str | None, limit: int = 5) -> list[str]:
    """Pull the top requirement-like sentences from an opportunity description.

    Deterministic: splits into fragments, keeps those containing a requirement
    cue and/or a technical term, ranks by signal, returns the top N.
    """
    if not text:
        return []
    parts = re.split(r"[\n\r•]+|(?<=[.;])\s+", text)
    seen: set[str] = set()
    scored: list[tuple[int, str]] = []
    for part in parts:
        s = part.strip(" -–—•\t")
        if not (20 <= len(s) <= 240):
            continue
        low = s.lower()
        cues = sum(1 for c in _REQ_CUES if c in low)
        tech = sum(1 for t in _TECH_HINTS if t in low)
        # Must reference something technical to count as a "technical requirement".
        if tech == 0:
            continue
        key = low[:60]
        if key in seen:
            continue
        seen.add(key)
        scored.append((cues * 2 + tech, s))
    scored.sort(key=lambda x: -x[0])
    return [s for _, s in scored[:limit]]


def analyze(text: str | None, capability_terms: list[str]) -> dict:
    """Compute the persisted analysis fields for one opportunity."""
    certs = detect_certs(text)
    score, matched = score_fit(text, capability_terms)
    return {
        "required_certs": ", ".join(certs),
        "fit_score": score,
        "fit_matched": ", ".join(matched),
        "tech_requirements": "\n".join(extract_tech_requirements(text)),
    }


# Words that shouldn't start (or wholly make up) a suggested capability phrase.
_CAP_STOP = {
    "the", "we", "our", "us", "this", "that", "these", "those", "it", "they",
    "you", "your", "for", "and", "or", "with", "to", "of", "in", "on", "as",
    "at", "by", "an", "a", "is", "are", "be", "will", "shall", "must", "can",
    "may", "all", "any", "more", "about", "contact", "overview", "introduction",
    "summary", "company", "ltd", "limited", "team", "client", "clients",
    "customer", "customers", "approach", "key", "section", "page", "who", "what",
    "why", "how", "when", "where", "please", "note", "important",
}


def _clean_phrase(p: str) -> str:
    return p.strip(" \t\r\n.,;:•-–—()[]\"'").strip()


def suggest_capability_terms(text: str | None, limit: int = 25) -> list[str]:
    """Generate candidate capability terms FROM an uploaded service document.

    Deterministic keyphrase extraction: known cyber terms found in the document,
    plus Title-Case service phrases and technical acronyms it contains. The user
    confirms which to add, so over-generation is fine.
    """
    if not text:
        return []

    scores: dict[str, list] = {}  # key(lower) -> [display, score]

    def add(display: str, pts: int) -> None:
        disp = _clean_phrase(display)
        if not disp:
            return
        key = disp.lower()
        if key in scores:
            scores[key][1] += pts
            if len(disp) > len(scores[key][0]):
                scores[key][0] = disp
        else:
            scores[key] = [disp, pts]

    # 1. Known cyber service terms present in the document (high confidence).
    for term in find_terms(text, CYBER_KEYWORDS):
        add(term, 6)

    # 2. Title-Case phrases and acronyms (likely service names). The token class
    # excludes '.', and joins only with spaces/tabs, so phrases don't swallow
    # sentence-ending periods or run across line breaks.
    for match in re.finditer(r"[A-Z][A-Za-z0-9&/+-]*(?:[ \t]+[A-Z][A-Za-z0-9&/+-]*){0,3}", text):
        phrase = _clean_phrase(match.group(0))
        words = phrase.split()
        if not words or not (3 <= len(phrase) <= 45):
            continue
        if words[0].lower() in _CAP_STOP or all(w.lower() in _CAP_STOP for w in words):
            continue
        low = phrase.lower()
        tech = sum(1 for h in _TECH_HINTS if h in low)
        multiword = len(words) >= 2
        if not (multiword or tech):  # skip single, non-technical words
            continue
        add(phrase, 1 + (len(words) - 1) + (3 if tech else 0))

    ranked = sorted(scores.values(), key=lambda x: -x[1])
    return [display for display, _ in ranked[:limit]]


def extract_text(filename: str, data: bytes) -> str:
    """Best-effort text extraction from PDF / DOCX / plain text. Never raises."""
    name = (filename or "").lower()
    try:
        if name.endswith(".pdf"):
            from pypdf import PdfReader

            reader = PdfReader(io.BytesIO(data))
            return "\n".join((page.extract_text() or "") for page in reader.pages)
        if name.endswith(".docx"):
            from docx import Document

            doc = Document(io.BytesIO(data))
            return "\n".join(p.text for p in doc.paragraphs)
        return data.decode("utf-8", errors="ignore")
    except Exception as exc:  # noqa: BLE001 - extraction must never crash the upload
        logger.warning("Text extraction failed for %r: %s", filename, exc)
        return ""


async def get_capability_terms(db: AsyncSession) -> list[str]:
    result = await db.execute(select(CapabilityTerm.term).order_by(CapabilityTerm.id))
    return [t for (t,) in result.all()]