#!/usr/bin/env python3
"""SEO Crawler Adapter — bridges the OurSEO async crawler stack into reference-curator.

Uses BaseAsyncClient (rate limiting, concurrency, retries) and PageAnalyzer
(full HTML parsing, metadata extraction) from the SEO toolkit to provide a
production-grade crawling backend for reference curation.

Outputs .raw.md files with YAML frontmatter compatible with --save-raw / --no-distill.

Usage:
    # Crawl a list of URLs
    python seo_crawler_adapter.py crawl \
        --urls "https://docs.example.com/page1,https://docs.example.com/page2" \
        --output ~/Documents/reference-library/topic-slug/raw/

    # Crawl from sitemap
    python seo_crawler_adapter.py crawl-sitemap \
        --sitemap https://docs.example.com/sitemap.xml \
        --output ~/Documents/reference-library/topic-slug/raw/ \
        --max-pages 50

    # Crawl from manifest file
    python seo_crawler_adapter.py crawl-manifest \
        --manifest ./manifest.json \
        --output ~/Documents/reference-library/topic-slug/raw/

    # Discover URLs from a root page (link extraction)
    python seo_crawler_adapter.py discover \
        --url https://docs.example.com \
        --max-depth 2 --same-domain
"""

import asyncio
import json
import logging
import re
import sys
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import urljoin, urlparse

import click
import requests
from bs4 import BeautifulSoup
from rich.console import Console
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskID
from rich.table import Table

# Import from SEO toolkit (shared base_client)
SEO_SCRIPTS = Path(__file__).resolve().parents[4] / "12-seo-technical-audit" / "code" / "scripts"
if str(SEO_SCRIPTS) not in sys.path:
    sys.path.insert(0, str(SEO_SCRIPTS))

from base_client import BaseAsyncClient, RateLimiter  # noqa: E402
from page_analyzer import PageAnalyzer, PageMetadata  # noqa: E402

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
console = Console()

# Progress tracking directory
PROGRESS_DIR = Path.home() / ".claude" / "reference-curator-progress"
PROGRESS_DIR.mkdir(parents=True, exist_ok=True)


@dataclass
class CrawlProgress:
    """Track crawl progress for reference curation."""
    run_id: str = ""
    total_urls: int = 0
    processed: int = 0
    succeeded: int = 0
    failed: int = 0
    skipped: int = 0
    start_time: datetime = field(default_factory=datetime.now)
    current_url: str = ""
    status: str = "running"

    def save(self) -> Path:
        filepath = PROGRESS_DIR / f"{self.run_id}.json"
        filepath.write_text(json.dumps({
            "run_id": self.run_id,
            "status": self.status,
            "total_urls": self.total_urls,
            "processed": self.processed,
            "succeeded": self.succeeded,
            "failed": self.failed,
            "skipped": self.skipped,
            "progress_pct": round(self.processed / max(self.total_urls, 1) * 100, 1),
            "current_url": self.current_url,
            "start_time": self.start_time.isoformat(),
            "updated_at": datetime.now().isoformat(),
        }, indent=2))
        return filepath


def _slugify(text: str) -> str:
    """Convert text to URL-friendly slug."""
    text = text.lower().strip()
    text = re.sub(r'[^\w\s-]', '', text)
    text = re.sub(r'[-\s]+', '-', text)
    return text[:60].rstrip('-')


def _extract_main_content(html: str, url: str) -> str:
    """Extract main content as markdown from HTML.

    Uses BeautifulSoup to strip nav/footer/sidebar, then converts
    remaining content to clean text with heading structure preserved.
    """
    soup = BeautifulSoup(html, "html.parser")

    # Remove non-content elements
    for tag in soup(["script", "style", "noscript", "nav", "footer",
                     "header", "aside", "iframe"]):
        tag.decompose()

    # Try to find main content area
    main = (soup.find("main") or soup.find("article") or
            soup.find(role="main") or soup.find(id=re.compile(r"content|main", re.I)) or
            soup.find(class_=re.compile(r"content|main|article", re.I)) or soup.body or soup)

    lines = []
    for element in main.descendants:
        if element.name and element.name.startswith('h') and len(element.name) == 2:
            level = int(element.name[1])
            text = element.get_text(strip=True)
            if text:
                lines.append(f"\n{'#' * level} {text}\n")
        elif element.name == 'p':
            text = element.get_text(strip=True)
            if text:
                lines.append(f"\n{text}\n")
        elif element.name == 'li':
            text = element.get_text(strip=True)
            if text:
                lines.append(f"- {text}")
        elif element.name == 'pre':
            code = element.get_text()
            lang = ""
            code_tag = element.find("code")
            if code_tag:
                classes = code_tag.get("class", [])
                for cls in classes:
                    if cls.startswith("language-"):
                        lang = cls[9:]
                        break
            lines.append(f"\n```{lang}\n{code}\n```\n")
        elif element.name == 'table':
            lines.append(_table_to_markdown(element))

    return "\n".join(lines).strip()


def _table_to_markdown(table_tag) -> str:
    """Convert an HTML table to markdown."""
    rows = table_tag.find_all("tr")
    if not rows:
        return ""

    md_rows = []
    for i, row in enumerate(rows):
        cells = row.find_all(["th", "td"])
        cell_texts = [c.get_text(strip=True).replace("|", "\\|") for c in cells]
        md_rows.append("| " + " | ".join(cell_texts) + " |")
        if i == 0:
            md_rows.append("| " + " | ".join(["---"] * len(cell_texts)) + " |")

    return "\n" + "\n".join(md_rows) + "\n"


def _write_raw_file(
    output_dir: Path,
    index: int,
    url: str,
    content: str,
    metadata: PageMetadata | None = None,
) -> Path:
    """Write a single .raw.md file with YAML frontmatter."""
    parsed = urlparse(url)
    slug = _slugify(parsed.path.rstrip('/').split('/')[-1] or parsed.netloc)
    filename = f"{index:02d}-{slug}.raw.md"
    filepath = output_dir / filename

    frontmatter_fields = {
        "source_url": url,
        "crawled_at": datetime.now().isoformat(),
        "crawler": "seo-aiohttp",
        "content_length": len(content),
        "raw": True,
    }

    if metadata:
        frontmatter_fields.update({
            "status_code": metadata.status_code,
            "title": metadata.title or "",
            "word_count": metadata.word_count,
            "internal_links": metadata.internal_link_count,
            "external_links": metadata.external_link_count,
            "h1": metadata.h1_text or "",
            "schema_types": metadata.schema_types_found,
            "response_time_ms": round(metadata.response_time_ms, 1),
        })

    # Build YAML frontmatter
    fm_lines = ["---"]
    for key, value in frontmatter_fields.items():
        if isinstance(value, bool):
            fm_lines.append(f"{key}: {'true' if value else 'false'}")
        elif isinstance(value, list):
            fm_lines.append(f"{key}: {json.dumps(value)}")
        elif isinstance(value, str) and (':' in value or '"' in value):
            fm_lines.append(f'{key}: "{value}"')
        else:
            fm_lines.append(f"{key}: {value}")
    fm_lines.append("---")

    filepath.write_text("\n".join(fm_lines) + "\n\n" + content, encoding="utf-8")
    return filepath


def discover_urls(root_url: str, max_depth: int = 2, same_domain: bool = True,
                  max_urls: int = 100) -> list[str]:
    """Discover URLs from a root page by following links.

    Uses PageAnalyzer to extract internal links, then follows them
    breadth-first up to max_depth.
    """
    analyzer = PageAnalyzer(timeout=15)
    parsed_root = urlparse(root_url)
    root_domain = parsed_root.netloc.lower()

    visited = set()
    to_visit = [(root_url, 0)]
    discovered = []

    while to_visit and len(discovered) < max_urls:
        url, depth = to_visit.pop(0)

        if url in visited:
            continue
        visited.add(url)

        try:
            metadata = analyzer.analyze_url(url)
            if metadata.status_code == 200:
                discovered.append(url)
                logger.info(f"[{len(discovered)}/{max_urls}] {url}")

                if depth < max_depth:
                    for link in metadata.internal_links:
                        link_domain = urlparse(link.url).netloc.lower()
                        if same_domain and link_domain != root_domain:
                            continue
                        if link.url not in visited:
                            to_visit.append((link.url, depth + 1))
        except Exception as e:
            logger.warning(f"Skip {url}: {e}")

    return discovered


def discover_from_sitemap(sitemap_url: str, max_urls: int = 100) -> list[str]:
    """Extract URLs from an XML sitemap."""
    urls = []
    try:
        resp = requests.get(sitemap_url, timeout=15,
                            headers={"User-Agent": "ReferenceBot/1.0"})
        resp.raise_for_status()

        root = ET.fromstring(resp.content)
        ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}

        # Check if it's a sitemap index
        sitemaps = root.findall("sm:sitemap/sm:loc", ns)
        if sitemaps:
            for sm in sitemaps[:5]:
                urls.extend(discover_from_sitemap(sm.text, max_urls - len(urls)))
                if len(urls) >= max_urls:
                    break
        else:
            for url_elem in root.findall("sm:url/sm:loc", ns):
                if url_elem.text:
                    urls.append(url_elem.text.strip())
                if len(urls) >= max_urls:
                    break
    except Exception as e:
        logger.error(f"Sitemap error: {e}")

    return urls[:max_urls]


WAYBACK_PREFIX = "https://web.archive.org/web/2024/"
CAPTCHA_THRESHOLD = 3  # Consecutive 429/CAPTCHA before switching to Wayback


def _is_captcha_response(resp: requests.Response) -> bool:
    """Detect Google CAPTCHA / IP block responses."""
    if resp.status_code == 429:
        return True
    # Google sorry page redirect (302 → google.com/sorry/)
    if resp.status_code == 302 and "google.com/sorry" in resp.headers.get("location", ""):
        return True
    # Check response body for CAPTCHA markers
    if resp.status_code == 200:
        body = resp.text[:2000].lower()
        if "unusual traffic" in body or "solvesimplechallenge" in body or "google.com/sorry" in body:
            return True
    return False


def crawl_urls(
    urls: list[str],
    output_dir: Path,
    include_metadata: bool = True,
    run_id: str | None = None,
    delay: float = 0.0,
    max_retries: int = 3,
    wayback_fallback: bool = False,
) -> dict:
    """Crawl a list of URLs and write .raw.md files.

    Args:
        delay: Seconds to wait between requests (polite crawling).
        max_retries: Number of retries on 429/5xx with exponential backoff.
        wayback_fallback: Auto-switch to Wayback Machine on persistent IP block/CAPTCHA.

    Returns crawl statistics dictionary.
    """
    import time
    import random

    output_dir.mkdir(parents=True, exist_ok=True)
    analyzer = PageAnalyzer(timeout=20)

    progress = CrawlProgress(
        run_id=run_id or datetime.now().strftime("%Y%m%d_%H%M%S"),
        total_urls=len(urls),
    )

    results = []
    consecutive_captcha = 0
    using_wayback = False

    def _fetch_with_retry(url: str) -> requests.Response | None:
        """Fetch URL with retry on 429/5xx and exponential backoff."""
        last_resp = None
        for attempt in range(max_retries + 1):
            try:
                last_resp = requests.get(url, timeout=20,
                                         headers={"User-Agent": PageAnalyzer.DEFAULT_USER_AGENT})
                if last_resp.status_code == 429 or last_resp.status_code >= 500:
                    if attempt < max_retries:
                        wait = (2 ** attempt) * 2 + random.uniform(1.0, 3.0)
                        logger.info(f"  ↳ {last_resp.status_code} — retry {attempt+1}/{max_retries} in {wait:.1f}s")
                        time.sleep(wait)
                        continue
                return last_resp
            except requests.RequestException as e:
                if attempt < max_retries:
                    wait = (2 ** attempt) * 2 + random.uniform(1.0, 3.0)
                    logger.info(f"  ↳ Error — retry {attempt+1}/{max_retries} in {wait:.1f}s")
                    time.sleep(wait)
                else:
                    raise
        return last_resp

    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        BarColumn(),
        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
        TextColumn("({task.completed}/{task.total})"),
        console=console,
    ) as pbar:
        task = pbar.add_task("Crawling...", total=len(urls))

        for i, url in enumerate(urls):
            progress.current_url = url

            # Determine fetch URL (direct or Wayback)
            fetch_url = url
            source = "direct"
            if using_wayback:
                fetch_url = WAYBACK_PREFIX + url
                source = "wayback"
                pbar.update(task, description=f"[yellow]WB[/yellow] [cyan]{urlparse(url).path[:35]}...")
            else:
                pbar.update(task, description=f"[cyan]{urlparse(url).path[:40]}...")

            try:
                # Polite delay between requests
                if delay > 0 and i > 0:
                    time.sleep(delay)

                # Fetch and analyze
                metadata = analyzer.analyze_url(url) if (include_metadata and not using_wayback) else None

                # Get raw HTML for content extraction (with retry)
                resp = _fetch_with_retry(fetch_url)

                # CAPTCHA / IP block detection
                if resp and not using_wayback and _is_captcha_response(resp):
                    consecutive_captcha += 1
                    logger.warning(f"  ⚠ CAPTCHA/IP block detected ({consecutive_captcha}/{CAPTCHA_THRESHOLD})")

                    if wayback_fallback and consecutive_captcha >= CAPTCHA_THRESHOLD:
                        using_wayback = True
                        console.print(
                            f"\n[bold yellow]⚠ CAPTCHA DETOUR:[/bold yellow] "
                            f"IP blocked after {consecutive_captcha} consecutive 429s. "
                            f"Switching to Wayback Machine for remaining {len(urls) - i} URLs.\n"
                        )
                        # Retry current URL via Wayback
                        fetch_url = WAYBACK_PREFIX + url
                        source = "wayback"
                        resp = _fetch_with_retry(fetch_url)
                    elif not wayback_fallback and consecutive_captcha >= CAPTCHA_THRESHOLD:
                        console.print(
                            f"\n[bold red]⚠ IP BLOCKED:[/bold red] "
                            f"{consecutive_captcha} consecutive CAPTCHA/429 responses. "
                            f"Consider re-running with --wayback-fallback flag.\n"
                        )

                if resp and resp.status_code == 200 and not _is_captcha_response(resp):
                    if not using_wayback:
                        consecutive_captcha = 0  # Reset on success
                    content = _extract_main_content(resp.text, url)
                    filepath = _write_raw_file(output_dir, i, url, content, metadata)
                    # Record source in frontmatter by re-reading and patching
                    if source == "wayback":
                        existing = filepath.read_text()
                        existing = existing.replace("crawler: seo-aiohttp", "crawler: seo-aiohttp\nsource: wayback", 1)
                        if "source: wayback" not in existing:
                            existing = existing.replace("---\n\n", f"source: {source}\n---\n\n", 1)
                        filepath.write_text(existing)
                    results.append({"url": url, "file": filepath.name, "status": "ok",
                                    "size": len(content), "source": source})
                    progress.succeeded += 1
                    logger.info(f"OK [{resp.status_code}] {url} → {filepath.name} ({source})")
                else:
                    status = f"http_{resp.status_code}" if resp else "no_response"
                    results.append({"url": url, "status": status})
                    progress.failed += 1
                    logger.warning(f"FAIL [{status}] {url}")

            except Exception as e:
                results.append({"url": url, "status": "error", "error": str(e)})
                progress.failed += 1
                logger.error(f"ERROR {url}: {e}")

            progress.processed += 1
            pbar.update(task, advance=1)
            progress.save()

    progress.status = "completed"
    progress.save()

    # Generate combined bundle
    raw_files = sorted(output_dir.glob("*.raw.md"))
    if raw_files:
        bundle_name = output_dir.parent.name or "reference"
        bundle_path = output_dir / f"{bundle_name}-raw-complete.md"
        with open(bundle_path, "w") as bundle:
            bundle.write(f"# {bundle_name} — Raw Content Bundle\n\n")
            bundle.write(f"> Combined raw crawled content from {len(raw_files)} sources\n")
            bundle.write(f"> Generated: {datetime.now().strftime('%Y-%m-%d')}\n\n")
            for rf in raw_files:
                bundle.write("---\n\n")
                bundle.write(rf.read_text())
                bundle.write("\n\n")
        results.append({"file": bundle_path.name, "status": "bundle",
                        "size": bundle_path.stat().st_size})

    stats = {
        "run_id": progress.run_id,
        "total_urls": len(urls),
        "succeeded": progress.succeeded,
        "failed": progress.failed,
        "skipped": progress.skipped,
        "files_written": len(raw_files),
        "output_dir": str(output_dir),
        "results": results,
    }

    # Write manifest
    manifest_path = output_dir / "crawl_manifest.json"
    manifest_path.write_text(json.dumps(stats, indent=2, default=str))

    return stats


# ─── CLI ───────────────────────────────────────────────────────────────────────

@click.group()
def cli():
    """SEO Crawler Adapter — reference-curator crawling backend."""
    pass


@cli.command()
@click.option("--urls", required=True, help="Comma-separated URLs or @file.txt")
@click.option("--output", required=True, type=click.Path(), help="Output directory for .raw.md files")
@click.option("--no-metadata", is_flag=True, help="Skip SEO metadata extraction")
@click.option("--run-id", help="Custom run ID for progress tracking")
@click.option("--delay", default=0.0, type=float, help="Seconds between requests (polite crawling)")
@click.option("--wayback-fallback", is_flag=True, help="Auto-switch to Wayback Machine on CAPTCHA/IP block")
def crawl(urls, output, no_metadata, run_id, delay, wayback_fallback):
    """Crawl a list of URLs and write .raw.md files."""
    if urls.startswith("@"):
        url_list = Path(urls[1:]).read_text().strip().splitlines()
    else:
        url_list = [u.strip() for u in urls.split(",") if u.strip()]

    output_path = Path(output).expanduser()
    stats = crawl_urls(url_list, output_path, include_metadata=not no_metadata, run_id=run_id,
                       delay=delay, wayback_fallback=wayback_fallback)

    wayback_count = sum(1 for r in stats.get("results", []) if r.get("source") == "wayback")
    console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")
    if wayback_count:
        console.print(f"[yellow]Wayback fallback used for {wayback_count} pages[/yellow]")
    console.print(f"Output: {stats['output_dir']}")


@cli.command("crawl-sitemap")
@click.option("--sitemap", required=True, help="Sitemap URL")
@click.option("--output", required=True, type=click.Path(), help="Output directory")
@click.option("--max-pages", default=50, help="Max pages to crawl")
@click.option("--run-id", help="Custom run ID")
def crawl_sitemap(sitemap, output, max_pages, run_id):
    """Crawl URLs discovered from an XML sitemap."""
    console.print(f"Parsing sitemap: {sitemap}")
    urls = discover_from_sitemap(sitemap, max_urls=max_pages)
    console.print(f"Found {len(urls)} URLs in sitemap")

    if not urls:
        console.print("[yellow]No URLs found in sitemap[/yellow]")
        return

    output_path = Path(output).expanduser()
    stats = crawl_urls(urls, output_path, run_id=run_id)
    console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")


@cli.command("discover")
@click.option("--url", required=True, help="Root URL to discover from")
@click.option("--output", required=True, type=click.Path(), help="Output directory")
@click.option("--max-depth", default=2, help="Max link-following depth")
@click.option("--max-pages", default=50, help="Max pages to discover and crawl")
@click.option("--same-domain/--follow-external", default=True, help="Stay on same domain")
@click.option("--run-id", help="Custom run ID")
def discover(url, output, max_depth, max_pages, same_domain, run_id):
    """Discover URLs from a root page, then crawl them all."""
    console.print(f"Discovering URLs from: {url} (depth={max_depth})")
    urls = discover_urls(url, max_depth=max_depth, same_domain=same_domain, max_urls=max_pages)
    console.print(f"Discovered {len(urls)} URLs")

    if not urls:
        console.print("[yellow]No URLs discovered[/yellow]")
        return

    output_path = Path(output).expanduser()
    stats = crawl_urls(urls, output_path, run_id=run_id)
    console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")


@cli.command("crawl-manifest")
@click.option("--manifest", required=True, type=click.Path(exists=True), help="Manifest JSON file")
@click.option("--output", required=True, type=click.Path(), help="Output directory")
@click.option("--run-id", help="Custom run ID")
def crawl_manifest(manifest, output, run_id):
    """Crawl URLs from a reference-curator manifest file."""
    data = json.loads(Path(manifest).read_text())

    urls = []
    if isinstance(data, dict) and "sources" in data:
        urls = [s["url"] for s in data["sources"] if s.get("url")]
    elif isinstance(data, list):
        urls = [item.get("url", item) if isinstance(item, dict) else item for item in data]

    console.print(f"Loaded {len(urls)} URLs from manifest")

    output_path = Path(output).expanduser()
    stats = crawl_urls(urls, output_path, run_id=run_id)
    console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")


@cli.command("status")
@click.option("--run-id", help="Specific run ID to check")
def status(run_id):
    """Check crawl progress."""
    if run_id:
        filepath = PROGRESS_DIR / f"{run_id}.json"
        if filepath.exists():
            data = json.loads(filepath.read_text())
            console.print_json(json.dumps(data, indent=2))
        else:
            console.print(f"[yellow]No progress file for run {run_id}[/yellow]")
    else:
        files = sorted(PROGRESS_DIR.glob("*.json"), reverse=True)[:10]
        if not files:
            console.print("[yellow]No crawl history found[/yellow]")
            return

        table = Table(title="Recent Crawls")
        table.add_column("Run ID")
        table.add_column("Status")
        table.add_column("Progress")
        table.add_column("Success/Total")

        for f in files:
            data = json.loads(f.read_text())
            table.add_row(
                data.get("run_id", "?"),
                data.get("status", "?"),
                f"{data.get('progress_pct', 0)}%",
                f"{data.get('succeeded', 0)}/{data.get('total_urls', 0)}",
            )
        console.print(table)


if __name__ == "__main__":
    cli()
