#!/usr/bin/env python3
"""Web Crawler Manager CLI — manage crawl results and crawler selection.

Claude performs actual crawling via Firecrawl MCP. This script handles:
- Storing crawl results metadata
- Selecting the optimal crawler backend
- Generating crawl result manifests

Usage:
    python crawl_mgr.py store-result --manifest manifest.json --raw-dir ~/reference-library/raw/
    python crawl_mgr.py select-crawler --url "https://docs.anthropic.com"
    python crawl_mgr.py list-crawls [--status completed]
"""

import json
import re
import sys
from datetime import datetime
from pathlib import Path
from urllib.parse import urlparse

import click
from rich.console import Console
from rich.table import Table

from refcurator.db import db_session
from refcurator.manifest import create_crawl_result, write_crawl_result, read_manifest
from refcurator.utils import count_tokens

console = Console()


# Known SPA frameworks and their indicators
SPA_INDICATORS = {
    "react": ["react", "next.js", "nextjs", "gatsby", "remix"],
    "vue": ["vue", "nuxt"],
    "angular": ["angular"],
    "svelte": ["svelte", "sveltekit"],
}

# Known static site generators
STATIC_INDICATORS = ["hugo", "jekyll", "mkdocs", "docusaurus", "gitbook", "sphinx",
                      "readthedocs", "vuepress", "docsify"]


@click.group()
def cli():
    """Web Crawler Manager — manage crawl results and backend selection."""
    pass


@cli.command("store-result")
@click.option("--raw-dir", required=True, type=click.Path(exists=True),
              help="Directory containing crawled raw files")
@click.option("--crawler", default="firecrawl",
              type=click.Choice(["firecrawl", "seo-aiohttp", "nodejs", "aiohttp", "scrapy"]))
@click.option("--source-id", type=int, help="Source ID to associate documents with")
@click.option("--output", type=click.Path(), help="Output crawl result JSON path")
def store_result(raw_dir, crawler, source_id, output):
    """Store crawl results from raw files into the repository.

    Scans raw_dir for .md files, creates document records, and generates a crawl result manifest.
    """
    raw_path = Path(raw_dir)
    md_files = sorted(raw_path.glob("**/*.md"))

    if not md_files:
        console.print(f"[yellow]No .md files found in {raw_dir}[/yellow]")
        sys.exit(0)

    entries = []
    stored = 0

    with db_session() as db:
        for md_file in md_files:
            content = md_file.read_text(errors="replace")
            content_size = md_file.stat().st_size

            # Extract title from first heading or filename
            title = _extract_title(content, md_file.stem)

            # Extract URL from frontmatter if available
            url = _extract_url_from_content(content)

            entry = {
                "url": url or f"file://{md_file.resolve()}",
                "title": title,
                "raw_path": str(md_file.resolve()),
                "content_size": content_size,
                "status": "completed",
            }
            entries.append(entry)

            if source_id:
                db.insert_returning_id(
                    """INSERT INTO documents
                       (source_id, title, url, doc_type, crawl_date,
                        crawl_method, crawl_status, raw_content_path, raw_content_size)
                       VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""",
                    (source_id, title, url, "markdown", datetime.now().isoformat(),
                     crawler, "completed", str(md_file.resolve()), content_size),
                )
                stored += 1

    result = create_crawl_result(entries, crawler)

    if output:
        write_crawl_result(result, Path(output))
        console.print(f"[green]Crawl result written to {output}[/green]")

    console.print(
        f"[green]Processed {len(entries)} files[/green]"
        + (f", stored {stored} documents" if stored else "")
    )
    click.echo(result.model_dump_json(indent=2))


@cli.command("select-crawler")
@click.option("--url", required=True, help="Target URL to analyze")
@click.option("--page-count", type=int, help="Expected page count")
@click.option("--json-output", is_flag=True, help="Output as JSON")
def select_crawler(url, page_count, json_output):
    """Recommend the best crawler backend for a URL.

    Analyzes URL patterns and site characteristics to suggest:
    - seo-aiohttp: Documentation sites, sitemaps, HTML-renderable (NEW default for docs)
    - firecrawl: SPAs, JS-rendered content
    - nodejs: Small static docs sites (<=50 pages)
    - aiohttp: Medium technical docs (<=200 pages, legacy)
    - scrapy: Large enterprise sites (>200 pages)
    """
    parsed = urlparse(url)
    domain = parsed.netloc.lower()
    path = parsed.path.lower()

    recommendation = "firecrawl"  # Default
    reason = "Default recommendation for general crawling"
    confidence = 0.7

    # Check for SPA indicators in domain/path
    url_lower = url.lower()
    for framework, indicators in SPA_INDICATORS.items():
        if any(ind in url_lower or ind in domain for ind in indicators):
            recommendation = "firecrawl"
            reason = f"Detected {framework} SPA framework — needs JS rendering"
            confidence = 0.9
            break

    # Check for static site indicators
    for gen in STATIC_INDICATORS:
        if gen in url_lower or gen in domain:
            if page_count and page_count <= 50:
                recommendation = "nodejs"
                reason = f"Static site ({gen}), small page count — lightweight crawler sufficient"
            else:
                recommendation = "aiohttp"
                reason = f"Static site ({gen}), moderate size — async crawler recommended"
            confidence = 0.85
            break

    # Page count override (takes precedence over default, not over SPA detection)
    spa_detected = confidence >= 0.9 and recommendation == "firecrawl"
    if page_count and not spa_detected:
        if page_count > 200:
            recommendation = "scrapy"
            reason = f"Large site ({page_count} pages) — enterprise crawler needed"
            confidence = 0.9
        elif page_count <= 50:
            recommendation = "nodejs"
            reason = f"Small site ({page_count} pages) — lightweight crawler sufficient"
            confidence = 0.8

    # Known documentation platforms → seo-aiohttp (preferred for docs)
    doc_platforms = ["docs.", "developer.", "api.", "reference.", "support."]
    if any(domain.startswith(p) for p in doc_platforms):
        if not spa_detected:
            recommendation = "seo-aiohttp"
            reason = "Documentation platform — SEO crawler provides metadata + rate limiting + resume"
            confidence = 0.90

    result = {
        "url": url,
        "recommendation": recommendation,
        "reason": reason,
        "confidence": confidence,
        "alternatives": _get_alternatives(recommendation),
    }

    if json_output:
        click.echo(json.dumps(result, indent=2))
    else:
        console.print(f"\n[bold]Crawler Recommendation for:[/bold] {url}")
        console.print(f"  [green]Recommended:[/green] {recommendation}")
        console.print(f"  [dim]Reason:[/dim] {reason}")
        console.print(f"  [dim]Confidence:[/dim] {confidence:.0%}")
        if result["alternatives"]:
            console.print(f"  [dim]Alternatives:[/dim] {', '.join(result['alternatives'])}")


@cli.command("list-crawls")
@click.option("--status", type=click.Choice(["pending", "completed", "failed", "stale"]),
              help="Filter by crawl status")
@click.option("--limit", default=20, type=int, help="Max results")
def list_crawls(status, limit):
    """List recent crawl records."""
    with db_session() as db:
        if status:
            rows = db.fetch_all(
                """SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date
                   FROM documents WHERE crawl_status = %s
                   ORDER BY crawl_date DESC LIMIT %s""",
                (status, limit),
            )
        else:
            rows = db.fetch_all(
                """SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date
                   FROM documents ORDER BY crawl_date DESC LIMIT %s""",
                (limit,),
            )

    table = Table(title="Crawl Records")
    table.add_column("ID", style="cyan")
    table.add_column("Title")
    table.add_column("Crawler")
    table.add_column("Status")
    table.add_column("Date")
    for r in rows:
        table.add_row(
            str(r.get("doc_id", "")),
            str(r.get("title", ""))[:40],
            str(r.get("crawl_method", "")),
            str(r.get("crawl_status", "")),
            str(r.get("crawl_date", ""))[:10],
        )
    console.print(table)


def _extract_title(content: str, fallback: str) -> str:
    """Extract title from markdown content."""
    for line in content.split("\n")[:10]:
        line = line.strip()
        if line.startswith("# ") and not line.startswith("##"):
            return line[2:].strip()
    return fallback.replace("-", " ").replace("_", " ").title()


def _extract_url_from_content(content: str) -> str | None:
    """Extract source URL from markdown frontmatter or content."""
    # YAML frontmatter
    if content.startswith("---"):
        fm_end = content.find("---", 3)
        if fm_end > 0:
            fm = content[3:fm_end]
            for line in fm.split("\n"):
                if line.strip().startswith("url:") or line.strip().startswith("source:"):
                    url = line.split(":", 1)[1].strip().strip("'\"")
                    if url.startswith("http"):
                        return url

    # Source: URL pattern in content
    m = re.search(r"\*\*Source:\*\*\s*(https?://\S+)", content[:500])
    if m:
        return m.group(1)

    return None


def _get_alternatives(primary: str) -> list[str]:
    """Get alternative crawler recommendations."""
    all_crawlers = ["seo-aiohttp", "firecrawl", "nodejs", "aiohttp", "scrapy"]
    return [c for c in all_crawlers if c != primary][:2]


if __name__ == "__main__":
    cli()
