"""
Keyword Researcher - Seed keyword expansion, intent classification, and topic clustering
========================================================================================
Purpose: Expand seed keywords via Ahrefs APIs, classify search intent,
         cluster topics, and support Korean market keyword discovery.
Python: 3.10+
"""

import argparse
import json
import logging
import re
import subprocess
import sys
from dataclasses import dataclass, field, asdict
from datetime import datetime
from typing import Optional

# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger("keyword_researcher")

# ---------------------------------------------------------------------------
# Constants - Korean suffix expansion
# ---------------------------------------------------------------------------
KOREAN_SUFFIXES: list[str] = [
    "추천",
    "가격",
    "후기",
    "잘하는곳",
    "부작용",
    "전후",
    "비용",
    "추천 병원",
    "후기 블로그",
    "방법",
    "종류",
    "비교",
    "효과",
    "주의사항",
    "장단점",
]

# ---------------------------------------------------------------------------
# Intent classification patterns
# ---------------------------------------------------------------------------
INTENT_PATTERNS: dict[str, list[str]] = {
    "transactional": [
        r"구매|구입|주문|buy|order|purchase|shop|deal|discount|coupon|할인|쿠폰",
        r"예약|booking|reserve|sign\s?up|register|등록|신청",
    ],
    "commercial": [
        r"가격|비용|얼마|price|cost|pricing|fee|요금",
        r"추천|best|top\s?\d|review|비교|compare|vs|versus|후기|리뷰|평점|평가",
        r"잘하는곳|잘하는|맛집|업체|병원|추천\s?병원",
    ],
    "navigational": [
        r"^(www\.|http|\.com|\.co\.kr|\.net)",
        r"공식|official|login|로그인|홈페이지|사이트|website",
        r"고객센터|contact|support|customer\s?service",
    ],
    "informational": [
        r"방법|how\s?to|what\s?is|why|when|where|who|which",
        r"뜻|의미|정의|definition|meaning|guide|tutorial",
        r"효과|부작용|증상|원인|차이|종류|type|cause|symptom|effect",
        r"전후|before\s?and\s?after|결과|result",
    ],
}

# ---------------------------------------------------------------------------
# Dataclasses
# ---------------------------------------------------------------------------


@dataclass
class KeywordEntry:
    """Single keyword with its metrics and classification."""

    keyword: str
    volume: int = 0
    kd: float = 0.0
    cpc: float = 0.0
    intent: str = "informational"
    cluster: str = ""
    source: str = ""
    country_volumes: dict[str, int] = field(default_factory=dict)

    def to_dict(self) -> dict:
        data = asdict(self)
        if not data["country_volumes"]:
            del data["country_volumes"]
        return data


@dataclass
class KeywordCluster:
    """Group of semantically related keywords."""

    topic: str
    keywords: list[str] = field(default_factory=list)
    total_volume: int = 0
    avg_kd: float = 0.0
    primary_intent: str = "informational"

    def to_dict(self) -> dict:
        return asdict(self)


@dataclass
class ResearchResult:
    """Full research result container."""

    seed_keyword: str
    country: str
    total_keywords: int = 0
    total_volume: int = 0
    clusters: list[KeywordCluster] = field(default_factory=list)
    keywords: list[KeywordEntry] = field(default_factory=list)
    timestamp: str = ""

    def to_dict(self) -> dict:
        return {
            "seed_keyword": self.seed_keyword,
            "country": self.country,
            "total_keywords": self.total_keywords,
            "total_volume": self.total_volume,
            "clusters": [c.to_dict() for c in self.clusters],
            "keywords": [k.to_dict() for k in self.keywords],
            "timestamp": self.timestamp,
        }


# ---------------------------------------------------------------------------
# MCP Helper - calls Ahrefs MCP tools via subprocess
# ---------------------------------------------------------------------------


def call_mcp_tool(tool_name: str, params: dict) -> dict:
    """
    Call an Ahrefs MCP tool and return parsed JSON response.

    In production this delegates to the MCP bridge. For standalone usage
    it invokes the Claude CLI with the appropriate tool call.
    """
    logger.info(f"Calling MCP tool: {tool_name} with params: {json.dumps(params, ensure_ascii=False)}")

    try:
        cmd = [
            "claude",
            "--print",
            "--output-format", "json",
            "-p",
            f"Call the tool mcp__claude_ai_Ahrefs__{tool_name} with these parameters: {json.dumps(params, ensure_ascii=False)}. Return ONLY the raw JSON result.",
        ]
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)

        if result.returncode != 0:
            logger.warning(f"MCP tool {tool_name} returned non-zero exit code: {result.returncode}")
            logger.debug(f"stderr: {result.stderr}")
            return {"error": result.stderr, "keywords": [], "items": []}

        try:
            return json.loads(result.stdout)
        except json.JSONDecodeError:
            return {"raw": result.stdout, "keywords": [], "items": []}

    except subprocess.TimeoutExpired:
        logger.error(f"MCP tool {tool_name} timed out")
        return {"error": "timeout", "keywords": [], "items": []}
    except FileNotFoundError:
        logger.warning("Claude CLI not found - returning empty result for standalone testing")
        return {"keywords": [], "items": []}


# ---------------------------------------------------------------------------
# KeywordResearcher
# ---------------------------------------------------------------------------


class KeywordResearcher:
    """Expand seed keywords, classify intent, and cluster topics."""

    def __init__(self, country: str = "kr", korean_suffixes: bool = False, compare_global: bool = False):
        self.country = country
        self.korean_suffixes = korean_suffixes
        self.compare_global = compare_global
        self._seen: set[str] = set()

    # ---- Keyword expansion via Ahrefs MCP ----

    def expand_keywords(self, seed: str) -> list[KeywordEntry]:
        """
        Expand a seed keyword using Ahrefs matching-terms, related-terms,
        and search-suggestions endpoints.
        """
        all_keywords: list[KeywordEntry] = []

        # 1. Matching terms
        logger.info(f"Fetching matching terms for: {seed}")
        matching = call_mcp_tool("keywords-explorer-matching-terms", {
            "keyword": seed,
            "country": self.country,
            "limit": 100,
        })
        for item in matching.get("keywords", matching.get("items", [])):
            kw = self._parse_keyword_item(item, source="matching-terms")
            if kw and kw.keyword not in self._seen:
                self._seen.add(kw.keyword)
                all_keywords.append(kw)

        # 2. Related terms
        logger.info(f"Fetching related terms for: {seed}")
        related = call_mcp_tool("keywords-explorer-related-terms", {
            "keyword": seed,
            "country": self.country,
            "limit": 100,
        })
        for item in related.get("keywords", related.get("items", [])):
            kw = self._parse_keyword_item(item, source="related-terms")
            if kw and kw.keyword not in self._seen:
                self._seen.add(kw.keyword)
                all_keywords.append(kw)

        # 3. Search suggestions
        logger.info(f"Fetching search suggestions for: {seed}")
        suggestions = call_mcp_tool("keywords-explorer-search-suggestions", {
            "keyword": seed,
            "country": self.country,
            "limit": 50,
        })
        for item in suggestions.get("keywords", suggestions.get("items", [])):
            kw = self._parse_keyword_item(item, source="search-suggestions")
            if kw and kw.keyword not in self._seen:
                self._seen.add(kw.keyword)
                all_keywords.append(kw)

        # 4. Add the seed itself if not already present
        if seed not in self._seen:
            self._seen.add(seed)
            overview = call_mcp_tool("keywords-explorer-overview", {
                "keyword": seed,
                "country": self.country,
            })
            seed_entry = self._parse_keyword_item(overview, source="seed")
            if seed_entry:
                seed_entry.keyword = seed
                all_keywords.insert(0, seed_entry)

        logger.info(f"Expanded to {len(all_keywords)} keywords from Ahrefs APIs")
        return all_keywords

    def expand_korean_suffixes(self, seed: str) -> list[KeywordEntry]:
        """
        Generate keyword variations by appending common Korean suffixes.
        Each variation is checked against Ahrefs for volume data.
        """
        suffix_keywords: list[KeywordEntry] = []

        for suffix in KOREAN_SUFFIXES:
            variation = f"{seed} {suffix}"
            if variation in self._seen:
                continue

            logger.info(f"Checking Korean suffix variation: {variation}")
            overview = call_mcp_tool("keywords-explorer-overview", {
                "keyword": variation,
                "country": self.country,
            })
            kw = self._parse_keyword_item(overview, source="korean-suffix")
            if kw:
                kw.keyword = variation
                if kw.volume > 0:
                    self._seen.add(variation)
                    suffix_keywords.append(kw)
            else:
                # Even if no data, include as zero-volume for completeness
                entry = KeywordEntry(
                    keyword=variation,
                    volume=0,
                    kd=0.0,
                    cpc=0.0,
                    intent=self.classify_intent(variation),
                    source="korean-suffix",
                )
                self._seen.add(variation)
                suffix_keywords.append(entry)

        logger.info(f"Korean suffix expansion yielded {len(suffix_keywords)} variations")
        return suffix_keywords

    def get_volume_by_country(self, keyword: str) -> dict[str, int]:
        """
        Get search volume breakdown by country for a keyword.
        Useful for comparing Korean vs global demand.
        """
        logger.info(f"Fetching volume-by-country for: {keyword}")
        result = call_mcp_tool("keywords-explorer-volume-by-country", {
            "keyword": keyword,
        })

        volumes: dict[str, int] = {}
        for item in result.get("countries", result.get("items", [])):
            if isinstance(item, dict):
                country_code = item.get("country", item.get("code", ""))
                volume = item.get("volume", item.get("search_volume", 0))
                if country_code and volume:
                    volumes[country_code.lower()] = int(volume)

        return volumes

    # ---- Intent classification ----

    def classify_intent(self, keyword: str) -> str:
        """
        Classify search intent based on keyword patterns.
        Priority: transactional > commercial > navigational > informational
        """
        keyword_lower = keyword.lower().strip()

        for intent, patterns in INTENT_PATTERNS.items():
            for pattern in patterns:
                if re.search(pattern, keyword_lower, re.IGNORECASE):
                    return intent

        return "informational"

    # ---- Keyword clustering ----

    def cluster_keywords(self, keywords: list[KeywordEntry]) -> list[KeywordCluster]:
        """
        Group keywords into topic clusters using shared n-gram tokens.
        Uses a simple token overlap approach: keywords sharing significant
        tokens (2+ character words) are grouped together.
        """
        if not keywords:
            return []

        # Extract meaningful tokens from each keyword
        def tokenize(text: str) -> set[str]:
            tokens = set()
            for word in re.split(r"\s+", text.strip().lower()):
                if len(word) >= 2:
                    tokens.add(word)
            return tokens

        # Build token-to-keyword mapping
        token_map: dict[str, list[int]] = {}
        kw_tokens: list[set[str]] = []

        for i, kw in enumerate(keywords):
            tokens = tokenize(kw.keyword)
            kw_tokens.append(tokens)
            for token in tokens:
                if token not in token_map:
                    token_map[token] = []
                token_map[token].append(i)

        # Find the most common significant tokens (cluster anchors)
        token_freq = sorted(token_map.items(), key=lambda x: len(x[1]), reverse=True)

        assigned: set[int] = set()
        clusters: list[KeywordCluster] = []

        for token, indices in token_freq:
            # Skip single-occurrence tokens or very common stop-like tokens
            if len(indices) < 2:
                continue

            # Gather unassigned keywords that share this token
            cluster_indices = [i for i in indices if i not in assigned]
            if len(cluster_indices) < 2:
                continue

            # Create the cluster
            cluster_kws = [keywords[i].keyword for i in cluster_indices]
            cluster_volumes = [keywords[i].volume for i in cluster_indices]
            cluster_kds = [keywords[i].kd for i in cluster_indices]
            cluster_intents = [keywords[i].intent for i in cluster_indices]

            # Determine primary intent by frequency
            intent_counts: dict[str, int] = {}
            for intent in cluster_intents:
                intent_counts[intent] = intent_counts.get(intent, 0) + 1
            primary_intent = max(intent_counts, key=intent_counts.get)

            cluster = KeywordCluster(
                topic=token,
                keywords=cluster_kws,
                total_volume=sum(cluster_volumes),
                avg_kd=round(sum(cluster_kds) / len(cluster_kds), 1) if cluster_kds else 0.0,
                primary_intent=primary_intent,
            )
            clusters.append(cluster)

            for i in cluster_indices:
                assigned.add(i)
                keywords[i].cluster = token

        # Assign unclustered keywords to an "other" cluster
        unclustered = [i for i in range(len(keywords)) if i not in assigned]
        if unclustered:
            other_kws = [keywords[i].keyword for i in unclustered]
            other_volumes = [keywords[i].volume for i in unclustered]
            other_kds = [keywords[i].kd for i in unclustered]

            other_cluster = KeywordCluster(
                topic="(unclustered)",
                keywords=other_kws,
                total_volume=sum(other_volumes),
                avg_kd=round(sum(other_kds) / len(other_kds), 1) if other_kds else 0.0,
                primary_intent="informational",
            )
            clusters.append(other_cluster)

            for i in unclustered:
                keywords[i].cluster = "(unclustered)"

        # Sort clusters by total volume descending
        clusters.sort(key=lambda c: c.total_volume, reverse=True)

        logger.info(f"Clustered {len(keywords)} keywords into {len(clusters)} clusters")
        return clusters

    # ---- Full analysis orchestration ----

    def analyze(self, seed_keyword: str) -> ResearchResult:
        """
        Orchestrate a full keyword research analysis:
        1. Expand seed via Ahrefs
        2. Optionally expand Korean suffixes
        3. Classify intent for all keywords
        4. Optionally fetch volume-by-country
        5. Cluster keywords into topics
        6. Compile results
        """
        logger.info(f"Starting keyword research for: {seed_keyword} (country={self.country})")

        # Step 1: Expand keywords
        keywords = self.expand_keywords(seed_keyword)

        # Step 2: Korean suffix expansion
        if self.korean_suffixes:
            suffix_keywords = self.expand_korean_suffixes(seed_keyword)
            keywords.extend(suffix_keywords)

        # Step 3: Classify intent for all keywords
        for kw in keywords:
            if not kw.intent or kw.intent == "informational":
                kw.intent = self.classify_intent(kw.keyword)

        # Step 4: Volume-by-country comparison
        if self.compare_global and keywords:
            # Fetch for the seed and top volume keywords
            top_keywords = sorted(keywords, key=lambda k: k.volume, reverse=True)[:10]
            for kw in top_keywords:
                volumes = self.get_volume_by_country(kw.keyword)
                kw.country_volumes = volumes

        # Step 5: Cluster keywords
        clusters = self.cluster_keywords(keywords)

        # Step 6: Compile result
        result = ResearchResult(
            seed_keyword=seed_keyword,
            country=self.country,
            total_keywords=len(keywords),
            total_volume=sum(kw.volume for kw in keywords),
            clusters=clusters,
            keywords=sorted(keywords, key=lambda k: k.volume, reverse=True),
            timestamp=datetime.now().isoformat(),
        )

        logger.info(
            f"Research complete: {result.total_keywords} keywords, "
            f"{len(result.clusters)} clusters, "
            f"total volume {result.total_volume}"
        )
        return result

    # ---- Internal helpers ----

    def _parse_keyword_item(self, item: dict, source: str = "") -> Optional[KeywordEntry]:
        """Parse an Ahrefs API response item into a KeywordEntry."""
        if not item or "error" in item:
            return None

        keyword = item.get("keyword", item.get("term", item.get("query", "")))
        if not keyword:
            return None

        volume = int(item.get("volume", item.get("search_volume", 0)) or 0)
        kd = float(item.get("keyword_difficulty", item.get("kd", 0)) or 0)
        cpc = float(item.get("cpc", item.get("cost_per_click", 0)) or 0)

        return KeywordEntry(
            keyword=keyword,
            volume=volume,
            kd=round(kd, 1),
            cpc=round(cpc, 2),
            intent=self.classify_intent(keyword),
            source=source,
        )


# ---------------------------------------------------------------------------
# Plain-text report formatter
# ---------------------------------------------------------------------------


def format_text_report(result: ResearchResult) -> str:
    """Format research result as a human-readable text report."""
    lines: list[str] = []
    lines.append("=" * 70)
    lines.append(f"Keyword Strategy Report: {result.seed_keyword}")
    lines.append(f"Country: {result.country.upper()} | Date: {result.timestamp[:10]}")
    lines.append("=" * 70)
    lines.append("")

    lines.append("## Overview")
    lines.append(f"  Total keywords discovered: {result.total_keywords}")
    lines.append(f"  Topic clusters: {len(result.clusters)}")
    lines.append(f"  Total search volume: {result.total_volume:,}")
    lines.append("")

    # Clusters summary
    if result.clusters:
        lines.append("## Top Clusters")
        lines.append(f"  {'Cluster':<25} {'Keywords':>8} {'Volume':>10} {'Avg KD':>8} {'Intent':<15}")
        lines.append("  " + "-" * 66)
        for cluster in result.clusters[:15]:
            lines.append(
                f"  {cluster.topic:<25} {len(cluster.keywords):>8} "
                f"{cluster.total_volume:>10,} {cluster.avg_kd:>8.1f} "
                f"{cluster.primary_intent:<15}"
            )
        lines.append("")

    # Top keywords
    if result.keywords:
        lines.append("## Top Keywords (by volume)")
        lines.append(f"  {'Keyword':<40} {'Vol':>8} {'KD':>6} {'CPC':>7} {'Intent':<15} {'Cluster':<15}")
        lines.append("  " + "-" * 91)
        for kw in result.keywords[:30]:
            kw_display = kw.keyword[:38] if len(kw.keyword) > 38 else kw.keyword
            cluster_display = kw.cluster[:13] if len(kw.cluster) > 13 else kw.cluster
            lines.append(
                f"  {kw_display:<40} {kw.volume:>8,} {kw.kd:>6.1f} "
                f"{kw.cpc:>7.2f} {kw.intent:<15} {cluster_display:<15}"
            )
        lines.append("")

    # Intent distribution
    intent_dist: dict[str, int] = {}
    for kw in result.keywords:
        intent_dist[kw.intent] = intent_dist.get(kw.intent, 0) + 1
    if intent_dist:
        lines.append("## Intent Distribution")
        for intent, count in sorted(intent_dist.items(), key=lambda x: x[1], reverse=True):
            pct = (count / len(result.keywords)) * 100 if result.keywords else 0
            lines.append(f"  {intent:<15}: {count:>5} ({pct:.1f}%)")
        lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def main():
    parser = argparse.ArgumentParser(
        description="Keyword Researcher - Expand, classify, and cluster keywords",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python keyword_researcher.py --keyword "치과 임플란트" --country kr --json
  python keyword_researcher.py --keyword "dental implant" --compare-global --json
  python keyword_researcher.py --keyword "치과 임플란트" --korean-suffixes --output report.json
        """,
    )
    parser.add_argument(
        "--keyword",
        required=True,
        help="Seed keyword to expand and research",
    )
    parser.add_argument(
        "--country",
        default="kr",
        help="Target country code (default: kr)",
    )
    parser.add_argument(
        "--korean-suffixes",
        action="store_true",
        help="Enable Korean suffix expansion (추천, 가격, 후기, etc.)",
    )
    parser.add_argument(
        "--compare-global",
        action="store_true",
        help="Fetch volume-by-country comparison for top keywords",
    )
    parser.add_argument(
        "--json",
        action="store_true",
        dest="output_json",
        help="Output results as JSON",
    )
    parser.add_argument(
        "--output",
        type=str,
        default=None,
        help="Write output to file (path)",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Enable verbose/debug logging",
    )

    args = parser.parse_args()

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    # Run analysis
    researcher = KeywordResearcher(
        country=args.country,
        korean_suffixes=args.korean_suffixes,
        compare_global=args.compare_global,
    )
    result = researcher.analyze(args.keyword)

    # Format output
    if args.output_json:
        output = json.dumps(result.to_dict(), ensure_ascii=False, indent=2)
    else:
        output = format_text_report(result)

    # Write or print
    if args.output:
        with open(args.output, "w", encoding="utf-8") as f:
            f.write(output)
        logger.info(f"Output written to: {args.output}")
    else:
        print(output)

    return 0


if __name__ == "__main__":
    sys.exit(main())
