our-claude-skills/custom-skills/90-reference-curator/shared/lib/src/refcurator/models.py

"""Pydantic v2 models matching the reference_library MySQL schema."""

from __future__ import annotations

from datetime import date, datetime
from enum import Enum
from typing import Any, Optional

from pydantic import BaseModel, Field, computed_field


# --- Enums matching MySQL ENUMs ---

class SourceType(str, Enum):
    official_docs = "official_docs"
    engineering_blog = "engineering_blog"
    research_paper = "research_paper"
    github_repo = "github_repo"
    community_guide = "community_guide"
    pdf_document = "pdf_document"
    api_reference = "api_reference"


class CredibilityTier(str, Enum):
    tier1_official = "tier1_official"
    tier2_verified = "tier2_verified"
    tier3_community = "tier3_community"


class DocType(str, Enum):
    webpage = "webpage"
    pdf = "pdf"
    markdown = "markdown"
    api_spec = "api_spec"
    code_sample = "code_sample"


class Language(str, Enum):
    en = "en"
    ko = "ko"
    mixed = "mixed"


class CrawlMethod(str, Enum):
    firecrawl = "firecrawl"
    scrapy = "scrapy"
    aiohttp = "aiohttp"
    nodejs = "nodejs"
    manual = "manual"
    api = "api"


class CrawlStatus(str, Enum):
    pending = "pending"
    completed = "completed"
    failed = "failed"
    stale = "stale"


class ReviewStatus(str, Enum):
    pending = "pending"
    in_review = "in_review"
    approved = "approved"
    needs_refactor = "needs_refactor"
    rejected = "rejected"


class ReviewerType(str, Enum):
    auto_qa = "auto_qa"
    human = "human"
    claude_review = "claude_review"
    gemini_review = "gemini_review"


class Decision(str, Enum):
    approve = "approve"
    refactor = "refactor"
    deep_research = "deep_research"
    reject = "reject"


class PipelineStatus(str, Enum):
    running = "running"
    completed = "completed"
    failed = "failed"
    paused = "paused"


class PipelineStage(str, Enum):
    discovery = "discovery"
    crawling = "crawling"
    storing = "storing"
    evaluating = "evaluating"
    distilling = "distilling"
    exporting = "exporting"


class RunType(str, Enum):
    topic = "topic"
    urls = "urls"
    manifest = "manifest"


class ExportType(str, Enum):
    project_files = "project_files"
    fine_tuning = "fine_tuning"
    training_dataset = "training_dataset"
    knowledge_base = "knowledge_base"


class OutputFormat(str, Enum):
    markdown = "markdown"
    jsonl = "jsonl"
    parquet = "parquet"
    sqlite = "sqlite"


class Frequency(str, Enum):
    daily = "daily"
    weekly = "weekly"
    biweekly = "biweekly"
    monthly = "monthly"
    on_demand = "on_demand"


class ChangeType(str, Enum):
    content_updated = "content_updated"
    url_moved = "url_moved"
    deleted = "deleted"
    new_version = "new_version"


class FinalDecision(str, Enum):
    approved = "approved"
    rejected = "rejected"
    needs_manual_review = "needs_manual_review"


# --- Core Table Models ---

class Source(BaseModel):
    source_id: Optional[int] = None
    source_name: str
    source_type: SourceType
    base_url: Optional[str] = None
    credibility_tier: CredibilityTier = CredibilityTier.tier3_community
    vendor: Optional[str] = None
    is_active: bool = True
    created_at: Optional[datetime] = None
    updated_at: Optional[datetime] = None


class Document(BaseModel):
    doc_id: Optional[int] = None
    source_id: int
    title: str
    url: Optional[str] = None
    url_hash: Optional[str] = None  # Generated column in MySQL
    doc_type: DocType
    language: Language = Language.en
    original_publish_date: Optional[date] = None
    last_modified_date: Optional[date] = None
    crawl_date: Optional[datetime] = None
    crawl_method: CrawlMethod = CrawlMethod.firecrawl
    crawl_status: CrawlStatus = CrawlStatus.pending
    raw_content_path: Optional[str] = None
    raw_content_size: Optional[int] = None
    version: int = 1
    previous_version_id: Optional[int] = None
    created_at: Optional[datetime] = None
    updated_at: Optional[datetime] = None


class DistilledContent(BaseModel):
    distill_id: Optional[int] = None
    doc_id: int
    summary: Optional[str] = None
    key_concepts: Optional[list[dict[str, Any]]] = None
    code_snippets: Optional[list[dict[str, Any]]] = None
    structured_content: Optional[str] = None
    token_count_original: Optional[int] = None
    token_count_distilled: Optional[int] = None
    distill_model: Optional[str] = None
    distill_date: Optional[datetime] = None
    review_status: ReviewStatus = ReviewStatus.pending

    @computed_field
    @property
    def compression_ratio(self) -> Optional[float]:
        if self.token_count_original and self.token_count_distilled:
            return round(self.token_count_distilled / self.token_count_original * 100, 2)
        return None


class ReviewLog(BaseModel):
    review_id: Optional[int] = None
    distill_id: int
    review_round: int = 1
    reviewer_type: ReviewerType
    quality_score: Optional[float] = None
    assessment: Optional[dict[str, float]] = None
    decision: Decision
    feedback: Optional[str] = None
    refactor_instructions: Optional[str] = None
    research_queries: Optional[list[str]] = None
    reviewed_at: Optional[datetime] = None


class Topic(BaseModel):
    topic_id: Optional[int] = None
    topic_name: str
    topic_slug: str
    parent_topic_id: Optional[int] = None
    description: Optional[str] = None


class DocumentTopic(BaseModel):
    doc_id: int
    topic_id: int
    relevance_score: float = 1.0


class ExportJob(BaseModel):
    export_id: Optional[int] = None
    export_name: str
    export_type: ExportType
    output_format: OutputFormat = OutputFormat.markdown
    topic_filter: Optional[list[int]] = None
    date_range_start: Optional[date] = None
    date_range_end: Optional[date] = None
    min_quality_score: float = 0.80
    output_path: Optional[str] = None
    total_documents: Optional[int] = None
    total_tokens: Optional[int] = None
    status: str = "pending"
    started_at: Optional[datetime] = None
    completed_at: Optional[datetime] = None
    error_message: Optional[str] = None
    created_at: Optional[datetime] = None


class PipelineRun(BaseModel):
    run_id: Optional[int] = None
    run_type: RunType
    input_value: str
    status: PipelineStatus = PipelineStatus.running
    current_stage: PipelineStage = PipelineStage.discovery
    options: Optional[dict[str, Any]] = None
    stats: Optional[dict[str, int]] = Field(default_factory=lambda: {
        "sources_discovered": 0,
        "pages_crawled": 0,
        "documents_stored": 0,
        "documents_distilled": 0,
        "approved": 0,
        "refactored": 0,
        "deep_researched": 0,
        "rejected": 0,
        "needs_manual_review": 0,
    })
    export_path: Optional[str] = None
    export_document_count: Optional[int] = None
    started_at: Optional[datetime] = None
    completed_at: Optional[datetime] = None
    error_message: Optional[str] = None
    error_stage: Optional[str] = None


class PipelineIterationTracker(BaseModel):
    tracker_id: Optional[int] = None
    run_id: int
    doc_id: int
    refactor_count: int = 0
    deep_research_count: int = 0
    final_decision: Optional[FinalDecision] = None
    created_at: Optional[datetime] = None
    updated_at: Optional[datetime] = None


# --- Non-DB Models (manifest/crawl/assessment) ---

class ManifestURL(BaseModel):
    url: str
    title: Optional[str] = None
    credibility_tier: Optional[str] = None
    credibility_score: Optional[float] = None
    source_type: Optional[str] = None
    vendor: Optional[str] = None


class Manifest(BaseModel):
    discovery_date: Optional[str] = None
    topic: Optional[str] = None
    total_urls: int = 0
    urls: list[ManifestURL] = Field(default_factory=list)


class CrawlResultEntry(BaseModel):
    url: str
    title: Optional[str] = None
    raw_path: str
    content_size: int = 0
    status: str = "completed"
    error: Optional[str] = None


class CrawlResult(BaseModel):
    crawl_date: Optional[str] = None
    crawler_used: str = "firecrawl"
    total_crawled: int = 0
    total_failed: int = 0
    documents: list[CrawlResultEntry] = Field(default_factory=list)


class QAAssessment(BaseModel):
    """Legacy model for post-distillation Claude self-review (deprecated)."""
    accuracy: float = 0.0
    completeness: float = 0.0
    clarity: float = 0.0
    prompt_engineering_quality: float = 0.0
    usability: float = 0.0

    @computed_field
    @property
    def weighted_score(self) -> float:
        return round(
            self.accuracy * 0.25
            + self.completeness * 0.20
            + self.clarity * 0.20
            + self.prompt_engineering_quality * 0.25
            + self.usability * 0.10,
            4,
        )


class SourceQAAssessment(BaseModel):
    """Pre-distillation source quality assessment via Gemini."""
    relevance: float = 0.0
    authority: float = 0.0
    completeness: float = 0.0
    freshness: float = 0.0
    distill_value: float = 0.0
    verdict: str = ""
    reason: str = ""

    @computed_field
    @property
    def weighted_score(self) -> float:
        return round(
            self.relevance * 0.25
            + self.authority * 0.25
            + self.completeness * 0.20
            + self.freshness * 0.15
            + self.distill_value * 0.15,
            4,
        )