"""Pydantic v2 models matching the reference_library MySQL schema.""" from __future__ import annotations from datetime import date, datetime from enum import Enum from typing import Any, Optional from pydantic import BaseModel, Field, computed_field # --- Enums matching MySQL ENUMs --- class SourceType(str, Enum): official_docs = "official_docs" engineering_blog = "engineering_blog" research_paper = "research_paper" github_repo = "github_repo" community_guide = "community_guide" pdf_document = "pdf_document" api_reference = "api_reference" class CredibilityTier(str, Enum): tier1_official = "tier1_official" tier2_verified = "tier2_verified" tier3_community = "tier3_community" class DocType(str, Enum): webpage = "webpage" pdf = "pdf" markdown = "markdown" api_spec = "api_spec" code_sample = "code_sample" class Language(str, Enum): en = "en" ko = "ko" mixed = "mixed" class CrawlMethod(str, Enum): firecrawl = "firecrawl" scrapy = "scrapy" aiohttp = "aiohttp" nodejs = "nodejs" manual = "manual" api = "api" class CrawlStatus(str, Enum): pending = "pending" completed = "completed" failed = "failed" stale = "stale" class ReviewStatus(str, Enum): pending = "pending" in_review = "in_review" approved = "approved" needs_refactor = "needs_refactor" rejected = "rejected" class ReviewerType(str, Enum): auto_qa = "auto_qa" human = "human" claude_review = "claude_review" gemini_review = "gemini_review" class Decision(str, Enum): approve = "approve" refactor = "refactor" deep_research = "deep_research" reject = "reject" class PipelineStatus(str, Enum): running = "running" completed = "completed" failed = "failed" paused = "paused" class PipelineStage(str, Enum): discovery = "discovery" crawling = "crawling" storing = "storing" evaluating = "evaluating" distilling = "distilling" exporting = "exporting" class RunType(str, Enum): topic = "topic" urls = "urls" manifest = "manifest" class ExportType(str, Enum): project_files = "project_files" fine_tuning = "fine_tuning" training_dataset = "training_dataset" knowledge_base = "knowledge_base" class OutputFormat(str, Enum): markdown = "markdown" jsonl = "jsonl" parquet = "parquet" sqlite = "sqlite" class Frequency(str, Enum): daily = "daily" weekly = "weekly" biweekly = "biweekly" monthly = "monthly" on_demand = "on_demand" class ChangeType(str, Enum): content_updated = "content_updated" url_moved = "url_moved" deleted = "deleted" new_version = "new_version" class FinalDecision(str, Enum): approved = "approved" rejected = "rejected" needs_manual_review = "needs_manual_review" # --- Core Table Models --- class Source(BaseModel): source_id: Optional[int] = None source_name: str source_type: SourceType base_url: Optional[str] = None credibility_tier: CredibilityTier = CredibilityTier.tier3_community vendor: Optional[str] = None is_active: bool = True created_at: Optional[datetime] = None updated_at: Optional[datetime] = None class Document(BaseModel): doc_id: Optional[int] = None source_id: int title: str url: Optional[str] = None url_hash: Optional[str] = None # Generated column in MySQL doc_type: DocType language: Language = Language.en original_publish_date: Optional[date] = None last_modified_date: Optional[date] = None crawl_date: Optional[datetime] = None crawl_method: CrawlMethod = CrawlMethod.firecrawl crawl_status: CrawlStatus = CrawlStatus.pending raw_content_path: Optional[str] = None raw_content_size: Optional[int] = None version: int = 1 previous_version_id: Optional[int] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None class DistilledContent(BaseModel): distill_id: Optional[int] = None doc_id: int summary: Optional[str] = None key_concepts: Optional[list[dict[str, Any]]] = None code_snippets: Optional[list[dict[str, Any]]] = None structured_content: Optional[str] = None token_count_original: Optional[int] = None token_count_distilled: Optional[int] = None distill_model: Optional[str] = None distill_date: Optional[datetime] = None review_status: ReviewStatus = ReviewStatus.pending @computed_field @property def compression_ratio(self) -> Optional[float]: if self.token_count_original and self.token_count_distilled: return round(self.token_count_distilled / self.token_count_original * 100, 2) return None class ReviewLog(BaseModel): review_id: Optional[int] = None distill_id: int review_round: int = 1 reviewer_type: ReviewerType quality_score: Optional[float] = None assessment: Optional[dict[str, float]] = None decision: Decision feedback: Optional[str] = None refactor_instructions: Optional[str] = None research_queries: Optional[list[str]] = None reviewed_at: Optional[datetime] = None class Topic(BaseModel): topic_id: Optional[int] = None topic_name: str topic_slug: str parent_topic_id: Optional[int] = None description: Optional[str] = None class DocumentTopic(BaseModel): doc_id: int topic_id: int relevance_score: float = 1.0 class ExportJob(BaseModel): export_id: Optional[int] = None export_name: str export_type: ExportType output_format: OutputFormat = OutputFormat.markdown topic_filter: Optional[list[int]] = None date_range_start: Optional[date] = None date_range_end: Optional[date] = None min_quality_score: float = 0.80 output_path: Optional[str] = None total_documents: Optional[int] = None total_tokens: Optional[int] = None status: str = "pending" started_at: Optional[datetime] = None completed_at: Optional[datetime] = None error_message: Optional[str] = None created_at: Optional[datetime] = None class PipelineRun(BaseModel): run_id: Optional[int] = None run_type: RunType input_value: str status: PipelineStatus = PipelineStatus.running current_stage: PipelineStage = PipelineStage.discovery options: Optional[dict[str, Any]] = None stats: Optional[dict[str, int]] = Field(default_factory=lambda: { "sources_discovered": 0, "pages_crawled": 0, "documents_stored": 0, "documents_distilled": 0, "approved": 0, "refactored": 0, "deep_researched": 0, "rejected": 0, "needs_manual_review": 0, }) export_path: Optional[str] = None export_document_count: Optional[int] = None started_at: Optional[datetime] = None completed_at: Optional[datetime] = None error_message: Optional[str] = None error_stage: Optional[str] = None class PipelineIterationTracker(BaseModel): tracker_id: Optional[int] = None run_id: int doc_id: int refactor_count: int = 0 deep_research_count: int = 0 final_decision: Optional[FinalDecision] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None # --- Non-DB Models (manifest/crawl/assessment) --- class ManifestURL(BaseModel): url: str title: Optional[str] = None credibility_tier: Optional[str] = None credibility_score: Optional[float] = None source_type: Optional[str] = None vendor: Optional[str] = None class Manifest(BaseModel): discovery_date: Optional[str] = None topic: Optional[str] = None total_urls: int = 0 urls: list[ManifestURL] = Field(default_factory=list) class CrawlResultEntry(BaseModel): url: str title: Optional[str] = None raw_path: str content_size: int = 0 status: str = "completed" error: Optional[str] = None class CrawlResult(BaseModel): crawl_date: Optional[str] = None crawler_used: str = "firecrawl" total_crawled: int = 0 total_failed: int = 0 documents: list[CrawlResultEntry] = Field(default_factory=list) class QAAssessment(BaseModel): """Legacy model for post-distillation Claude self-review (deprecated).""" accuracy: float = 0.0 completeness: float = 0.0 clarity: float = 0.0 prompt_engineering_quality: float = 0.0 usability: float = 0.0 @computed_field @property def weighted_score(self) -> float: return round( self.accuracy * 0.25 + self.completeness * 0.20 + self.clarity * 0.20 + self.prompt_engineering_quality * 0.25 + self.usability * 0.10, 4, ) class SourceQAAssessment(BaseModel): """Pre-distillation source quality assessment via Gemini.""" relevance: float = 0.0 authority: float = 0.0 completeness: float = 0.0 freshness: float = 0.0 distill_value: float = 0.0 verdict: str = "" reason: str = "" @computed_field @property def weighted_score(self) -> float: return round( self.relevance * 0.25 + self.authority * 0.25 + self.completeness * 0.20 + self.freshness * 0.15 + self.distill_value * 0.15, 4, )