feat(reference-curator): implement Python scripts + Gemini quality gate
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
"""Pydantic v2 models matching the reference_library MySQL schema."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, computed_field
|
||||
|
||||
|
||||
# --- Enums matching MySQL ENUMs ---
|
||||
|
||||
class SourceType(str, Enum):
|
||||
official_docs = "official_docs"
|
||||
engineering_blog = "engineering_blog"
|
||||
research_paper = "research_paper"
|
||||
github_repo = "github_repo"
|
||||
community_guide = "community_guide"
|
||||
pdf_document = "pdf_document"
|
||||
api_reference = "api_reference"
|
||||
|
||||
|
||||
class CredibilityTier(str, Enum):
|
||||
tier1_official = "tier1_official"
|
||||
tier2_verified = "tier2_verified"
|
||||
tier3_community = "tier3_community"
|
||||
|
||||
|
||||
class DocType(str, Enum):
|
||||
webpage = "webpage"
|
||||
pdf = "pdf"
|
||||
markdown = "markdown"
|
||||
api_spec = "api_spec"
|
||||
code_sample = "code_sample"
|
||||
|
||||
|
||||
class Language(str, Enum):
|
||||
en = "en"
|
||||
ko = "ko"
|
||||
mixed = "mixed"
|
||||
|
||||
|
||||
class CrawlMethod(str, Enum):
|
||||
firecrawl = "firecrawl"
|
||||
scrapy = "scrapy"
|
||||
aiohttp = "aiohttp"
|
||||
nodejs = "nodejs"
|
||||
manual = "manual"
|
||||
api = "api"
|
||||
|
||||
|
||||
class CrawlStatus(str, Enum):
|
||||
pending = "pending"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
stale = "stale"
|
||||
|
||||
|
||||
class ReviewStatus(str, Enum):
|
||||
pending = "pending"
|
||||
in_review = "in_review"
|
||||
approved = "approved"
|
||||
needs_refactor = "needs_refactor"
|
||||
rejected = "rejected"
|
||||
|
||||
|
||||
class ReviewerType(str, Enum):
|
||||
auto_qa = "auto_qa"
|
||||
human = "human"
|
||||
claude_review = "claude_review"
|
||||
gemini_review = "gemini_review"
|
||||
|
||||
|
||||
class Decision(str, Enum):
|
||||
approve = "approve"
|
||||
refactor = "refactor"
|
||||
deep_research = "deep_research"
|
||||
reject = "reject"
|
||||
|
||||
|
||||
class PipelineStatus(str, Enum):
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
paused = "paused"
|
||||
|
||||
|
||||
class PipelineStage(str, Enum):
|
||||
discovery = "discovery"
|
||||
crawling = "crawling"
|
||||
storing = "storing"
|
||||
evaluating = "evaluating"
|
||||
distilling = "distilling"
|
||||
exporting = "exporting"
|
||||
|
||||
|
||||
class RunType(str, Enum):
|
||||
topic = "topic"
|
||||
urls = "urls"
|
||||
manifest = "manifest"
|
||||
|
||||
|
||||
class ExportType(str, Enum):
|
||||
project_files = "project_files"
|
||||
fine_tuning = "fine_tuning"
|
||||
training_dataset = "training_dataset"
|
||||
knowledge_base = "knowledge_base"
|
||||
|
||||
|
||||
class OutputFormat(str, Enum):
|
||||
markdown = "markdown"
|
||||
jsonl = "jsonl"
|
||||
parquet = "parquet"
|
||||
sqlite = "sqlite"
|
||||
|
||||
|
||||
class Frequency(str, Enum):
|
||||
daily = "daily"
|
||||
weekly = "weekly"
|
||||
biweekly = "biweekly"
|
||||
monthly = "monthly"
|
||||
on_demand = "on_demand"
|
||||
|
||||
|
||||
class ChangeType(str, Enum):
|
||||
content_updated = "content_updated"
|
||||
url_moved = "url_moved"
|
||||
deleted = "deleted"
|
||||
new_version = "new_version"
|
||||
|
||||
|
||||
class FinalDecision(str, Enum):
|
||||
approved = "approved"
|
||||
rejected = "rejected"
|
||||
needs_manual_review = "needs_manual_review"
|
||||
|
||||
|
||||
# --- Core Table Models ---
|
||||
|
||||
class Source(BaseModel):
|
||||
source_id: Optional[int] = None
|
||||
source_name: str
|
||||
source_type: SourceType
|
||||
base_url: Optional[str] = None
|
||||
credibility_tier: CredibilityTier = CredibilityTier.tier3_community
|
||||
vendor: Optional[str] = None
|
||||
is_active: bool = True
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class Document(BaseModel):
|
||||
doc_id: Optional[int] = None
|
||||
source_id: int
|
||||
title: str
|
||||
url: Optional[str] = None
|
||||
url_hash: Optional[str] = None # Generated column in MySQL
|
||||
doc_type: DocType
|
||||
language: Language = Language.en
|
||||
original_publish_date: Optional[date] = None
|
||||
last_modified_date: Optional[date] = None
|
||||
crawl_date: Optional[datetime] = None
|
||||
crawl_method: CrawlMethod = CrawlMethod.firecrawl
|
||||
crawl_status: CrawlStatus = CrawlStatus.pending
|
||||
raw_content_path: Optional[str] = None
|
||||
raw_content_size: Optional[int] = None
|
||||
version: int = 1
|
||||
previous_version_id: Optional[int] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class DistilledContent(BaseModel):
|
||||
distill_id: Optional[int] = None
|
||||
doc_id: int
|
||||
summary: Optional[str] = None
|
||||
key_concepts: Optional[list[dict[str, Any]]] = None
|
||||
code_snippets: Optional[list[dict[str, Any]]] = None
|
||||
structured_content: Optional[str] = None
|
||||
token_count_original: Optional[int] = None
|
||||
token_count_distilled: Optional[int] = None
|
||||
distill_model: Optional[str] = None
|
||||
distill_date: Optional[datetime] = None
|
||||
review_status: ReviewStatus = ReviewStatus.pending
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def compression_ratio(self) -> Optional[float]:
|
||||
if self.token_count_original and self.token_count_distilled:
|
||||
return round(self.token_count_distilled / self.token_count_original * 100, 2)
|
||||
return None
|
||||
|
||||
|
||||
class ReviewLog(BaseModel):
|
||||
review_id: Optional[int] = None
|
||||
distill_id: int
|
||||
review_round: int = 1
|
||||
reviewer_type: ReviewerType
|
||||
quality_score: Optional[float] = None
|
||||
assessment: Optional[dict[str, float]] = None
|
||||
decision: Decision
|
||||
feedback: Optional[str] = None
|
||||
refactor_instructions: Optional[str] = None
|
||||
research_queries: Optional[list[str]] = None
|
||||
reviewed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class Topic(BaseModel):
|
||||
topic_id: Optional[int] = None
|
||||
topic_name: str
|
||||
topic_slug: str
|
||||
parent_topic_id: Optional[int] = None
|
||||
description: Optional[str] = None
|
||||
|
||||
|
||||
class DocumentTopic(BaseModel):
|
||||
doc_id: int
|
||||
topic_id: int
|
||||
relevance_score: float = 1.0
|
||||
|
||||
|
||||
class ExportJob(BaseModel):
|
||||
export_id: Optional[int] = None
|
||||
export_name: str
|
||||
export_type: ExportType
|
||||
output_format: OutputFormat = OutputFormat.markdown
|
||||
topic_filter: Optional[list[int]] = None
|
||||
date_range_start: Optional[date] = None
|
||||
date_range_end: Optional[date] = None
|
||||
min_quality_score: float = 0.80
|
||||
output_path: Optional[str] = None
|
||||
total_documents: Optional[int] = None
|
||||
total_tokens: Optional[int] = None
|
||||
status: str = "pending"
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class PipelineRun(BaseModel):
|
||||
run_id: Optional[int] = None
|
||||
run_type: RunType
|
||||
input_value: str
|
||||
status: PipelineStatus = PipelineStatus.running
|
||||
current_stage: PipelineStage = PipelineStage.discovery
|
||||
options: Optional[dict[str, Any]] = None
|
||||
stats: Optional[dict[str, int]] = Field(default_factory=lambda: {
|
||||
"sources_discovered": 0,
|
||||
"pages_crawled": 0,
|
||||
"documents_stored": 0,
|
||||
"documents_distilled": 0,
|
||||
"approved": 0,
|
||||
"refactored": 0,
|
||||
"deep_researched": 0,
|
||||
"rejected": 0,
|
||||
"needs_manual_review": 0,
|
||||
})
|
||||
export_path: Optional[str] = None
|
||||
export_document_count: Optional[int] = None
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
error_stage: Optional[str] = None
|
||||
|
||||
|
||||
class PipelineIterationTracker(BaseModel):
|
||||
tracker_id: Optional[int] = None
|
||||
run_id: int
|
||||
doc_id: int
|
||||
refactor_count: int = 0
|
||||
deep_research_count: int = 0
|
||||
final_decision: Optional[FinalDecision] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
# --- Non-DB Models (manifest/crawl/assessment) ---
|
||||
|
||||
class ManifestURL(BaseModel):
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
credibility_tier: Optional[str] = None
|
||||
credibility_score: Optional[float] = None
|
||||
source_type: Optional[str] = None
|
||||
vendor: Optional[str] = None
|
||||
|
||||
|
||||
class Manifest(BaseModel):
|
||||
discovery_date: Optional[str] = None
|
||||
topic: Optional[str] = None
|
||||
total_urls: int = 0
|
||||
urls: list[ManifestURL] = Field(default_factory=list)
|
||||
|
||||
|
||||
class CrawlResultEntry(BaseModel):
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
raw_path: str
|
||||
content_size: int = 0
|
||||
status: str = "completed"
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class CrawlResult(BaseModel):
|
||||
crawl_date: Optional[str] = None
|
||||
crawler_used: str = "firecrawl"
|
||||
total_crawled: int = 0
|
||||
total_failed: int = 0
|
||||
documents: list[CrawlResultEntry] = Field(default_factory=list)
|
||||
|
||||
|
||||
class QAAssessment(BaseModel):
|
||||
"""Legacy model for post-distillation Claude self-review (deprecated)."""
|
||||
accuracy: float = 0.0
|
||||
completeness: float = 0.0
|
||||
clarity: float = 0.0
|
||||
prompt_engineering_quality: float = 0.0
|
||||
usability: float = 0.0
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def weighted_score(self) -> float:
|
||||
return round(
|
||||
self.accuracy * 0.25
|
||||
+ self.completeness * 0.20
|
||||
+ self.clarity * 0.20
|
||||
+ self.prompt_engineering_quality * 0.25
|
||||
+ self.usability * 0.10,
|
||||
4,
|
||||
)
|
||||
|
||||
|
||||
class SourceQAAssessment(BaseModel):
|
||||
"""Pre-distillation source quality assessment via Gemini."""
|
||||
relevance: float = 0.0
|
||||
authority: float = 0.0
|
||||
completeness: float = 0.0
|
||||
freshness: float = 0.0
|
||||
distill_value: float = 0.0
|
||||
verdict: str = ""
|
||||
reason: str = ""
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def weighted_score(self) -> float:
|
||||
return round(
|
||||
self.relevance * 0.25
|
||||
+ self.authority * 0.25
|
||||
+ self.completeness * 0.20
|
||||
+ self.freshness * 0.15
|
||||
+ self.distill_value * 0.15,
|
||||
4,
|
||||
)
|
||||
Reference in New Issue
Block a user