feat(reference-curator): implement Python scripts + Gemini quality gate
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Web Crawler Manager CLI — manage crawl results and crawler selection.
|
||||
|
||||
Claude performs actual crawling via Firecrawl MCP. This script handles:
|
||||
- Storing crawl results metadata
|
||||
- Selecting the optimal crawler backend
|
||||
- Generating crawl result manifests
|
||||
|
||||
Usage:
|
||||
python crawl_mgr.py store-result --manifest manifest.json --raw-dir ~/reference-library/raw/
|
||||
python crawl_mgr.py select-crawler --url "https://docs.anthropic.com"
|
||||
python crawl_mgr.py list-crawls [--status completed]
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.manifest import create_crawl_result, write_crawl_result, read_manifest
|
||||
from refcurator.utils import count_tokens
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
# Known SPA frameworks and their indicators
|
||||
SPA_INDICATORS = {
|
||||
"react": ["react", "next.js", "nextjs", "gatsby", "remix"],
|
||||
"vue": ["vue", "nuxt"],
|
||||
"angular": ["angular"],
|
||||
"svelte": ["svelte", "sveltekit"],
|
||||
}
|
||||
|
||||
# Known static site generators
|
||||
STATIC_INDICATORS = ["hugo", "jekyll", "mkdocs", "docusaurus", "gitbook", "sphinx",
|
||||
"readthedocs", "vuepress", "docsify"]
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Web Crawler Manager — manage crawl results and backend selection."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("store-result")
|
||||
@click.option("--raw-dir", required=True, type=click.Path(exists=True),
|
||||
help="Directory containing crawled raw files")
|
||||
@click.option("--crawler", default="firecrawl",
|
||||
type=click.Choice(["firecrawl", "nodejs", "aiohttp", "scrapy"]))
|
||||
@click.option("--source-id", type=int, help="Source ID to associate documents with")
|
||||
@click.option("--output", type=click.Path(), help="Output crawl result JSON path")
|
||||
def store_result(raw_dir, crawler, source_id, output):
|
||||
"""Store crawl results from raw files into the repository.
|
||||
|
||||
Scans raw_dir for .md files, creates document records, and generates a crawl result manifest.
|
||||
"""
|
||||
raw_path = Path(raw_dir)
|
||||
md_files = sorted(raw_path.glob("**/*.md"))
|
||||
|
||||
if not md_files:
|
||||
console.print(f"[yellow]No .md files found in {raw_dir}[/yellow]")
|
||||
sys.exit(0)
|
||||
|
||||
entries = []
|
||||
stored = 0
|
||||
|
||||
with db_session() as db:
|
||||
for md_file in md_files:
|
||||
content = md_file.read_text(errors="replace")
|
||||
content_size = md_file.stat().st_size
|
||||
|
||||
# Extract title from first heading or filename
|
||||
title = _extract_title(content, md_file.stem)
|
||||
|
||||
# Extract URL from frontmatter if available
|
||||
url = _extract_url_from_content(content)
|
||||
|
||||
entry = {
|
||||
"url": url or f"file://{md_file.resolve()}",
|
||||
"title": title,
|
||||
"raw_path": str(md_file.resolve()),
|
||||
"content_size": content_size,
|
||||
"status": "completed",
|
||||
}
|
||||
entries.append(entry)
|
||||
|
||||
if source_id:
|
||||
db.insert_returning_id(
|
||||
"""INSERT INTO documents
|
||||
(source_id, title, url, doc_type, crawl_date,
|
||||
crawl_method, crawl_status, raw_content_path, raw_content_size)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(source_id, title, url, "markdown", datetime.now().isoformat(),
|
||||
crawler, "completed", str(md_file.resolve()), content_size),
|
||||
)
|
||||
stored += 1
|
||||
|
||||
result = create_crawl_result(entries, crawler)
|
||||
|
||||
if output:
|
||||
write_crawl_result(result, Path(output))
|
||||
console.print(f"[green]Crawl result written to {output}[/green]")
|
||||
|
||||
console.print(
|
||||
f"[green]Processed {len(entries)} files[/green]"
|
||||
+ (f", stored {stored} documents" if stored else "")
|
||||
)
|
||||
click.echo(result.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@cli.command("select-crawler")
|
||||
@click.option("--url", required=True, help="Target URL to analyze")
|
||||
@click.option("--page-count", type=int, help="Expected page count")
|
||||
@click.option("--json-output", is_flag=True, help="Output as JSON")
|
||||
def select_crawler(url, page_count, json_output):
|
||||
"""Recommend the best crawler backend for a URL.
|
||||
|
||||
Analyzes URL patterns and site characteristics to suggest:
|
||||
- firecrawl: SPAs, JS-rendered content
|
||||
- nodejs: Small static docs sites (<=50 pages)
|
||||
- aiohttp: Medium technical docs (<=200 pages)
|
||||
- scrapy: Large enterprise sites (>200 pages)
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.netloc.lower()
|
||||
path = parsed.path.lower()
|
||||
|
||||
recommendation = "firecrawl" # Default
|
||||
reason = "Default recommendation for general crawling"
|
||||
confidence = 0.7
|
||||
|
||||
# Check for SPA indicators in domain/path
|
||||
url_lower = url.lower()
|
||||
for framework, indicators in SPA_INDICATORS.items():
|
||||
if any(ind in url_lower or ind in domain for ind in indicators):
|
||||
recommendation = "firecrawl"
|
||||
reason = f"Detected {framework} SPA framework — needs JS rendering"
|
||||
confidence = 0.9
|
||||
break
|
||||
|
||||
# Check for static site indicators
|
||||
for gen in STATIC_INDICATORS:
|
||||
if gen in url_lower or gen in domain:
|
||||
if page_count and page_count <= 50:
|
||||
recommendation = "nodejs"
|
||||
reason = f"Static site ({gen}), small page count — lightweight crawler sufficient"
|
||||
else:
|
||||
recommendation = "aiohttp"
|
||||
reason = f"Static site ({gen}), moderate size — async crawler recommended"
|
||||
confidence = 0.85
|
||||
break
|
||||
|
||||
# Page count override (takes precedence over default, not over SPA detection)
|
||||
spa_detected = confidence >= 0.9 and recommendation == "firecrawl"
|
||||
if page_count and not spa_detected:
|
||||
if page_count > 200:
|
||||
recommendation = "scrapy"
|
||||
reason = f"Large site ({page_count} pages) — enterprise crawler needed"
|
||||
confidence = 0.9
|
||||
elif page_count <= 50:
|
||||
recommendation = "nodejs"
|
||||
reason = f"Small site ({page_count} pages) — lightweight crawler sufficient"
|
||||
confidence = 0.8
|
||||
|
||||
# Known documentation platforms → firecrawl
|
||||
doc_platforms = ["docs.", "developer.", "api.", "reference."]
|
||||
if any(domain.startswith(p) for p in doc_platforms):
|
||||
if recommendation == "firecrawl":
|
||||
reason = "Documentation platform — Firecrawl handles dynamic docs well"
|
||||
confidence = 0.85
|
||||
|
||||
result = {
|
||||
"url": url,
|
||||
"recommendation": recommendation,
|
||||
"reason": reason,
|
||||
"confidence": confidence,
|
||||
"alternatives": _get_alternatives(recommendation),
|
||||
}
|
||||
|
||||
if json_output:
|
||||
click.echo(json.dumps(result, indent=2))
|
||||
else:
|
||||
console.print(f"\n[bold]Crawler Recommendation for:[/bold] {url}")
|
||||
console.print(f" [green]Recommended:[/green] {recommendation}")
|
||||
console.print(f" [dim]Reason:[/dim] {reason}")
|
||||
console.print(f" [dim]Confidence:[/dim] {confidence:.0%}")
|
||||
if result["alternatives"]:
|
||||
console.print(f" [dim]Alternatives:[/dim] {', '.join(result['alternatives'])}")
|
||||
|
||||
|
||||
@cli.command("list-crawls")
|
||||
@click.option("--status", type=click.Choice(["pending", "completed", "failed", "stale"]),
|
||||
help="Filter by crawl status")
|
||||
@click.option("--limit", default=20, type=int, help="Max results")
|
||||
def list_crawls(status, limit):
|
||||
"""List recent crawl records."""
|
||||
with db_session() as db:
|
||||
if status:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date
|
||||
FROM documents WHERE crawl_status = %s
|
||||
ORDER BY crawl_date DESC LIMIT %s""",
|
||||
(status, limit),
|
||||
)
|
||||
else:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date
|
||||
FROM documents ORDER BY crawl_date DESC LIMIT %s""",
|
||||
(limit,),
|
||||
)
|
||||
|
||||
table = Table(title="Crawl Records")
|
||||
table.add_column("ID", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("Crawler")
|
||||
table.add_column("Status")
|
||||
table.add_column("Date")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:40],
|
||||
str(r.get("crawl_method", "")),
|
||||
str(r.get("crawl_status", "")),
|
||||
str(r.get("crawl_date", ""))[:10],
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
def _extract_title(content: str, fallback: str) -> str:
|
||||
"""Extract title from markdown content."""
|
||||
for line in content.split("\n")[:10]:
|
||||
line = line.strip()
|
||||
if line.startswith("# ") and not line.startswith("##"):
|
||||
return line[2:].strip()
|
||||
return fallback.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
|
||||
def _extract_url_from_content(content: str) -> str | None:
|
||||
"""Extract source URL from markdown frontmatter or content."""
|
||||
# YAML frontmatter
|
||||
if content.startswith("---"):
|
||||
fm_end = content.find("---", 3)
|
||||
if fm_end > 0:
|
||||
fm = content[3:fm_end]
|
||||
for line in fm.split("\n"):
|
||||
if line.strip().startswith("url:") or line.strip().startswith("source:"):
|
||||
url = line.split(":", 1)[1].strip().strip("'\"")
|
||||
if url.startswith("http"):
|
||||
return url
|
||||
|
||||
# Source: URL pattern in content
|
||||
m = re.search(r"\*\*Source:\*\*\s*(https?://\S+)", content[:500])
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _get_alternatives(primary: str) -> list[str]:
|
||||
"""Get alternative crawler recommendations."""
|
||||
all_crawlers = ["firecrawl", "nodejs", "aiohttp", "scrapy"]
|
||||
return [c for c in all_crawlers if c != primary][:2]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
Reference in New Issue
Block a user