Andrew Yim
f215c11c32
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
329 lines
12 KiB
Python
329 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Content Repository CLI — CRUD operations for the reference library.
|
|
|
|
Usage:
|
|
python repo.py store --source-id 1 --title "Doc" --url "https://..." --doc-type webpage --raw-path /path/to/file
|
|
python repo.py check-dup --url "https://..."
|
|
python repo.py query-topic --topic-slug prompt-engineering [--min-quality 0.80]
|
|
python repo.py find-stale [--output stale.json]
|
|
python repo.py pending-reviews [--output pending.json]
|
|
python repo.py export-ready [--min-score 0.85] [--output ready.json]
|
|
python repo.py stats
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import click
|
|
from rich.console import Console
|
|
from rich.table import Table
|
|
|
|
from refcurator.db import db_session
|
|
from refcurator.utils import url_hash, normalize_url
|
|
|
|
console = Console()
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
"""Content Repository — manage documents in the reference library."""
|
|
pass
|
|
|
|
|
|
@cli.command()
|
|
@click.option("--source-id", required=True, type=int, help="Source ID from sources table")
|
|
@click.option("--title", required=True, help="Document title")
|
|
@click.option("--url", required=True, help="Document URL")
|
|
@click.option("--doc-type", required=True,
|
|
type=click.Choice(["webpage", "pdf", "markdown", "api_spec", "code_sample"]))
|
|
@click.option("--raw-path", required=True, type=click.Path(), help="Path to raw content file")
|
|
@click.option("--crawl-method", default="firecrawl",
|
|
type=click.Choice(["firecrawl", "scrapy", "aiohttp", "nodejs", "manual", "api"]))
|
|
@click.option("--language", default="en", type=click.Choice(["en", "ko", "mixed"]))
|
|
def store(source_id, title, url, doc_type, raw_path, crawl_method, language):
|
|
"""Store a new document in the repository."""
|
|
raw = Path(raw_path)
|
|
if not raw.is_file():
|
|
console.print(f"[red]Error:[/red] Raw file not found: {raw_path}")
|
|
sys.exit(1)
|
|
|
|
content_size = raw.stat().st_size
|
|
|
|
with db_session() as db:
|
|
# Check for duplicate
|
|
existing = db.fetch_one(
|
|
"SELECT doc_id, title FROM documents WHERE url_hash = %s",
|
|
(url_hash(url),),
|
|
)
|
|
if existing:
|
|
console.print(
|
|
f"[yellow]Duplicate:[/yellow] URL already stored as doc_id={existing['doc_id']} "
|
|
f"({existing['title']})"
|
|
)
|
|
sys.exit(0)
|
|
|
|
doc_id = db.insert_returning_id(
|
|
"""INSERT INTO documents
|
|
(source_id, title, url, doc_type, language, crawl_date,
|
|
crawl_method, crawl_status, raw_content_path, raw_content_size)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
|
(source_id, title, url, doc_type, language, datetime.now().isoformat(),
|
|
crawl_method, "completed", str(raw.resolve()), content_size),
|
|
)
|
|
|
|
console.print(f"[green]Stored:[/green] doc_id={doc_id} — {title}")
|
|
click.echo(json.dumps({"doc_id": doc_id, "title": title, "url": url}))
|
|
|
|
|
|
@cli.command("check-dup")
|
|
@click.option("--url", required=True, help="URL to check for duplicates")
|
|
def check_dup(url):
|
|
"""Check if a URL already exists in the repository."""
|
|
h = url_hash(url)
|
|
|
|
with db_session() as db:
|
|
existing = db.fetch_one(
|
|
"SELECT doc_id, title, url, crawl_status FROM documents WHERE url_hash = %s",
|
|
(h,),
|
|
)
|
|
|
|
if existing:
|
|
console.print(f"[yellow]Duplicate found:[/yellow] doc_id={existing['doc_id']}")
|
|
click.echo(json.dumps(existing, default=str))
|
|
else:
|
|
console.print("[green]No duplicate found.[/green]")
|
|
click.echo(json.dumps({"duplicate": False, "url": url}))
|
|
|
|
|
|
@cli.command("query-topic")
|
|
@click.option("--topic-slug", required=True, help="Topic slug to query")
|
|
@click.option("--min-quality", default=0.0, type=float, help="Minimum quality score")
|
|
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
|
def query_topic(topic_slug, min_quality, output):
|
|
"""Query documents by topic."""
|
|
with db_session() as db:
|
|
rows = db.fetch_all(
|
|
"""SELECT d.doc_id, d.title, d.url, d.crawl_status,
|
|
dt.relevance_score, t.topic_name
|
|
FROM documents d
|
|
JOIN document_topics dt ON d.doc_id = dt.doc_id
|
|
JOIN topics t ON dt.topic_id = t.topic_id
|
|
WHERE t.topic_slug = %s
|
|
ORDER BY dt.relevance_score DESC""",
|
|
(topic_slug,),
|
|
)
|
|
|
|
if min_quality > 0:
|
|
# Filter by review score if available
|
|
rows = [r for r in rows if r.get("relevance_score", 0) >= min_quality]
|
|
|
|
if output:
|
|
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
|
console.print(f"[green]Wrote {len(rows)} results to {output}[/green]")
|
|
else:
|
|
_print_doc_table(rows, f"Topic: {topic_slug}")
|
|
|
|
|
|
@cli.command("find-stale")
|
|
@click.option("--days", default=30, type=int, help="Documents older than N days")
|
|
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
|
def find_stale(days, output):
|
|
"""Find documents that may be outdated."""
|
|
with db_session() as db:
|
|
rows = db.fetch_all(
|
|
"""SELECT doc_id, title, url, crawl_date, crawl_status
|
|
FROM documents
|
|
WHERE crawl_status = %s
|
|
ORDER BY crawl_date ASC""",
|
|
("completed",),
|
|
)
|
|
|
|
# Filter by age
|
|
cutoff = datetime.now()
|
|
stale = []
|
|
for r in rows:
|
|
crawl_date = r.get("crawl_date")
|
|
if crawl_date:
|
|
if isinstance(crawl_date, str):
|
|
crawl_date = datetime.fromisoformat(crawl_date)
|
|
age_days = (cutoff - crawl_date).days
|
|
if age_days >= days:
|
|
r["age_days"] = age_days
|
|
stale.append(r)
|
|
|
|
if output:
|
|
Path(output).write_text(json.dumps(stale, indent=2, default=str))
|
|
console.print(f"[green]Found {len(stale)} stale documents, wrote to {output}[/green]")
|
|
else:
|
|
_print_doc_table(stale, f"Stale documents (>{days} days)")
|
|
|
|
|
|
@cli.command("pending-reviews")
|
|
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
|
def pending_reviews(output):
|
|
"""Get documents pending quality review."""
|
|
with db_session() as db:
|
|
rows = db.fetch_all(
|
|
"""SELECT dc.distill_id, d.doc_id, d.title, d.url,
|
|
dc.token_count_distilled, dc.distill_date
|
|
FROM distilled_content dc
|
|
JOIN documents d ON dc.doc_id = d.doc_id
|
|
WHERE dc.review_status = %s
|
|
ORDER BY dc.distill_date ASC""",
|
|
("pending",),
|
|
)
|
|
|
|
if output:
|
|
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
|
console.print(f"[green]{len(rows)} pending reviews, wrote to {output}[/green]")
|
|
else:
|
|
table = Table(title="Pending Reviews")
|
|
table.add_column("distill_id", style="cyan")
|
|
table.add_column("doc_id")
|
|
table.add_column("Title")
|
|
table.add_column("Tokens", justify="right")
|
|
for r in rows:
|
|
table.add_row(
|
|
str(r.get("distill_id", "")),
|
|
str(r.get("doc_id", "")),
|
|
str(r.get("title", ""))[:50],
|
|
str(r.get("token_count_distilled", "")),
|
|
)
|
|
console.print(table)
|
|
|
|
|
|
@cli.command("export-ready")
|
|
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
|
|
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
|
def export_ready(min_score, output):
|
|
"""Get documents approved and ready for export."""
|
|
with db_session() as db:
|
|
rows = db.fetch_all(
|
|
"""SELECT d.doc_id, d.title, d.url,
|
|
dc.structured_content, dc.token_count_distilled,
|
|
rl.quality_score, rl.decision
|
|
FROM documents d
|
|
JOIN distilled_content dc ON d.doc_id = dc.doc_id
|
|
JOIN review_logs rl ON dc.distill_id = rl.distill_id
|
|
WHERE dc.review_status = %s
|
|
AND rl.decision = %s
|
|
AND rl.review_id = (
|
|
SELECT MAX(rl2.review_id)
|
|
FROM review_logs rl2
|
|
WHERE rl2.distill_id = dc.distill_id
|
|
)
|
|
ORDER BY rl.quality_score DESC""",
|
|
("approved", "approve"),
|
|
)
|
|
|
|
# Filter by min score
|
|
rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]
|
|
|
|
if output:
|
|
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
|
console.print(f"[green]{len(rows)} export-ready documents, wrote to {output}[/green]")
|
|
else:
|
|
table = Table(title=f"Export-Ready (score >= {min_score})")
|
|
table.add_column("doc_id", style="cyan")
|
|
table.add_column("Title")
|
|
table.add_column("Score", justify="right")
|
|
table.add_column("Tokens", justify="right")
|
|
for r in rows:
|
|
table.add_row(
|
|
str(r.get("doc_id", "")),
|
|
str(r.get("title", ""))[:50],
|
|
f"{r.get('quality_score', 0):.2f}",
|
|
str(r.get("token_count_distilled", "")),
|
|
)
|
|
console.print(table)
|
|
|
|
|
|
@cli.command()
|
|
def stats():
|
|
"""Show repository statistics."""
|
|
with db_session() as db:
|
|
doc_count = db.fetch_one("SELECT COUNT(*) as cnt FROM documents") or {"cnt": 0}
|
|
source_count = db.fetch_one("SELECT COUNT(*) as cnt FROM sources") or {"cnt": 0}
|
|
|
|
status_rows = db.fetch_all(
|
|
"""SELECT crawl_status, COUNT(*) as cnt
|
|
FROM documents
|
|
GROUP BY crawl_status"""
|
|
)
|
|
|
|
review_rows = db.fetch_all(
|
|
"""SELECT review_status, COUNT(*) as cnt
|
|
FROM distilled_content
|
|
GROUP BY review_status"""
|
|
)
|
|
|
|
topic_rows = db.fetch_all(
|
|
"""SELECT t.topic_name, COUNT(dt.doc_id) as cnt
|
|
FROM topics t
|
|
LEFT JOIN document_topics dt ON t.topic_id = dt.topic_id
|
|
GROUP BY t.topic_id
|
|
ORDER BY cnt DESC"""
|
|
)
|
|
|
|
console.print()
|
|
console.print(f"[bold]Reference Library Statistics[/bold]")
|
|
console.print(f" Sources: {source_count['cnt']}")
|
|
console.print(f" Documents: {doc_count['cnt']}")
|
|
console.print()
|
|
|
|
if status_rows:
|
|
table = Table(title="Documents by Crawl Status")
|
|
table.add_column("Status")
|
|
table.add_column("Count", justify="right")
|
|
for r in status_rows:
|
|
table.add_row(str(r["crawl_status"]), str(r["cnt"]))
|
|
console.print(table)
|
|
|
|
if review_rows:
|
|
table = Table(title="Distilled Content by Review Status")
|
|
table.add_column("Status")
|
|
table.add_column("Count", justify="right")
|
|
for r in review_rows:
|
|
table.add_row(str(r["review_status"]), str(r["cnt"]))
|
|
console.print(table)
|
|
|
|
if topic_rows:
|
|
table = Table(title="Documents by Topic")
|
|
table.add_column("Topic")
|
|
table.add_column("Documents", justify="right")
|
|
for r in topic_rows:
|
|
table.add_row(str(r["topic_name"]), str(r["cnt"]))
|
|
console.print(table)
|
|
|
|
result = {
|
|
"sources": source_count["cnt"],
|
|
"documents": doc_count["cnt"],
|
|
"by_status": {r["crawl_status"]: r["cnt"] for r in status_rows},
|
|
"by_review": {r["review_status"]: r["cnt"] for r in review_rows},
|
|
"by_topic": {r["topic_name"]: r["cnt"] for r in topic_rows},
|
|
}
|
|
click.echo(json.dumps(result, default=str))
|
|
|
|
|
|
def _print_doc_table(rows: list[dict], title: str):
|
|
"""Print a table of documents."""
|
|
table = Table(title=title)
|
|
table.add_column("doc_id", style="cyan")
|
|
table.add_column("Title")
|
|
table.add_column("URL")
|
|
table.add_column("Status")
|
|
for r in rows:
|
|
table.add_row(
|
|
str(r.get("doc_id", "")),
|
|
str(r.get("title", ""))[:40],
|
|
str(r.get("url", ""))[:50],
|
|
str(r.get("crawl_status", "")),
|
|
)
|
|
console.print(table)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|