#!/usr/bin/env python3 """Content Repository CLI — CRUD operations for the reference library. Usage: python repo.py store --source-id 1 --title "Doc" --url "https://..." --doc-type webpage --raw-path /path/to/file python repo.py check-dup --url "https://..." python repo.py query-topic --topic-slug prompt-engineering [--min-quality 0.80] python repo.py find-stale [--output stale.json] python repo.py pending-reviews [--output pending.json] python repo.py export-ready [--min-score 0.85] [--output ready.json] python repo.py stats """ import json import sys from datetime import datetime from pathlib import Path import click from rich.console import Console from rich.table import Table from refcurator.db import db_session from refcurator.utils import url_hash, normalize_url console = Console() @click.group() def cli(): """Content Repository — manage documents in the reference library.""" pass @cli.command() @click.option("--source-id", required=True, type=int, help="Source ID from sources table") @click.option("--title", required=True, help="Document title") @click.option("--url", required=True, help="Document URL") @click.option("--doc-type", required=True, type=click.Choice(["webpage", "pdf", "markdown", "api_spec", "code_sample"])) @click.option("--raw-path", required=True, type=click.Path(), help="Path to raw content file") @click.option("--crawl-method", default="firecrawl", type=click.Choice(["firecrawl", "scrapy", "aiohttp", "nodejs", "manual", "api"])) @click.option("--language", default="en", type=click.Choice(["en", "ko", "mixed"])) def store(source_id, title, url, doc_type, raw_path, crawl_method, language): """Store a new document in the repository.""" raw = Path(raw_path) if not raw.is_file(): console.print(f"[red]Error:[/red] Raw file not found: {raw_path}") sys.exit(1) content_size = raw.stat().st_size with db_session() as db: # Check for duplicate existing = db.fetch_one( "SELECT doc_id, title FROM documents WHERE url_hash = %s", (url_hash(url),), ) if existing: console.print( f"[yellow]Duplicate:[/yellow] URL already stored as doc_id={existing['doc_id']} " f"({existing['title']})" ) sys.exit(0) doc_id = db.insert_returning_id( """INSERT INTO documents (source_id, title, url, doc_type, language, crawl_date, crawl_method, crawl_status, raw_content_path, raw_content_size) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", (source_id, title, url, doc_type, language, datetime.now().isoformat(), crawl_method, "completed", str(raw.resolve()), content_size), ) console.print(f"[green]Stored:[/green] doc_id={doc_id} — {title}") click.echo(json.dumps({"doc_id": doc_id, "title": title, "url": url})) @cli.command("check-dup") @click.option("--url", required=True, help="URL to check for duplicates") def check_dup(url): """Check if a URL already exists in the repository.""" h = url_hash(url) with db_session() as db: existing = db.fetch_one( "SELECT doc_id, title, url, crawl_status FROM documents WHERE url_hash = %s", (h,), ) if existing: console.print(f"[yellow]Duplicate found:[/yellow] doc_id={existing['doc_id']}") click.echo(json.dumps(existing, default=str)) else: console.print("[green]No duplicate found.[/green]") click.echo(json.dumps({"duplicate": False, "url": url})) @cli.command("query-topic") @click.option("--topic-slug", required=True, help="Topic slug to query") @click.option("--min-quality", default=0.0, type=float, help="Minimum quality score") @click.option("--output", type=click.Path(), help="Output JSON file path") def query_topic(topic_slug, min_quality, output): """Query documents by topic.""" with db_session() as db: rows = db.fetch_all( """SELECT d.doc_id, d.title, d.url, d.crawl_status, dt.relevance_score, t.topic_name FROM documents d JOIN document_topics dt ON d.doc_id = dt.doc_id JOIN topics t ON dt.topic_id = t.topic_id WHERE t.topic_slug = %s ORDER BY dt.relevance_score DESC""", (topic_slug,), ) if min_quality > 0: # Filter by review score if available rows = [r for r in rows if r.get("relevance_score", 0) >= min_quality] if output: Path(output).write_text(json.dumps(rows, indent=2, default=str)) console.print(f"[green]Wrote {len(rows)} results to {output}[/green]") else: _print_doc_table(rows, f"Topic: {topic_slug}") @cli.command("find-stale") @click.option("--days", default=30, type=int, help="Documents older than N days") @click.option("--output", type=click.Path(), help="Output JSON file path") def find_stale(days, output): """Find documents that may be outdated.""" with db_session() as db: rows = db.fetch_all( """SELECT doc_id, title, url, crawl_date, crawl_status FROM documents WHERE crawl_status = %s ORDER BY crawl_date ASC""", ("completed",), ) # Filter by age cutoff = datetime.now() stale = [] for r in rows: crawl_date = r.get("crawl_date") if crawl_date: if isinstance(crawl_date, str): crawl_date = datetime.fromisoformat(crawl_date) age_days = (cutoff - crawl_date).days if age_days >= days: r["age_days"] = age_days stale.append(r) if output: Path(output).write_text(json.dumps(stale, indent=2, default=str)) console.print(f"[green]Found {len(stale)} stale documents, wrote to {output}[/green]") else: _print_doc_table(stale, f"Stale documents (>{days} days)") @cli.command("pending-reviews") @click.option("--output", type=click.Path(), help="Output JSON file path") def pending_reviews(output): """Get documents pending quality review.""" with db_session() as db: rows = db.fetch_all( """SELECT dc.distill_id, d.doc_id, d.title, d.url, dc.token_count_distilled, dc.distill_date FROM distilled_content dc JOIN documents d ON dc.doc_id = d.doc_id WHERE dc.review_status = %s ORDER BY dc.distill_date ASC""", ("pending",), ) if output: Path(output).write_text(json.dumps(rows, indent=2, default=str)) console.print(f"[green]{len(rows)} pending reviews, wrote to {output}[/green]") else: table = Table(title="Pending Reviews") table.add_column("distill_id", style="cyan") table.add_column("doc_id") table.add_column("Title") table.add_column("Tokens", justify="right") for r in rows: table.add_row( str(r.get("distill_id", "")), str(r.get("doc_id", "")), str(r.get("title", ""))[:50], str(r.get("token_count_distilled", "")), ) console.print(table) @cli.command("export-ready") @click.option("--min-score", default=0.80, type=float, help="Minimum quality score") @click.option("--output", type=click.Path(), help="Output JSON file path") def export_ready(min_score, output): """Get documents approved and ready for export.""" with db_session() as db: rows = db.fetch_all( """SELECT d.doc_id, d.title, d.url, dc.structured_content, dc.token_count_distilled, rl.quality_score, rl.decision FROM documents d JOIN distilled_content dc ON d.doc_id = dc.doc_id JOIN review_logs rl ON dc.distill_id = rl.distill_id WHERE dc.review_status = %s AND rl.decision = %s AND rl.review_id = ( SELECT MAX(rl2.review_id) FROM review_logs rl2 WHERE rl2.distill_id = dc.distill_id ) ORDER BY rl.quality_score DESC""", ("approved", "approve"), ) # Filter by min score rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score] if output: Path(output).write_text(json.dumps(rows, indent=2, default=str)) console.print(f"[green]{len(rows)} export-ready documents, wrote to {output}[/green]") else: table = Table(title=f"Export-Ready (score >= {min_score})") table.add_column("doc_id", style="cyan") table.add_column("Title") table.add_column("Score", justify="right") table.add_column("Tokens", justify="right") for r in rows: table.add_row( str(r.get("doc_id", "")), str(r.get("title", ""))[:50], f"{r.get('quality_score', 0):.2f}", str(r.get("token_count_distilled", "")), ) console.print(table) @cli.command() def stats(): """Show repository statistics.""" with db_session() as db: doc_count = db.fetch_one("SELECT COUNT(*) as cnt FROM documents") or {"cnt": 0} source_count = db.fetch_one("SELECT COUNT(*) as cnt FROM sources") or {"cnt": 0} status_rows = db.fetch_all( """SELECT crawl_status, COUNT(*) as cnt FROM documents GROUP BY crawl_status""" ) review_rows = db.fetch_all( """SELECT review_status, COUNT(*) as cnt FROM distilled_content GROUP BY review_status""" ) topic_rows = db.fetch_all( """SELECT t.topic_name, COUNT(dt.doc_id) as cnt FROM topics t LEFT JOIN document_topics dt ON t.topic_id = dt.topic_id GROUP BY t.topic_id ORDER BY cnt DESC""" ) console.print() console.print(f"[bold]Reference Library Statistics[/bold]") console.print(f" Sources: {source_count['cnt']}") console.print(f" Documents: {doc_count['cnt']}") console.print() if status_rows: table = Table(title="Documents by Crawl Status") table.add_column("Status") table.add_column("Count", justify="right") for r in status_rows: table.add_row(str(r["crawl_status"]), str(r["cnt"])) console.print(table) if review_rows: table = Table(title="Distilled Content by Review Status") table.add_column("Status") table.add_column("Count", justify="right") for r in review_rows: table.add_row(str(r["review_status"]), str(r["cnt"])) console.print(table) if topic_rows: table = Table(title="Documents by Topic") table.add_column("Topic") table.add_column("Documents", justify="right") for r in topic_rows: table.add_row(str(r["topic_name"]), str(r["cnt"])) console.print(table) result = { "sources": source_count["cnt"], "documents": doc_count["cnt"], "by_status": {r["crawl_status"]: r["cnt"] for r in status_rows}, "by_review": {r["review_status"]: r["cnt"] for r in review_rows}, "by_topic": {r["topic_name"]: r["cnt"] for r in topic_rows}, } click.echo(json.dumps(result, default=str)) def _print_doc_table(rows: list[dict], title: str): """Print a table of documents.""" table = Table(title=title) table.add_column("doc_id", style="cyan") table.add_column("Title") table.add_column("URL") table.add_column("Status") for r in rows: table.add_row( str(r.get("doc_id", "")), str(r.get("title", ""))[:40], str(r.get("url", ""))[:50], str(r.get("crawl_status", "")), ) console.print(table) if __name__ == "__main__": cli()