#!/usr/bin/env python3 """Quality Reviewer CLI — scoring, routing, and review logging. Claude performs qualitative assessment. This script handles: - Loading pending reviews - Calculating weighted quality scores - Routing decisions based on thresholds - Logging review decisions to the database Usage: python reviewer.py load-pending [--output pending.json] python reviewer.py calculate-score --assessment assessment.json python reviewer.py route --score 0.78 python reviewer.py log-review --distill-id 123 --decision refactor --score 0.78 [--feedback "..."] python reviewer.py history --distill-id 123 """ import json import sys from datetime import datetime from pathlib import Path import click from rich.console import Console from rich.table import Table from refcurator.db import db_session from refcurator.models import QAAssessment console = Console() # Thresholds from pipeline config APPROVE_THRESHOLD = 0.85 REFACTOR_THRESHOLD = 0.60 DEEP_RESEARCH_THRESHOLD = 0.40 @click.group() def cli(): """Quality Reviewer — scoring, routing, and review logging.""" pass @cli.command("load-pending") @click.option("--output", type=click.Path(), help="Output JSON file path") @click.option("--limit", default=50, type=int, help="Max results") def load_pending(output, limit): """Load distilled content pending quality review.""" with db_session() as db: rows = db.fetch_all( """SELECT dc.distill_id, d.doc_id, d.title, d.url, dc.token_count_distilled, dc.distill_date, dc.summary, s.credibility_tier, s.vendor FROM distilled_content dc JOIN documents d ON dc.doc_id = d.doc_id JOIN sources s ON d.source_id = s.source_id WHERE dc.review_status = %s ORDER BY s.credibility_tier ASC, dc.distill_date ASC LIMIT %s""", ("pending", limit), ) if output: Path(output).write_text(json.dumps(rows, indent=2, default=str)) console.print(f"[green]{len(rows)} pending reviews written to {output}[/green]") else: table = Table(title=f"Pending Reviews ({len(rows)})") table.add_column("distill_id", style="cyan") table.add_column("Title") table.add_column("Tier") table.add_column("Tokens", justify="right") for r in rows: table.add_row( str(r.get("distill_id", "")), str(r.get("title", ""))[:40], str(r.get("credibility_tier", "")), str(r.get("token_count_distilled", "")), ) console.print(table) click.echo(json.dumps({"count": len(rows)}, default=str)) @cli.command("calculate-score") @click.option("--assessment", required=True, type=click.Path(exists=True), help="Assessment JSON file with criteria scores") @click.option("--json-output", is_flag=True, help="Output as JSON only") def calculate_score(assessment, json_output): """Calculate weighted quality score from assessment criteria. Input JSON format: { "accuracy": 0.90, "completeness": 0.85, "clarity": 0.95, "prompt_engineering_quality": 0.88, "usability": 0.82 } Weights: accuracy 0.25, completeness 0.20, clarity 0.20, prompt_engineering_quality 0.25, usability 0.10 """ data = json.loads(Path(assessment).read_text()) qa = QAAssessment(**data) result = { "criteria": data, "weighted_score": qa.weighted_score, "decision": _route_score(qa.weighted_score), } if json_output: click.echo(json.dumps(result, indent=2)) else: console.print(f"\n[bold]Quality Assessment[/bold]") for criterion, score in data.items(): bar = "█" * int(score * 20) + "░" * (20 - int(score * 20)) console.print(f" {criterion:<30} {bar} {score:.2f}") console.print(f"\n [bold]Weighted Score:[/bold] {qa.weighted_score:.4f}") console.print(f" [bold]Decision:[/bold] {_route_score(qa.weighted_score)}") @cli.command() @click.option("--score", required=True, type=float, help="Quality score to route") @click.option("--tier", type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]), help="Source credibility tier (for auto-approve)") @click.option("--auto-approve", is_flag=True, help="Enable auto-approve for tier1 sources") def route(score, tier, auto_approve): """Route a quality score to a decision. Thresholds: >= 0.85 → approve 0.60-0.84 → refactor 0.40-0.59 → deep_research < 0.40 → reject """ decision = _route_score(score) # Auto-approve tier1 sources with lower threshold if auto_approve and tier == "tier1_official" and score >= 0.80: decision = "approve" result = {"score": score, "decision": decision} if decision == "approve": console.print(f"[green]APPROVE[/green] (score: {score:.2f})") elif decision == "refactor": console.print(f"[yellow]REFACTOR[/yellow] (score: {score:.2f})") elif decision == "deep_research": console.print(f"[blue]DEEP_RESEARCH[/blue] (score: {score:.2f})") else: console.print(f"[red]REJECT[/red] (score: {score:.2f})") click.echo(json.dumps(result)) @cli.command("log-review") @click.option("--distill-id", required=True, type=int, help="Distilled content ID") @click.option("--decision", required=True, type=click.Choice(["approve", "refactor", "deep_research", "reject"])) @click.option("--score", required=True, type=float, help="Quality score") @click.option("--assessment", type=click.Path(exists=True), help="Assessment JSON file") @click.option("--feedback", help="Review feedback text") @click.option("--instructions", help="Refactor instructions") @click.option("--queries", type=click.Path(exists=True), help="Research queries JSON (for deep_research)") @click.option("--reviewer", default="claude_review", type=click.Choice(["auto_qa", "human", "claude_review"])) def log_review(distill_id, decision, score, assessment, feedback, instructions, queries, reviewer): """Log a review decision for distilled content.""" assessment_json = json.loads(Path(assessment).read_text()) if assessment else None queries_json = json.loads(Path(queries).read_text()) if queries else None with db_session() as db: # Get current review round last_review = db.fetch_one( "SELECT MAX(review_round) as max_round FROM review_logs WHERE distill_id = %s", (distill_id,), ) review_round = (last_review.get("max_round") or 0) + 1 if last_review else 1 review_id = db.insert_returning_id( """INSERT INTO review_logs (distill_id, review_round, reviewer_type, quality_score, assessment, decision, feedback, refactor_instructions, research_queries, reviewed_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", (distill_id, review_round, reviewer, score, json.dumps(assessment_json) if assessment_json else None, decision, feedback, instructions, json.dumps(queries_json) if queries_json else None, datetime.now().isoformat()), ) # Update distilled_content review_status status_map = { "approve": "approved", "refactor": "needs_refactor", "deep_research": "needs_refactor", "reject": "rejected", } db.execute( "UPDATE distilled_content SET review_status = %s WHERE distill_id = %s", (status_map[decision], distill_id), ) console.print( f"[green]Logged:[/green] review_id={review_id}, round={review_round}, " f"decision={decision}, score={score:.2f}" ) click.echo(json.dumps({ "review_id": review_id, "distill_id": distill_id, "review_round": review_round, "decision": decision, "score": score, })) @cli.command() @click.option("--distill-id", required=True, type=int, help="Distilled content ID") def history(distill_id): """Show review history for a distilled content record.""" with db_session() as db: reviews = db.fetch_all( """SELECT review_id, review_round, reviewer_type, quality_score, decision, feedback, refactor_instructions, reviewed_at FROM review_logs WHERE distill_id = %s ORDER BY review_round ASC""", (distill_id,), ) if not reviews: console.print(f"[dim]No reviews found for distill_id={distill_id}[/dim]") return table = Table(title=f"Review History — distill_id={distill_id}") table.add_column("Round", style="cyan") table.add_column("Score", justify="right") table.add_column("Decision") table.add_column("Reviewer") table.add_column("Feedback") for r in reviews: decision = str(r.get("decision", "")) style = {"approve": "green", "refactor": "yellow", "deep_research": "blue", "reject": "red"}.get(decision, "") table.add_row( str(r.get("review_round", "")), f"{float(r['quality_score']):.2f}" if r.get("quality_score") else "", f"[{style}]{decision}[/{style}]" if style else decision, str(r.get("reviewer_type", "")), str(r.get("feedback", ""))[:40] if r.get("feedback") else "", ) console.print(table) @cli.command("gemini-evaluate") @click.option("--doc-id", required=True, type=int, help="Document ID to evaluate") @click.option("--topic", required=True, help="Curation topic for relevance scoring") @click.option("--auto-approve", is_flag=True, help="Auto-log decision based on Gemini verdict") def gemini_evaluate(doc_id, topic, auto_approve): """Evaluate raw crawled content using Gemini CLI (pre-distillation gate). Sends raw content to Gemini for independent quality assessment. Scores: relevance, authority, completeness, freshness, distill_value. """ from refcurator.gemini import evaluate_content, is_available if not is_available(): console.print("[red]Error:[/red] Gemini CLI not available. Install: npm install -g @google/gemini-cli") sys.exit(1) with db_session() as db: doc = db.fetch_one( "SELECT doc_id, title, url, raw_content_path FROM documents WHERE doc_id = %s", (doc_id,), ) if not doc: console.print(f"[red]Error:[/red] doc_id={doc_id} not found") sys.exit(1) raw_path = doc.get("raw_content_path") if not raw_path or not Path(raw_path).is_file(): console.print(f"[red]Error:[/red] Raw content file not found: {raw_path}") sys.exit(1) content = Path(raw_path).read_text(errors="replace") url = doc.get("url", "") console.print(f"[dim]Evaluating doc_id={doc_id}: {doc.get('title', '')}[/dim]") console.print(f"[dim]Sending {len(content):,} chars to Gemini...[/dim]") result = evaluate_content(content, topic, url) if result is None: console.print("[yellow]Warning:[/yellow] Gemini evaluation failed — manual review needed") click.echo(json.dumps({"doc_id": doc_id, "status": "gemini_failed"})) return # Display results console.print(f"\n[bold]Gemini Evaluation — doc_id={doc_id}[/bold]") for criterion in ["relevance", "authority", "completeness", "freshness", "distill_value"]: score = result.get(criterion, 0) bar = "█" * int(score * 20) + "░" * (20 - int(score * 20)) console.print(f" {criterion:<15} {bar} {score:.2f}") ws = result.get("weighted_score", 0) verdict = result.get("verdict", "unknown") reason = result.get("reason", "") console.print(f"\n [bold]Weighted Score:[/bold] {ws:.4f}") verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white") console.print(f" [bold]Verdict:[/bold] [{verdict_color}]{verdict}[/{verdict_color}]") if reason: console.print(f" [dim]Reason:[/dim] {reason}") if auto_approve: # Map verdict to decision (no 'refactor' for raw content) decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject" with db_session() as db: # Log to review_logs (using doc_id context, not distill_id since pre-distillation) # We create a placeholder distill_id = 0 entry or log against the document directly review_id = db.insert_returning_id( """INSERT INTO review_logs (distill_id, review_round, reviewer_type, quality_score, assessment, decision, feedback, reviewed_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""", (0, 1, "auto_qa", ws, json.dumps(result), decision, reason, datetime.now().isoformat()), ) console.print(f"\n[green]Auto-logged:[/green] review_id={review_id}, decision={decision}") click.echo(json.dumps({ "doc_id": doc_id, "evaluation": result, "auto_logged": auto_approve, }, default=str)) @cli.command("gemini-evaluate-pending") @click.option("--topic", required=True, help="Curation topic for relevance scoring") @click.option("--auto-approve", is_flag=True, help="Auto-log decisions") @click.option("--limit", default=20, type=int, help="Max documents to evaluate") def gemini_evaluate_pending(topic, auto_approve, limit): """Evaluate all crawled-but-not-evaluated documents using Gemini.""" from refcurator.gemini import evaluate_content, is_available if not is_available(): console.print("[red]Error:[/red] Gemini CLI not available") sys.exit(1) with db_session() as db: rows = db.fetch_all( """SELECT doc_id, title, url, raw_content_path FROM documents WHERE crawl_status = %s ORDER BY doc_id ASC LIMIT %s""", ("completed", limit), ) if not rows: console.print("[dim]No pending documents to evaluate.[/dim]") return console.print(f"[bold]Evaluating {len(rows)} documents with Gemini...[/bold]\n") stats = {"approve": 0, "reject": 0, "deep_research": 0, "failed": 0} for row in rows: doc_id = row["doc_id"] raw_path = row.get("raw_content_path") if not raw_path or not Path(raw_path).is_file(): console.print(f" [yellow]Skip[/yellow] doc_id={doc_id}: raw file not found") stats["failed"] += 1 continue content = Path(raw_path).read_text(errors="replace") result = evaluate_content(content, topic, row.get("url", "")) if result is None: console.print(f" [yellow]Failed[/yellow] doc_id={doc_id}: Gemini returned no result") stats["failed"] += 1 continue verdict = result.get("verdict", "reject") ws = result.get("weighted_score", 0) verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white") console.print( f" [{verdict_color}]{verdict:>13}[/{verdict_color}] " f"({ws:.2f}) doc_id={doc_id} — {row.get('title', '')[:40]}" ) stats[verdict] = stats.get(verdict, 0) + 1 if auto_approve: decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject" with db_session() as db: db.insert_returning_id( """INSERT INTO review_logs (distill_id, review_round, reviewer_type, quality_score, assessment, decision, feedback, reviewed_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""", (0, 1, "auto_qa", ws, json.dumps(result), decision, result.get("reason", ""), datetime.now().isoformat()), ) console.print(f"\n[bold]Results:[/bold]") console.print(f" [green]Approved:[/green] {stats['approve']}") console.print(f" [red]Rejected:[/red] {stats['reject']}") console.print(f" [blue]Deep Research:[/blue] {stats['deep_research']}") console.print(f" [yellow]Failed:[/yellow] {stats['failed']}") click.echo(json.dumps(stats)) def _route_score(score: float) -> str: """Route a quality score to a decision based on thresholds.""" if score >= APPROVE_THRESHOLD: return "approve" elif score >= REFACTOR_THRESHOLD: return "refactor" elif score >= DEEP_RESEARCH_THRESHOLD: return "deep_research" else: return "reject" if __name__ == "__main__": cli()