our-claude-skills/custom-skills/90-reference-curator/05-quality-reviewer/code/scripts/reviewer.py

#!/usr/bin/env python3
"""Quality Reviewer CLI — scoring, routing, and review logging.

Claude performs qualitative assessment. This script handles:
- Loading pending reviews
- Calculating weighted quality scores
- Routing decisions based on thresholds
- Logging review decisions to the database

Usage:
    python reviewer.py load-pending [--output pending.json]
    python reviewer.py calculate-score --assessment assessment.json
    python reviewer.py route --score 0.78
    python reviewer.py log-review --distill-id 123 --decision refactor --score 0.78 [--feedback "..."]
    python reviewer.py history --distill-id 123
"""

import json
import sys
from datetime import datetime
from pathlib import Path

import click
from rich.console import Console
from rich.table import Table

from refcurator.db import db_session
from refcurator.models import QAAssessment

console = Console()

# Thresholds from pipeline config
APPROVE_THRESHOLD = 0.85
REFACTOR_THRESHOLD = 0.60
DEEP_RESEARCH_THRESHOLD = 0.40


@click.group()
def cli():
    """Quality Reviewer — scoring, routing, and review logging."""
    pass


@cli.command("load-pending")
@click.option("--output", type=click.Path(), help="Output JSON file path")
@click.option("--limit", default=50, type=int, help="Max results")
def load_pending(output, limit):
    """Load distilled content pending quality review."""
    with db_session() as db:
        rows = db.fetch_all(
            """SELECT dc.distill_id, d.doc_id, d.title, d.url,
                      dc.token_count_distilled, dc.distill_date,
                      dc.summary, s.credibility_tier, s.vendor
               FROM distilled_content dc
               JOIN documents d ON dc.doc_id = d.doc_id
               JOIN sources s ON d.source_id = s.source_id
               WHERE dc.review_status = %s
               ORDER BY s.credibility_tier ASC, dc.distill_date ASC
               LIMIT %s""",
            ("pending", limit),
        )

    if output:
        Path(output).write_text(json.dumps(rows, indent=2, default=str))
        console.print(f"[green]{len(rows)} pending reviews written to {output}[/green]")
    else:
        table = Table(title=f"Pending Reviews ({len(rows)})")
        table.add_column("distill_id", style="cyan")
        table.add_column("Title")
        table.add_column("Tier")
        table.add_column("Tokens", justify="right")
        for r in rows:
            table.add_row(
                str(r.get("distill_id", "")),
                str(r.get("title", ""))[:40],
                str(r.get("credibility_tier", "")),
                str(r.get("token_count_distilled", "")),
            )
        console.print(table)

    click.echo(json.dumps({"count": len(rows)}, default=str))


@cli.command("calculate-score")
@click.option("--assessment", required=True, type=click.Path(exists=True),
              help="Assessment JSON file with criteria scores")
@click.option("--json-output", is_flag=True, help="Output as JSON only")
def calculate_score(assessment, json_output):
    """Calculate weighted quality score from assessment criteria.

    Input JSON format:
    {
        "accuracy": 0.90,
        "completeness": 0.85,
        "clarity": 0.95,
        "prompt_engineering_quality": 0.88,
        "usability": 0.82
    }

    Weights: accuracy 0.25, completeness 0.20, clarity 0.20,
             prompt_engineering_quality 0.25, usability 0.10
    """
    data = json.loads(Path(assessment).read_text())
    qa = QAAssessment(**data)

    result = {
        "criteria": data,
        "weighted_score": qa.weighted_score,
        "decision": _route_score(qa.weighted_score),
    }

    if json_output:
        click.echo(json.dumps(result, indent=2))
    else:
        console.print(f"\n[bold]Quality Assessment[/bold]")
        for criterion, score in data.items():
            bar = "█" * int(score * 20) + "░" * (20 - int(score * 20))
            console.print(f"  {criterion:<30} {bar} {score:.2f}")
        console.print(f"\n  [bold]Weighted Score:[/bold] {qa.weighted_score:.4f}")
        console.print(f"  [bold]Decision:[/bold] {_route_score(qa.weighted_score)}")


@cli.command()
@click.option("--score", required=True, type=float, help="Quality score to route")
@click.option("--tier", type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]),
              help="Source credibility tier (for auto-approve)")
@click.option("--auto-approve", is_flag=True, help="Enable auto-approve for tier1 sources")
def route(score, tier, auto_approve):
    """Route a quality score to a decision.

    Thresholds:
      >= 0.85 → approve
      0.60-0.84 → refactor
      0.40-0.59 → deep_research
      < 0.40 → reject
    """
    decision = _route_score(score)

    # Auto-approve tier1 sources with lower threshold
    if auto_approve and tier == "tier1_official" and score >= 0.80:
        decision = "approve"

    result = {"score": score, "decision": decision}

    if decision == "approve":
        console.print(f"[green]APPROVE[/green] (score: {score:.2f})")
    elif decision == "refactor":
        console.print(f"[yellow]REFACTOR[/yellow] (score: {score:.2f})")
    elif decision == "deep_research":
        console.print(f"[blue]DEEP_RESEARCH[/blue] (score: {score:.2f})")
    else:
        console.print(f"[red]REJECT[/red] (score: {score:.2f})")

    click.echo(json.dumps(result))


@cli.command("log-review")
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
@click.option("--decision", required=True,
              type=click.Choice(["approve", "refactor", "deep_research", "reject"]))
@click.option("--score", required=True, type=float, help="Quality score")
@click.option("--assessment", type=click.Path(exists=True), help="Assessment JSON file")
@click.option("--feedback", help="Review feedback text")
@click.option("--instructions", help="Refactor instructions")
@click.option("--queries", type=click.Path(exists=True), help="Research queries JSON (for deep_research)")
@click.option("--reviewer", default="claude_review",
              type=click.Choice(["auto_qa", "human", "claude_review"]))
def log_review(distill_id, decision, score, assessment, feedback, instructions, queries, reviewer):
    """Log a review decision for distilled content."""
    assessment_json = json.loads(Path(assessment).read_text()) if assessment else None
    queries_json = json.loads(Path(queries).read_text()) if queries else None

    with db_session() as db:
        # Get current review round
        last_review = db.fetch_one(
            "SELECT MAX(review_round) as max_round FROM review_logs WHERE distill_id = %s",
            (distill_id,),
        )
        review_round = (last_review.get("max_round") or 0) + 1 if last_review else 1

        review_id = db.insert_returning_id(
            """INSERT INTO review_logs
               (distill_id, review_round, reviewer_type, quality_score, assessment,
                decision, feedback, refactor_instructions, research_queries, reviewed_at)
               VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
            (distill_id, review_round, reviewer, score,
             json.dumps(assessment_json) if assessment_json else None,
             decision, feedback, instructions,
             json.dumps(queries_json) if queries_json else None,
             datetime.now().isoformat()),
        )

        # Update distilled_content review_status
        status_map = {
            "approve": "approved",
            "refactor": "needs_refactor",
            "deep_research": "needs_refactor",
            "reject": "rejected",
        }
        db.execute(
            "UPDATE distilled_content SET review_status = %s WHERE distill_id = %s",
            (status_map[decision], distill_id),
        )

    console.print(
        f"[green]Logged:[/green] review_id={review_id}, round={review_round}, "
        f"decision={decision}, score={score:.2f}"
    )
    click.echo(json.dumps({
        "review_id": review_id,
        "distill_id": distill_id,
        "review_round": review_round,
        "decision": decision,
        "score": score,
    }))


@cli.command()
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
def history(distill_id):
    """Show review history for a distilled content record."""
    with db_session() as db:
        reviews = db.fetch_all(
            """SELECT review_id, review_round, reviewer_type, quality_score,
                      decision, feedback, refactor_instructions, reviewed_at
               FROM review_logs
               WHERE distill_id = %s
               ORDER BY review_round ASC""",
            (distill_id,),
        )

    if not reviews:
        console.print(f"[dim]No reviews found for distill_id={distill_id}[/dim]")
        return

    table = Table(title=f"Review History — distill_id={distill_id}")
    table.add_column("Round", style="cyan")
    table.add_column("Score", justify="right")
    table.add_column("Decision")
    table.add_column("Reviewer")
    table.add_column("Feedback")
    for r in reviews:
        decision = str(r.get("decision", ""))
        style = {"approve": "green", "refactor": "yellow",
                 "deep_research": "blue", "reject": "red"}.get(decision, "")
        table.add_row(
            str(r.get("review_round", "")),
            f"{float(r['quality_score']):.2f}" if r.get("quality_score") else "",
            f"[{style}]{decision}[/{style}]" if style else decision,
            str(r.get("reviewer_type", "")),
            str(r.get("feedback", ""))[:40] if r.get("feedback") else "",
        )
    console.print(table)


@cli.command("gemini-evaluate")
@click.option("--doc-id", required=True, type=int, help="Document ID to evaluate")
@click.option("--topic", required=True, help="Curation topic for relevance scoring")
@click.option("--auto-approve", is_flag=True, help="Auto-log decision based on Gemini verdict")
def gemini_evaluate(doc_id, topic, auto_approve):
    """Evaluate raw crawled content using Gemini CLI (pre-distillation gate).

    Sends raw content to Gemini for independent quality assessment.
    Scores: relevance, authority, completeness, freshness, distill_value.
    """
    from refcurator.gemini import evaluate_content, is_available

    if not is_available():
        console.print("[red]Error:[/red] Gemini CLI not available. Install: npm install -g @google/gemini-cli")
        sys.exit(1)

    with db_session() as db:
        doc = db.fetch_one(
            "SELECT doc_id, title, url, raw_content_path FROM documents WHERE doc_id = %s",
            (doc_id,),
        )

    if not doc:
        console.print(f"[red]Error:[/red] doc_id={doc_id} not found")
        sys.exit(1)

    raw_path = doc.get("raw_content_path")
    if not raw_path or not Path(raw_path).is_file():
        console.print(f"[red]Error:[/red] Raw content file not found: {raw_path}")
        sys.exit(1)

    content = Path(raw_path).read_text(errors="replace")
    url = doc.get("url", "")

    console.print(f"[dim]Evaluating doc_id={doc_id}: {doc.get('title', '')}[/dim]")
    console.print(f"[dim]Sending {len(content):,} chars to Gemini...[/dim]")

    result = evaluate_content(content, topic, url)

    if result is None:
        console.print("[yellow]Warning:[/yellow] Gemini evaluation failed — manual review needed")
        click.echo(json.dumps({"doc_id": doc_id, "status": "gemini_failed"}))
        return

    # Display results
    console.print(f"\n[bold]Gemini Evaluation — doc_id={doc_id}[/bold]")
    for criterion in ["relevance", "authority", "completeness", "freshness", "distill_value"]:
        score = result.get(criterion, 0)
        bar = "█" * int(score * 20) + "░" * (20 - int(score * 20))
        console.print(f"  {criterion:<15} {bar} {score:.2f}")

    ws = result.get("weighted_score", 0)
    verdict = result.get("verdict", "unknown")
    reason = result.get("reason", "")

    console.print(f"\n  [bold]Weighted Score:[/bold] {ws:.4f}")
    verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white")
    console.print(f"  [bold]Verdict:[/bold] [{verdict_color}]{verdict}[/{verdict_color}]")
    if reason:
        console.print(f"  [dim]Reason:[/dim] {reason}")

    if auto_approve:
        # Map verdict to decision (no 'refactor' for raw content)
        decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject"

        with db_session() as db:
            # Log to review_logs (using doc_id context, not distill_id since pre-distillation)
            # We create a placeholder distill_id = 0 entry or log against the document directly
            review_id = db.insert_returning_id(
                """INSERT INTO review_logs
                   (distill_id, review_round, reviewer_type, quality_score, assessment,
                    decision, feedback, reviewed_at)
                   VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
                (0, 1, "auto_qa", ws,
                 json.dumps(result),
                 decision, reason,
                 datetime.now().isoformat()),
            )

        console.print(f"\n[green]Auto-logged:[/green] review_id={review_id}, decision={decision}")

    click.echo(json.dumps({
        "doc_id": doc_id,
        "evaluation": result,
        "auto_logged": auto_approve,
    }, default=str))


@cli.command("gemini-evaluate-pending")
@click.option("--topic", required=True, help="Curation topic for relevance scoring")
@click.option("--auto-approve", is_flag=True, help="Auto-log decisions")
@click.option("--limit", default=20, type=int, help="Max documents to evaluate")
def gemini_evaluate_pending(topic, auto_approve, limit):
    """Evaluate all crawled-but-not-evaluated documents using Gemini."""
    from refcurator.gemini import evaluate_content, is_available

    if not is_available():
        console.print("[red]Error:[/red] Gemini CLI not available")
        sys.exit(1)

    with db_session() as db:
        rows = db.fetch_all(
            """SELECT doc_id, title, url, raw_content_path
               FROM documents
               WHERE crawl_status = %s
               ORDER BY doc_id ASC
               LIMIT %s""",
            ("completed", limit),
        )

    if not rows:
        console.print("[dim]No pending documents to evaluate.[/dim]")
        return

    console.print(f"[bold]Evaluating {len(rows)} documents with Gemini...[/bold]\n")

    stats = {"approve": 0, "reject": 0, "deep_research": 0, "failed": 0}

    for row in rows:
        doc_id = row["doc_id"]
        raw_path = row.get("raw_content_path")

        if not raw_path or not Path(raw_path).is_file():
            console.print(f"  [yellow]Skip[/yellow] doc_id={doc_id}: raw file not found")
            stats["failed"] += 1
            continue

        content = Path(raw_path).read_text(errors="replace")
        result = evaluate_content(content, topic, row.get("url", ""))

        if result is None:
            console.print(f"  [yellow]Failed[/yellow] doc_id={doc_id}: Gemini returned no result")
            stats["failed"] += 1
            continue

        verdict = result.get("verdict", "reject")
        ws = result.get("weighted_score", 0)
        verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white")
        console.print(
            f"  [{verdict_color}]{verdict:>13}[/{verdict_color}] "
            f"({ws:.2f}) doc_id={doc_id} — {row.get('title', '')[:40]}"
        )

        stats[verdict] = stats.get(verdict, 0) + 1

        if auto_approve:
            decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject"
            with db_session() as db:
                db.insert_returning_id(
                    """INSERT INTO review_logs
                       (distill_id, review_round, reviewer_type, quality_score, assessment,
                        decision, feedback, reviewed_at)
                       VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
                    (0, 1, "auto_qa", ws,
                     json.dumps(result),
                     decision, result.get("reason", ""),
                     datetime.now().isoformat()),
                )

    console.print(f"\n[bold]Results:[/bold]")
    console.print(f"  [green]Approved:[/green] {stats['approve']}")
    console.print(f"  [red]Rejected:[/red] {stats['reject']}")
    console.print(f"  [blue]Deep Research:[/blue] {stats['deep_research']}")
    console.print(f"  [yellow]Failed:[/yellow] {stats['failed']}")
    click.echo(json.dumps(stats))


def _route_score(score: float) -> str:
    """Route a quality score to a decision based on thresholds."""
    if score >= APPROVE_THRESHOLD:
        return "approve"
    elif score >= REFACTOR_THRESHOLD:
        return "refactor"
    elif score >= DEEP_RESEARCH_THRESHOLD:
        return "deep_research"
    else:
        return "reject"


if __name__ == "__main__":
    cli()