our-claude-skills/custom-skills/90-reference-curator/04-content-distiller/code/scripts/distiller.py

#!/usr/bin/env python3
"""Content Distiller CLI — manage distillation data I/O.

Claude performs the actual distillation (summarization, extraction).
This script handles loading raw content from the DB and storing distilled output.

Usage:
    python distiller.py load-pending [--output pending.json]
    python distiller.py store --doc-id 123 --content distilled.md [--model claude-opus-4-6]
    python distiller.py refactor --distill-id 456 [--output context.json]
    python distiller.py show --distill-id 456
"""

import json
import sys
from datetime import datetime
from pathlib import Path

import click
from rich.console import Console
from rich.table import Table

from refcurator.db import db_session
from refcurator.utils import count_tokens

console = Console()


@click.group()
def cli():
    """Content Distiller — manage distillation data."""
    pass


@cli.command("load-pending")
@click.option("--output", type=click.Path(), help="Output JSON file path")
@click.option("--limit", default=50, type=int, help="Max documents to load")
def load_pending(output, limit):
    """Load documents pending distillation.

    Finds documents with crawl_status='completed' that have no distilled content yet.
    """
    with db_session() as db:
        rows = db.fetch_all(
            """SELECT d.doc_id, d.title, d.url, d.raw_content_path,
                      d.raw_content_size, d.doc_type,
                      s.source_name, s.credibility_tier
               FROM documents d
               JOIN sources s ON d.source_id = s.source_id
               LEFT JOIN distilled_content dc ON d.doc_id = dc.doc_id
               WHERE d.crawl_status = %s AND dc.distill_id IS NULL
               ORDER BY s.credibility_tier ASC, d.crawl_date ASC
               LIMIT %s""",
            ("completed", limit),
        )

    # Enrich with raw content preview
    for row in rows:
        raw_path = row.get("raw_content_path")
        if raw_path and Path(raw_path).is_file():
            content = Path(raw_path).read_text(errors="replace")
            row["token_count_estimate"] = count_tokens(content)
            row["content_preview"] = content[:200] + "..." if len(content) > 200 else content
        else:
            row["token_count_estimate"] = 0
            row["content_preview"] = "[file not found]"

    if output:
        Path(output).write_text(json.dumps(rows, indent=2, default=str))
        console.print(f"[green]{len(rows)} pending documents written to {output}[/green]")
    else:
        table = Table(title=f"Pending Distillation ({len(rows)} documents)")
        table.add_column("doc_id", style="cyan")
        table.add_column("Title")
        table.add_column("Source")
        table.add_column("Tier")
        table.add_column("~Tokens", justify="right")
        for r in rows:
            table.add_row(
                str(r.get("doc_id", "")),
                str(r.get("title", ""))[:40],
                str(r.get("source_name", ""))[:20],
                str(r.get("credibility_tier", "")),
                str(r.get("token_count_estimate", "")),
            )
        console.print(table)

    click.echo(json.dumps({"count": len(rows)}, default=str))


@cli.command()
@click.option("--doc-id", required=True, type=int, help="Document ID to store distilled content for")
@click.option("--content", required=True, type=click.Path(exists=True),
              help="Path to distilled markdown content")
@click.option("--summary", type=click.Path(exists=True), help="Path to summary text file")
@click.option("--concepts", type=click.Path(exists=True), help="Path to key concepts JSON")
@click.option("--snippets", type=click.Path(exists=True), help="Path to code snippets JSON")
@click.option("--model", default="claude-opus-4-6", help="Model used for distillation")
def store(doc_id, content, summary, concepts, snippets, model):
    """Store distilled content for a document."""
    structured = Path(content).read_text(errors="replace")
    token_distilled = count_tokens(structured)

    summary_text = Path(summary).read_text() if summary else None
    concepts_json = json.loads(Path(concepts).read_text()) if concepts else None
    snippets_json = json.loads(Path(snippets).read_text()) if snippets else None

    with db_session() as db:
        # Get original token count
        doc = db.fetch_one(
            "SELECT raw_content_path, raw_content_size FROM documents WHERE doc_id = %s",
            (doc_id,),
        )
        token_original = 0
        if doc and doc.get("raw_content_path"):
            raw_path = Path(doc["raw_content_path"])
            if raw_path.is_file():
                token_original = count_tokens(raw_path.read_text(errors="replace"))

        # Check for existing distilled_content row (refactor case)
        existing = db.fetch_one(
            "SELECT distill_id, review_status FROM distilled_content WHERE doc_id = %s",
            (doc_id,),
        )

        if existing and existing.get("review_status") in ("needs_refactor", "pending"):
            # Update existing row instead of creating a new one
            distill_id = existing["distill_id"]
            db.execute(
                """UPDATE distilled_content
                   SET summary = %s, key_concepts = %s, code_snippets = %s,
                       structured_content = %s, token_count_original = %s,
                       token_count_distilled = %s, distill_model = %s,
                       distill_date = %s, review_status = %s
                   WHERE distill_id = %s""",
                (summary_text,
                 json.dumps(concepts_json) if concepts_json else None,
                 json.dumps(snippets_json) if snippets_json else None,
                 structured, token_original, token_distilled, model,
                 datetime.now().isoformat(), "pending", distill_id),
            )
            action = "Updated"
        else:
            # First distillation for this document
            distill_id = db.insert_returning_id(
                """INSERT INTO distilled_content
                   (doc_id, summary, key_concepts, code_snippets, structured_content,
                    token_count_original, token_count_distilled, distill_model, distill_date,
                    review_status)
                   VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
                (doc_id, summary_text,
                 json.dumps(concepts_json) if concepts_json else None,
                 json.dumps(snippets_json) if snippets_json else None,
                 structured, token_original, token_distilled, model,
                 datetime.now().isoformat(), "pending"),
            )
            action = "Stored"

    ratio = (token_distilled / token_original * 100) if token_original else 0
    console.print(
        f"[green]{action}:[/green] distill_id={distill_id} for doc_id={doc_id} "
        f"({token_original} → {token_distilled} tokens, {ratio:.0f}% compression)"
    )
    click.echo(json.dumps({
        "distill_id": distill_id,
        "doc_id": doc_id,
        "token_original": token_original,
        "token_distilled": token_distilled,
        "compression_ratio": round(ratio, 2),
    }))


@cli.command()
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
@click.option("--output", type=click.Path(), help="Output context JSON for re-distillation")
def refactor(distill_id, output):
    """Load existing distilled content + review feedback for re-distillation.

    Outputs a context bundle that Claude can use to re-distill with improvements.
    """
    with db_session() as db:
        distilled = db.fetch_one(
            """SELECT dc.*, d.title, d.url, d.raw_content_path
               FROM distilled_content dc
               JOIN documents d ON dc.doc_id = d.doc_id
               WHERE dc.distill_id = %s""",
            (distill_id,),
        )

        if not distilled:
            console.print(f"[red]Error:[/red] distill_id={distill_id} not found")
            sys.exit(1)

        # Get review feedback
        reviews = db.fetch_all(
            """SELECT review_round, quality_score, decision, feedback, refactor_instructions
               FROM review_logs
               WHERE distill_id = %s
               ORDER BY review_round ASC""",
            (distill_id,),
        )

    # Load raw content if available
    raw_content = ""
    raw_path = distilled.get("raw_content_path")
    if raw_path and Path(raw_path).is_file():
        raw_content = Path(raw_path).read_text(errors="replace")

    context = {
        "distill_id": distill_id,
        "doc_id": distilled.get("doc_id"),
        "title": distilled.get("title"),
        "url": distilled.get("url"),
        "current_distilled": distilled.get("structured_content"),
        "raw_content": raw_content,
        "review_history": [
            {
                "round": r.get("review_round"),
                "score": float(r["quality_score"]) if r.get("quality_score") else None,
                "decision": r.get("decision"),
                "feedback": r.get("feedback"),
                "instructions": r.get("refactor_instructions"),
            }
            for r in reviews
        ],
    }

    if output:
        Path(output).write_text(json.dumps(context, indent=2, default=str))
        console.print(f"[green]Refactor context written to {output}[/green]")
    else:
        click.echo(json.dumps(context, indent=2, default=str))


@cli.command()
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
def show(distill_id):
    """Show details of a distilled content record."""
    with db_session() as db:
        row = db.fetch_one(
            """SELECT dc.*, d.title, d.url
               FROM distilled_content dc
               JOIN documents d ON dc.doc_id = d.doc_id
               WHERE dc.distill_id = %s""",
            (distill_id,),
        )

    if not row:
        console.print(f"[red]Not found:[/red] distill_id={distill_id}")
        sys.exit(1)

    console.print(f"\n[bold]Distilled Content #{distill_id}[/bold]")
    console.print(f"  Document: {row.get('title')} (doc_id={row.get('doc_id')})")
    console.print(f"  URL: {row.get('url')}")
    console.print(f"  Review Status: {row.get('review_status')}")
    console.print(f"  Model: {row.get('distill_model')}")
    console.print(f"  Tokens: {row.get('token_count_original')} → {row.get('token_count_distilled')}")
    if row.get("summary"):
        console.print(f"\n[bold]Summary:[/bold]\n{row['summary'][:500]}")


if __name__ == "__main__":
    cli()