#!/usr/bin/env python3 """Content Distiller CLI — manage distillation data I/O. Claude performs the actual distillation (summarization, extraction). This script handles loading raw content from the DB and storing distilled output. Usage: python distiller.py load-pending [--output pending.json] python distiller.py store --doc-id 123 --content distilled.md [--model claude-opus-4-6] python distiller.py refactor --distill-id 456 [--output context.json] python distiller.py show --distill-id 456 """ import json import sys from datetime import datetime from pathlib import Path import click from rich.console import Console from rich.table import Table from refcurator.db import db_session from refcurator.utils import count_tokens console = Console() @click.group() def cli(): """Content Distiller — manage distillation data.""" pass @cli.command("load-pending") @click.option("--output", type=click.Path(), help="Output JSON file path") @click.option("--limit", default=50, type=int, help="Max documents to load") def load_pending(output, limit): """Load documents pending distillation. Finds documents with crawl_status='completed' that have no distilled content yet. """ with db_session() as db: rows = db.fetch_all( """SELECT d.doc_id, d.title, d.url, d.raw_content_path, d.raw_content_size, d.doc_type, s.source_name, s.credibility_tier FROM documents d JOIN sources s ON d.source_id = s.source_id LEFT JOIN distilled_content dc ON d.doc_id = dc.doc_id WHERE d.crawl_status = %s AND dc.distill_id IS NULL ORDER BY s.credibility_tier ASC, d.crawl_date ASC LIMIT %s""", ("completed", limit), ) # Enrich with raw content preview for row in rows: raw_path = row.get("raw_content_path") if raw_path and Path(raw_path).is_file(): content = Path(raw_path).read_text(errors="replace") row["token_count_estimate"] = count_tokens(content) row["content_preview"] = content[:200] + "..." if len(content) > 200 else content else: row["token_count_estimate"] = 0 row["content_preview"] = "[file not found]" if output: Path(output).write_text(json.dumps(rows, indent=2, default=str)) console.print(f"[green]{len(rows)} pending documents written to {output}[/green]") else: table = Table(title=f"Pending Distillation ({len(rows)} documents)") table.add_column("doc_id", style="cyan") table.add_column("Title") table.add_column("Source") table.add_column("Tier") table.add_column("~Tokens", justify="right") for r in rows: table.add_row( str(r.get("doc_id", "")), str(r.get("title", ""))[:40], str(r.get("source_name", ""))[:20], str(r.get("credibility_tier", "")), str(r.get("token_count_estimate", "")), ) console.print(table) click.echo(json.dumps({"count": len(rows)}, default=str)) @cli.command() @click.option("--doc-id", required=True, type=int, help="Document ID to store distilled content for") @click.option("--content", required=True, type=click.Path(exists=True), help="Path to distilled markdown content") @click.option("--summary", type=click.Path(exists=True), help="Path to summary text file") @click.option("--concepts", type=click.Path(exists=True), help="Path to key concepts JSON") @click.option("--snippets", type=click.Path(exists=True), help="Path to code snippets JSON") @click.option("--model", default="claude-opus-4-6", help="Model used for distillation") def store(doc_id, content, summary, concepts, snippets, model): """Store distilled content for a document.""" structured = Path(content).read_text(errors="replace") token_distilled = count_tokens(structured) summary_text = Path(summary).read_text() if summary else None concepts_json = json.loads(Path(concepts).read_text()) if concepts else None snippets_json = json.loads(Path(snippets).read_text()) if snippets else None with db_session() as db: # Get original token count doc = db.fetch_one( "SELECT raw_content_path, raw_content_size FROM documents WHERE doc_id = %s", (doc_id,), ) token_original = 0 if doc and doc.get("raw_content_path"): raw_path = Path(doc["raw_content_path"]) if raw_path.is_file(): token_original = count_tokens(raw_path.read_text(errors="replace")) # Check for existing distilled_content row (refactor case) existing = db.fetch_one( "SELECT distill_id, review_status FROM distilled_content WHERE doc_id = %s", (doc_id,), ) if existing and existing.get("review_status") in ("needs_refactor", "pending"): # Update existing row instead of creating a new one distill_id = existing["distill_id"] db.execute( """UPDATE distilled_content SET summary = %s, key_concepts = %s, code_snippets = %s, structured_content = %s, token_count_original = %s, token_count_distilled = %s, distill_model = %s, distill_date = %s, review_status = %s WHERE distill_id = %s""", (summary_text, json.dumps(concepts_json) if concepts_json else None, json.dumps(snippets_json) if snippets_json else None, structured, token_original, token_distilled, model, datetime.now().isoformat(), "pending", distill_id), ) action = "Updated" else: # First distillation for this document distill_id = db.insert_returning_id( """INSERT INTO distilled_content (doc_id, summary, key_concepts, code_snippets, structured_content, token_count_original, token_count_distilled, distill_model, distill_date, review_status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", (doc_id, summary_text, json.dumps(concepts_json) if concepts_json else None, json.dumps(snippets_json) if snippets_json else None, structured, token_original, token_distilled, model, datetime.now().isoformat(), "pending"), ) action = "Stored" ratio = (token_distilled / token_original * 100) if token_original else 0 console.print( f"[green]{action}:[/green] distill_id={distill_id} for doc_id={doc_id} " f"({token_original} → {token_distilled} tokens, {ratio:.0f}% compression)" ) click.echo(json.dumps({ "distill_id": distill_id, "doc_id": doc_id, "token_original": token_original, "token_distilled": token_distilled, "compression_ratio": round(ratio, 2), })) @cli.command() @click.option("--distill-id", required=True, type=int, help="Distilled content ID") @click.option("--output", type=click.Path(), help="Output context JSON for re-distillation") def refactor(distill_id, output): """Load existing distilled content + review feedback for re-distillation. Outputs a context bundle that Claude can use to re-distill with improvements. """ with db_session() as db: distilled = db.fetch_one( """SELECT dc.*, d.title, d.url, d.raw_content_path FROM distilled_content dc JOIN documents d ON dc.doc_id = d.doc_id WHERE dc.distill_id = %s""", (distill_id,), ) if not distilled: console.print(f"[red]Error:[/red] distill_id={distill_id} not found") sys.exit(1) # Get review feedback reviews = db.fetch_all( """SELECT review_round, quality_score, decision, feedback, refactor_instructions FROM review_logs WHERE distill_id = %s ORDER BY review_round ASC""", (distill_id,), ) # Load raw content if available raw_content = "" raw_path = distilled.get("raw_content_path") if raw_path and Path(raw_path).is_file(): raw_content = Path(raw_path).read_text(errors="replace") context = { "distill_id": distill_id, "doc_id": distilled.get("doc_id"), "title": distilled.get("title"), "url": distilled.get("url"), "current_distilled": distilled.get("structured_content"), "raw_content": raw_content, "review_history": [ { "round": r.get("review_round"), "score": float(r["quality_score"]) if r.get("quality_score") else None, "decision": r.get("decision"), "feedback": r.get("feedback"), "instructions": r.get("refactor_instructions"), } for r in reviews ], } if output: Path(output).write_text(json.dumps(context, indent=2, default=str)) console.print(f"[green]Refactor context written to {output}[/green]") else: click.echo(json.dumps(context, indent=2, default=str)) @cli.command() @click.option("--distill-id", required=True, type=int, help="Distilled content ID") def show(distill_id): """Show details of a distilled content record.""" with db_session() as db: row = db.fetch_one( """SELECT dc.*, d.title, d.url FROM distilled_content dc JOIN documents d ON dc.doc_id = d.doc_id WHERE dc.distill_id = %s""", (distill_id,), ) if not row: console.print(f"[red]Not found:[/red] distill_id={distill_id}") sys.exit(1) console.print(f"\n[bold]Distilled Content #{distill_id}[/bold]") console.print(f" Document: {row.get('title')} (doc_id={row.get('doc_id')})") console.print(f" URL: {row.get('url')}") console.print(f" Review Status: {row.get('review_status')}") console.print(f" Model: {row.get('distill_model')}") console.print(f" Tokens: {row.get('token_count_original')} → {row.get('token_count_distilled')}") if row.get("summary"): console.print(f"\n[bold]Summary:[/bold]\n{row['summary'][:500]}") if __name__ == "__main__": cli()