our-claude-skills/custom-skills/90-reference-curator/06-markdown-exporter/code/scripts/exporter.py

#!/usr/bin/env python3
"""Markdown Exporter CLI — export approved content as structured files.

Usage:
    python exporter.py project --output ~/reference-library/exports/ [--min-score 0.80]
    python exporter.py finetuning --output ~/reference-library/exports/training.jsonl
    python exporter.py index --output ~/reference-library/exports/INDEX.md
    python exporter.py crossrefs --input ~/reference-library/exports/
    python exporter.py verify --path ~/reference-library/exports/
    python exporter.py log --name "Jan 2025 Export" --type project_files --docs 40
"""

import json
import re
import sys
from collections import defaultdict
from datetime import datetime
from pathlib import Path

import click
from rich.console import Console
from rich.table import Table

from refcurator.db import db_session
from refcurator.utils import count_tokens, slugify

console = Console()


@click.group()
def cli():
    """Markdown Exporter — export approved references."""
    pass


@cli.command()
@click.option("--output", required=True, type=click.Path(), help="Output directory path")
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
@click.option("--structure", default="nested_by_topic",
              type=click.Choice(["nested_by_topic", "flat"]))
@click.option("--include-metadata", is_flag=True, default=True, help="Include source metadata")
def project(output, min_score, structure, include_metadata):
    """Export approved content as markdown project files."""
    out_dir = Path(output).expanduser()
    out_dir.mkdir(parents=True, exist_ok=True)

    with db_session() as db:
        rows = db.fetch_all(
            """SELECT d.doc_id, d.title, d.url,
                      dc.structured_content, dc.summary, dc.key_concepts,
                      dc.token_count_distilled,
                      rl.quality_score,
                      s.credibility_tier, s.vendor, s.source_name
               FROM documents d
               JOIN distilled_content dc ON d.doc_id = dc.doc_id
               JOIN review_logs rl ON dc.distill_id = rl.distill_id
               JOIN sources s ON d.source_id = s.source_id
               WHERE dc.review_status = %s
                 AND rl.decision = %s
                 AND rl.review_id = (
                     SELECT MAX(rl2.review_id)
                     FROM review_logs rl2
                     WHERE rl2.distill_id = dc.distill_id
                 )
               ORDER BY rl.quality_score DESC""",
            ("approved", "approve"),
        )

    # Filter by min score
    rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]

    if not rows:
        console.print("[yellow]No approved documents found above minimum score.[/yellow]")
        return

    # Get topic mappings
    with db_session() as db:
        topic_rows = db.fetch_all(
            """SELECT dt.doc_id, t.topic_name, t.topic_slug
               FROM document_topics dt
               JOIN topics t ON dt.topic_id = t.topic_id"""
        )

    doc_topics = defaultdict(list)
    for tr in topic_rows:
        doc_topics[tr["doc_id"]].append(tr)

    exported = 0
    total_tokens = 0

    if structure == "nested_by_topic":
        exported, total_tokens = _export_nested(rows, doc_topics, out_dir, include_metadata)
    else:
        exported, total_tokens = _export_flat(rows, doc_topics, out_dir, include_metadata)

    console.print(
        f"[green]Exported {exported} documents ({total_tokens:,} tokens) to {out_dir}[/green]"
    )


@cli.command()
@click.option("--output", required=True, type=click.Path(), help="Output JSONL file path")
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
@click.option("--max-tokens", default=4096, type=int, help="Max tokens per sample")
@click.option("--system-prompt", default="You are an expert on AI and prompt engineering.",
              help="System prompt for training samples")
def finetuning(output, min_score, max_tokens, system_prompt):
    """Export approved content as JSONL fine-tuning dataset."""
    out_path = Path(output).expanduser()
    out_path.parent.mkdir(parents=True, exist_ok=True)

    with db_session() as db:
        rows = db.fetch_all(
            """SELECT d.doc_id, d.title, d.url,
                      dc.structured_content, dc.summary,
                      dc.token_count_distilled,
                      rl.quality_score
               FROM documents d
               JOIN distilled_content dc ON d.doc_id = dc.doc_id
               JOIN review_logs rl ON dc.distill_id = rl.distill_id
               WHERE dc.review_status = %s
                 AND rl.decision = %s
                 AND rl.review_id = (
                     SELECT MAX(rl2.review_id)
                     FROM review_logs rl2
                     WHERE rl2.distill_id = dc.distill_id
                 )
               ORDER BY rl.quality_score DESC""",
            ("approved", "approve"),
        )

    rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]

    count = 0
    with open(out_path, "w") as f:
        for row in rows:
            content = row.get("structured_content", "")
            if count_tokens(content) > max_tokens:
                content = content[:max_tokens * 4]  # Approximate truncation

            sample = {
                "messages": [
                    {"role": "system", "content": system_prompt},
                    {"role": "user", "content": f"Explain {row.get('title', 'this topic')}"},
                    {"role": "assistant", "content": content},
                ],
                "metadata": {
                    "source": row.get("url"),
                    "quality_score": float(row["quality_score"]) if row.get("quality_score") else None,
                    "doc_id": row.get("doc_id"),
                },
            }
            f.write(json.dumps(sample, ensure_ascii=False) + "\n")
            count += 1

    console.print(f"[green]Exported {count} samples to {out_path}[/green]")


@cli.command()
@click.option("--output", required=True, type=click.Path(), help="Output INDEX.md path")
@click.option("--exports-dir", type=click.Path(exists=True),
              help="Exports directory to scan (defaults to parent of output)")
def index(output, exports_dir):
    """Generate INDEX.md with table of contents."""
    out_path = Path(output).expanduser()
    scan_dir = Path(exports_dir).expanduser() if exports_dir else out_path.parent

    lines = [
        "# Reference Library Index",
        "",
        f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}*",
        "",
    ]

    # Scan for topic directories
    topic_dirs = sorted([d for d in scan_dir.iterdir() if d.is_dir() and not d.name.startswith("_")])

    if topic_dirs:
        lines.append("## Topics\n")
        for topic_dir in topic_dirs:
            md_files = sorted(topic_dir.glob("*.md"))
            if md_files:
                topic_name = topic_dir.name.replace("-", " ").title()
                lines.append(f"### {topic_name}\n")
                for md_file in md_files:
                    if md_file.name.startswith("_"):
                        continue
                    title = _extract_md_title(md_file)
                    rel_path = md_file.relative_to(scan_dir)
                    lines.append(f"- [{title}]({rel_path})")
                lines.append("")

    # Scan for flat files
    flat_files = sorted([f for f in scan_dir.glob("*.md")
                         if f.name not in ("INDEX.md", "_index.md")])
    if flat_files:
        lines.append("## Documents\n")
        for md_file in flat_files:
            title = _extract_md_title(md_file)
            lines.append(f"- [{title}]({md_file.name})")
        lines.append("")

    out_path.write_text("\n".join(lines))
    console.print(f"[green]Generated INDEX.md at {out_path}[/green]")


@cli.command()
@click.option("--input", "input_dir", required=True, type=click.Path(exists=True),
              help="Exports directory to add cross-references to")
def crossrefs(input_dir):
    """Add cross-reference links between related documents."""
    scan_dir = Path(input_dir).expanduser()
    md_files = list(scan_dir.rglob("*.md"))

    if not md_files:
        console.print("[yellow]No markdown files found.[/yellow]")
        return

    # Build concept index: concept → list of (file, title)
    concept_index: dict[str, list[tuple[Path, str]]] = defaultdict(list)
    file_concepts: dict[Path, set[str]] = {}

    for md_file in md_files:
        if md_file.name in ("INDEX.md", "_index.md"):
            continue
        content = md_file.read_text(errors="replace")
        title = _extract_md_title(md_file)
        concepts = _extract_concepts(content)
        file_concepts[md_file] = concepts
        for concept in concepts:
            concept_index[concept].append((md_file, title))

    # Add cross-references
    modified = 0
    for md_file in md_files:
        if md_file.name in ("INDEX.md", "_index.md"):
            continue
        my_concepts = file_concepts.get(md_file, set())
        related: dict[str, str] = {}  # title → relative path

        for concept in my_concepts:
            for other_file, other_title in concept_index.get(concept, []):
                if other_file != md_file and other_title not in related:
                    try:
                        rel = other_file.relative_to(scan_dir)
                    except ValueError:
                        rel = other_file
                    related[other_title] = str(rel)

        if related and len(related) <= 10:
            content = md_file.read_text(errors="replace")
            # Remove existing Related section if present
            content = re.sub(r"\n## Related\n.*$", "", content, flags=re.DOTALL)
            content = content.rstrip() + "\n\n## Related\n\n"
            for title, path in sorted(related.items())[:5]:
                content += f"- [{title}]({path})\n"
            md_file.write_text(content)
            modified += 1

    console.print(f"[green]Added cross-references to {modified} files[/green]")


@cli.command()
@click.option("--path", required=True, type=click.Path(exists=True), help="Exports directory to verify")
def verify(path):
    """Verify export integrity."""
    scan_dir = Path(path).expanduser()
    md_files = list(scan_dir.rglob("*.md"))

    issues = []
    total_tokens = 0
    total_files = len(md_files)

    for md_file in md_files:
        content = md_file.read_text(errors="replace")
        tokens = count_tokens(content)
        total_tokens += tokens

        # Check for empty files
        if len(content.strip()) < 10:
            issues.append(f"Empty or near-empty: {md_file.relative_to(scan_dir)}")

        # Check for broken internal links
        for match in re.finditer(r"\[([^\]]+)\]\(([^)]+)\)", content):
            link_path = match.group(2)
            if link_path.startswith("http"):
                continue
            resolved = (md_file.parent / link_path).resolve()
            if not resolved.exists():
                issues.append(
                    f"Broken link in {md_file.relative_to(scan_dir)}: {link_path}"
                )

    # Check INDEX.md exists
    if not (scan_dir / "INDEX.md").is_file():
        issues.append("Missing INDEX.md")

    # Report
    console.print(f"\n[bold]Export Verification: {scan_dir}[/bold]")
    console.print(f"  Files: {total_files}")
    console.print(f"  Total tokens: {total_tokens:,}")

    if issues:
        console.print(f"\n[red]Issues ({len(issues)}):[/red]")
        for issue in issues[:20]:
            console.print(f"  - {issue}")
    else:
        console.print(f"\n[green]All checks passed.[/green]")

    click.echo(json.dumps({
        "files": total_files,
        "total_tokens": total_tokens,
        "issues": len(issues),
        "issue_details": issues[:20],
    }))


@cli.command()
@click.option("--name", required=True, help="Export job name")
@click.option("--type", "export_type", required=True,
              type=click.Choice(["project_files", "fine_tuning", "knowledge_base"]))
@click.option("--docs", required=True, type=int, help="Number of documents exported")
@click.option("--path", type=click.Path(), help="Export output path")
@click.option("--tokens", type=int, help="Total tokens exported")
def log(name, export_type, docs, path, tokens):
    """Log an export job to the database."""
    with db_session() as db:
        export_id = db.insert_returning_id(
            """INSERT INTO export_jobs
               (export_name, export_type, output_path, total_documents, total_tokens,
                status, started_at, completed_at)
               VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
            (name, export_type, path, docs, tokens,
             "completed", datetime.now().isoformat(), datetime.now().isoformat()),
        )

    console.print(f"[green]Logged export:[/green] export_id={export_id} — {name}")


# --- Helpers ---

def _export_nested(rows, doc_topics, out_dir, include_metadata):
    """Export documents in nested topic directory structure."""
    exported = 0
    total_tokens = 0
    topic_docs = defaultdict(list)

    for row in rows:
        topics = doc_topics.get(row["doc_id"], [])
        if topics:
            for t in topics:
                topic_docs[t["topic_slug"]].append(row)
        else:
            topic_docs["uncategorized"].append(row)

    for topic_slug, docs in sorted(topic_docs.items()):
        topic_dir = out_dir / topic_slug
        topic_dir.mkdir(parents=True, exist_ok=True)

        # Topic index
        topic_name = topic_slug.replace("-", " ").title()
        index_lines = [f"# {topic_name}\n"]

        for i, doc in enumerate(docs):
            filename = f"{i:02d}-{slugify(doc.get('title', 'untitled'))}.md"
            content = _format_document(doc, include_metadata)
            (topic_dir / filename).write_text(content)
            index_lines.append(f"- [{doc.get('title', 'Untitled')}]({filename})")
            exported += 1
            total_tokens += doc.get("token_count_distilled") or count_tokens(content)

        (topic_dir / "_index.md").write_text("\n".join(index_lines) + "\n")

    return exported, total_tokens


def _export_flat(rows, doc_topics, out_dir, include_metadata):
    """Export documents as flat files."""
    exported = 0
    total_tokens = 0

    for row in rows:
        topics = doc_topics.get(row["doc_id"], [])
        topic_prefix = topics[0]["topic_slug"] + "-" if topics else ""
        filename = f"{topic_prefix}{slugify(row.get('title', 'untitled'))}.md"
        content = _format_document(row, include_metadata)
        (out_dir / filename).write_text(content)
        exported += 1
        total_tokens += row.get("token_count_distilled") or count_tokens(content)

    return exported, total_tokens


def _format_document(row: dict, include_metadata: bool) -> str:
    """Format a document for export."""
    lines = []

    if include_metadata:
        lines.extend([
            f"# {row.get('title', 'Untitled')}",
            "",
            f"**Source:** {row.get('url', 'N/A')}",
            f"**Tier:** {row.get('credibility_tier', 'N/A')} | "
            f"**Vendor:** {row.get('vendor', 'N/A')} | "
            f"**Score:** {float(row['quality_score']):.2f}" if row.get("quality_score") else "",
            "",
            "---",
            "",
        ])

    content = row.get("structured_content", "")
    if content:
        lines.append(content)

    return "\n".join(lines)


def _extract_md_title(md_file: Path) -> str:
    """Extract title from a markdown file."""
    try:
        for line in md_file.read_text(errors="replace").split("\n")[:10]:
            if line.startswith("# ") and not line.startswith("##"):
                return line[2:].strip()
    except Exception:
        pass
    return md_file.stem.replace("-", " ").title()


def _extract_concepts(content: str) -> set[str]:
    """Extract key concepts from markdown content for cross-referencing."""
    concepts = set()

    # Extract from ## Key Concepts section
    m = re.search(r"## Key Concepts?\n(.*?)(?=\n##|\Z)", content, re.DOTALL)
    if m:
        for line in m.group(1).split("\n"):
            line = line.strip().lstrip("- *")
            if line and ":" in line:
                concept = line.split(":")[0].strip("*").strip()
                if 2 < len(concept) < 50:
                    concepts.add(concept.lower())

    # Extract bold terms
    for m in re.finditer(r"\*\*([^*]{3,40})\*\*", content):
        concepts.add(m.group(1).lower())

    return concepts


if __name__ == "__main__":
    cli()