diff --git a/custom-skills/90-reference-curator/01-reference-discovery/code/CLAUDE.md b/custom-skills/90-reference-curator/01-reference-discovery/code/CLAUDE.md index 8538377..2476b35 100644 --- a/custom-skills/90-reference-curator/01-reference-discovery/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/01-reference-discovery/code/CLAUDE.md @@ -36,8 +36,25 @@ Apply credibility scoring: - Relevance signals (0.15) ### Step 4: Output URL Manifest -Generate JSON manifest for the crawler skill: +Save discovered URLs as a manifest JSON, deduplicating against the existing repository: +```bash +# Create manifest from discovered URLs +uv run python scripts/discover.py create-manifest --topic "prompt engineering" --output manifest.json < urls.json + +# Deduplicate against existing DB +uv run python scripts/discover.py dedup --manifest manifest.json + +# Register a new source +uv run python scripts/discover.py register-source \ + --name "Anthropic Docs" --type official_docs \ + --url "https://docs.anthropic.com" --tier tier1_official --vendor anthropic + +# List registered sources +uv run python scripts/discover.py list-sources --vendor anthropic +``` + +Manifest format: ```json { "discovery_date": "2025-01-28T10:30:00", @@ -56,20 +73,31 @@ Generate JSON manifest for the crawler skill: } ``` -## Scripts - -### `discover_sources.py` -Main discovery script. Usage: -```bash -python scripts/discover_sources.py --topic "prompt engineering" --vendors anthropic,openai --output manifest.json -``` - ## Output - `manifest.json` → Handoff to `02-web-crawler-orchestrator` -- Register new sources in `sources` table via `03-content-repository` +- New sources registered in `sources` table via `register-source` ## Deduplication Before outputting: - Normalize URLs (remove trailing slashes, query params) - Check against existing `documents` table - Merge duplicates, keeping highest credibility score + +## Scripts + +All scripts require the `refcurator` package. Run with `uv run python` from the skill directory. + +| Command | Purpose | +|---------|---------| +| `discover.py create-manifest` | Create manifest from URL entries JSON | +| `discover.py dedup` | Deduplicate manifest against DB | +| `discover.py register-source` | Register a new source | +| `discover.py list-sources` | List registered sources | + +## Integration + +| From | To | +|------|-----| +| WebSearch results | → manifest.json | +| → manifest.json | web-crawler-orchestrator | +| → register-source | content-repository (sources table) | diff --git a/custom-skills/90-reference-curator/01-reference-discovery/code/scripts/discover.py b/custom-skills/90-reference-curator/01-reference-discovery/code/scripts/discover.py new file mode 100644 index 0000000..9870e5b --- /dev/null +++ b/custom-skills/90-reference-curator/01-reference-discovery/code/scripts/discover.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Reference Discovery CLI — manage source discovery and manifests. + +Claude performs the actual web search. This script handles: +- Receiving search results and creating manifests +- Deduplicating URLs against the existing repository +- Registering new sources in the database + +Usage: + python discover.py create-manifest --topic "prompt engineering" --output manifest.json < urls.json + python discover.py dedup --manifest manifest.json [--output deduped.json] + python discover.py register-source --name "Anthropic Docs" --type official_docs --url "https://docs.anthropic.com" --tier tier1_official --vendor anthropic + python discover.py list-sources [--vendor anthropic] +""" + +import json +import sys +from datetime import datetime +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.manifest import write_manifest, read_manifest, dedup_manifest_urls +from refcurator.models import Manifest, ManifestURL +from refcurator.utils import normalize_url + +console = Console() + + +@click.group() +def cli(): + """Reference Discovery — manage source manifests.""" + pass + + +@cli.command("create-manifest") +@click.option("--topic", required=True, help="Discovery topic") +@click.option("--output", required=True, type=click.Path(), help="Output manifest path") +@click.option("--input", "input_file", type=click.Path(exists=True), + help="Input JSON file with URL entries (or pipe via stdin)") +def create_manifest(topic, output, input_file): + """Create a manifest from discovered URLs. + + Input format (JSON array): + [ + {"url": "https://...", "title": "...", "credibility_score": 0.85, "source_type": "official_docs"}, + ... + ] + """ + if input_file: + data = json.loads(Path(input_file).read_text()) + elif not sys.stdin.isatty(): + data = json.load(sys.stdin) + else: + console.print("[red]Error:[/red] Provide URLs via --input file or stdin") + sys.exit(1) + + urls = [ManifestURL(**entry) for entry in data] + + # Normalize and deduplicate + seen = {} + for u in urls: + norm = normalize_url(u.url) + if norm not in seen or (u.credibility_score or 0) > (seen[norm].credibility_score or 0): + seen[norm] = u + + unique_urls = list(seen.values()) + + manifest = Manifest( + discovery_date=datetime.now().isoformat(), + topic=topic, + total_urls=len(unique_urls), + urls=unique_urls, + ) + + out_path = Path(output) + write_manifest(manifest, out_path) + console.print(f"[green]Created manifest:[/green] {len(unique_urls)} URLs → {output}") + + +@cli.command() +@click.option("--manifest", required=True, type=click.Path(exists=True), help="Manifest to dedup") +@click.option("--output", type=click.Path(), help="Output path (defaults to overwriting input)") +def dedup(manifest, output): + """Deduplicate manifest URLs against existing repository.""" + m = read_manifest(Path(manifest)) + original_count = len(m.urls) + + with db_session() as db: + existing = db.fetch_all("SELECT url FROM documents WHERE url IS NOT NULL") + existing_urls = {r["url"] for r in existing} + + deduped = dedup_manifest_urls(m, existing_urls) + removed = original_count - len(deduped.urls) + + out_path = Path(output) if output else Path(manifest) + write_manifest(deduped, out_path) + + console.print( + f"[green]Deduped:[/green] {original_count} → {len(deduped.urls)} URLs " + f"({removed} duplicates removed)" + ) + + +@cli.command("register-source") +@click.option("--name", required=True, help="Source name") +@click.option("--type", "source_type", required=True, + type=click.Choice(["official_docs", "engineering_blog", "research_paper", + "github_repo", "community_guide", "pdf_document", "api_reference"])) +@click.option("--url", required=True, help="Base URL") +@click.option("--tier", default="tier3_community", + type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"])) +@click.option("--vendor", help="Vendor name (e.g., anthropic, openai)") +def register_source(name, source_type, url, tier, vendor): + """Register a new source in the repository.""" + with db_session() as db: + # Check for existing source with same base_url + existing = db.fetch_one( + "SELECT source_id, source_name FROM sources WHERE base_url = %s", + (url,), + ) + if existing: + console.print( + f"[yellow]Already registered:[/yellow] source_id={existing['source_id']} " + f"({existing['source_name']})" + ) + return + + source_id = db.insert_returning_id( + """INSERT INTO sources (source_name, source_type, base_url, credibility_tier, vendor) + VALUES (%s, %s, %s, %s, %s)""", + (name, source_type, url, tier, vendor), + ) + + console.print(f"[green]Registered:[/green] source_id={source_id} — {name}") + click.echo(json.dumps({"source_id": source_id, "name": name, "url": url})) + + +@cli.command("list-sources") +@click.option("--vendor", help="Filter by vendor") +@click.option("--tier", type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]), + help="Filter by credibility tier") +def list_sources(vendor, tier): + """List registered sources.""" + with db_session() as db: + if vendor: + rows = db.fetch_all( + "SELECT * FROM sources WHERE vendor = %s ORDER BY credibility_tier", + (vendor,), + ) + elif tier: + rows = db.fetch_all( + "SELECT * FROM sources WHERE credibility_tier = %s ORDER BY vendor", + (tier,), + ) + else: + rows = db.fetch_all("SELECT * FROM sources ORDER BY credibility_tier, vendor") + + table = Table(title="Registered Sources") + table.add_column("ID", style="cyan") + table.add_column("Name") + table.add_column("Type") + table.add_column("Tier") + table.add_column("Vendor") + table.add_column("URL") + for r in rows: + table.add_row( + str(r.get("source_id", "")), + str(r.get("source_name", "")), + str(r.get("source_type", "")), + str(r.get("credibility_tier", "")), + str(r.get("vendor", "")), + str(r.get("base_url", ""))[:40], + ) + console.print(table) + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md index c4e05d2..5565071 100644 --- a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md @@ -1,115 +1,48 @@ # Web Crawler Orchestrator -Orchestrates web crawling with intelligent backend selection. Automatically chooses the best crawler based on site characteristics. +Orchestrates web crawling with intelligent backend selection. Claude performs actual crawling via Firecrawl MCP tools. This skill manages crawl results, selects crawlers, and tracks crawl metadata. ## Trigger Keywords "crawl URLs", "fetch documents", "scrape pages", "download references" ## Intelligent Crawler Selection -Claude automatically selects the optimal crawler based on the request: +```bash +# Get crawler recommendation for a URL +uv run python scripts/crawl_mgr.py select-crawler --url "https://docs.anthropic.com" +``` | Crawler | Best For | Auto-Selected When | |---------|----------|-------------------| -| **Node.js** (default) | Small docs sites | ≤50 pages, static content | +| **Firecrawl MCP** (default) | Dynamic sites, SPAs | React/Vue/Angular, JS-rendered | +| **Node.js** | Small docs sites | ≤50 pages, static content | | **Python aiohttp** | Technical docs | ≤200 pages, needs SEO data | | **Scrapy** | Enterprise crawls | >200 pages, multi-domain | -| **Firecrawl MCP** | Dynamic sites | SPAs, JS-rendered content | - -### Decision Flow - -``` -[Crawl Request] - │ - ├─ Is it SPA/React/Vue/Angular? → Firecrawl MCP - │ - ├─ >200 pages or multi-domain? → Scrapy - │ - ├─ Needs SEO extraction? → Python aiohttp - │ - └─ Default (small site) → Node.js -``` - -## Crawler Backends - -### Node.js (Default) -Fast, lightweight crawler for small documentation sites. -```bash -cd ~/Project/our-seo-agent/util/js-crawler -node src/crawler.js --max-pages 50 -``` - -### Python aiohttp -Async crawler with full SEO extraction. -```bash -cd ~/Project/our-seo-agent -python -m seo_agent.crawler --url --max-pages 100 -``` - -### Scrapy -Enterprise-grade crawler with pipelines. -```bash -cd ~/Project/our-seo-agent -scrapy crawl seo_spider -a start_url= -a max_pages=500 -``` - -### Firecrawl MCP -Use MCP tools for JavaScript-heavy sites: -``` -firecrawl_scrape(url, formats=["markdown"], only_main_content=true) -firecrawl_crawl(url, max_depth=2, limit=50) -firecrawl_map(url, limit=100) # Discover URLs first -``` ## Workflow ### Step 1: Analyze Target Site -Determine site characteristics: -- Is it a SPA? (React, Vue, Angular, Next.js) -- How many pages expected? -- Does it need JavaScript rendering? -- Is SEO data extraction needed? +Run `select-crawler` to determine site characteristics and get a recommendation. -### Step 2: Select Crawler -Based on analysis, select the appropriate backend. +### Step 2: Execute Crawl +Use Firecrawl MCP tools directly: +``` +firecrawl_map(url, limit=100) # Discover URLs +firecrawl_scrape(url, formats=["markdown"], only_main_content=true) +firecrawl_crawl(url, max_depth=2, limit=50) +``` -### Step 3: Load URL Manifest +### Step 3: Store Crawl Results ```bash -# From reference-discovery output -cat manifest.json | jq '.urls[].url' -``` +# Store crawled files and create result manifest +uv run python scripts/crawl_mgr.py store-result \ + --raw-dir ~/Documents/reference-library/raw/ \ + --crawler firecrawl \ + --source-id 1 \ + --output crawl_result.json -### Step 4: Execute Crawl - -**For Node.js:** -```bash -cd ~/Project/our-seo-agent/util/js-crawler -for url in $(cat urls.txt); do - node src/crawler.js "$url" --max-pages 50 - sleep 2 -done -``` - -**For Firecrawl MCP (Claude Desktop/Code):** -Use the firecrawl MCP tools directly in conversation. - -### Step 5: Save Raw Content -``` -~/reference-library/raw/ -└── 2025/01/ - ├── a1b2c3d4.md - └── b2c3d4e5.md -``` - -### Step 6: Generate Crawl Manifest -```json -{ - "crawl_date": "2025-01-28T12:00:00", - "crawler_used": "nodejs", - "total_crawled": 45, - "total_failed": 5, - "documents": [...] -} +# List recent crawls +uv run python scripts/crawl_mgr.py list-crawls --status completed ``` ## Rate Limiting @@ -129,31 +62,19 @@ All crawlers respect these limits: | Access denied (403) | Log, mark as `failed` | | JS rendering needed | Switch to Firecrawl | -## Site Type Detection - -Indicators for automatic routing: - -**SPA (→ Firecrawl):** -- URL contains `#/` or uses hash routing -- Page source shows React/Vue/Angular markers -- Content loads dynamically after initial load - -**Static docs (→ Node.js/aiohttp):** -- Built with Hugo, Jekyll, MkDocs, Docusaurus, GitBook -- Clean HTML structure -- Server-side rendered - ## Scripts -- `scripts/select_crawler.py` - Intelligent crawler selection -- `scripts/crawl_with_nodejs.sh` - Node.js wrapper -- `scripts/crawl_with_aiohttp.sh` - Python wrapper -- `scripts/crawl_with_firecrawl.py` - Firecrawl MCP wrapper +| Command | Purpose | +|---------|---------| +| `crawl_mgr.py select-crawler` | Recommend optimal crawler for a URL | +| `crawl_mgr.py store-result` | Store crawl results and create manifest | +| `crawl_mgr.py list-crawls` | List recent crawl records | ## Integration | From | To | |------|-----| | reference-discovery | URL manifest input | +| Firecrawl MCP | Raw crawled files | | → | content-repository (crawl manifest + raw files) | | quality-reviewer (deep_research) | Additional crawl requests | diff --git a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/crawl_mgr.py b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/crawl_mgr.py new file mode 100644 index 0000000..8ec946d --- /dev/null +++ b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/crawl_mgr.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Web Crawler Manager CLI — manage crawl results and crawler selection. + +Claude performs actual crawling via Firecrawl MCP. This script handles: +- Storing crawl results metadata +- Selecting the optimal crawler backend +- Generating crawl result manifests + +Usage: + python crawl_mgr.py store-result --manifest manifest.json --raw-dir ~/reference-library/raw/ + python crawl_mgr.py select-crawler --url "https://docs.anthropic.com" + python crawl_mgr.py list-crawls [--status completed] +""" + +import json +import re +import sys +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.manifest import create_crawl_result, write_crawl_result, read_manifest +from refcurator.utils import count_tokens + +console = Console() + + +# Known SPA frameworks and their indicators +SPA_INDICATORS = { + "react": ["react", "next.js", "nextjs", "gatsby", "remix"], + "vue": ["vue", "nuxt"], + "angular": ["angular"], + "svelte": ["svelte", "sveltekit"], +} + +# Known static site generators +STATIC_INDICATORS = ["hugo", "jekyll", "mkdocs", "docusaurus", "gitbook", "sphinx", + "readthedocs", "vuepress", "docsify"] + + +@click.group() +def cli(): + """Web Crawler Manager — manage crawl results and backend selection.""" + pass + + +@cli.command("store-result") +@click.option("--raw-dir", required=True, type=click.Path(exists=True), + help="Directory containing crawled raw files") +@click.option("--crawler", default="firecrawl", + type=click.Choice(["firecrawl", "nodejs", "aiohttp", "scrapy"])) +@click.option("--source-id", type=int, help="Source ID to associate documents with") +@click.option("--output", type=click.Path(), help="Output crawl result JSON path") +def store_result(raw_dir, crawler, source_id, output): + """Store crawl results from raw files into the repository. + + Scans raw_dir for .md files, creates document records, and generates a crawl result manifest. + """ + raw_path = Path(raw_dir) + md_files = sorted(raw_path.glob("**/*.md")) + + if not md_files: + console.print(f"[yellow]No .md files found in {raw_dir}[/yellow]") + sys.exit(0) + + entries = [] + stored = 0 + + with db_session() as db: + for md_file in md_files: + content = md_file.read_text(errors="replace") + content_size = md_file.stat().st_size + + # Extract title from first heading or filename + title = _extract_title(content, md_file.stem) + + # Extract URL from frontmatter if available + url = _extract_url_from_content(content) + + entry = { + "url": url or f"file://{md_file.resolve()}", + "title": title, + "raw_path": str(md_file.resolve()), + "content_size": content_size, + "status": "completed", + } + entries.append(entry) + + if source_id: + db.insert_returning_id( + """INSERT INTO documents + (source_id, title, url, doc_type, crawl_date, + crawl_method, crawl_status, raw_content_path, raw_content_size) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""", + (source_id, title, url, "markdown", datetime.now().isoformat(), + crawler, "completed", str(md_file.resolve()), content_size), + ) + stored += 1 + + result = create_crawl_result(entries, crawler) + + if output: + write_crawl_result(result, Path(output)) + console.print(f"[green]Crawl result written to {output}[/green]") + + console.print( + f"[green]Processed {len(entries)} files[/green]" + + (f", stored {stored} documents" if stored else "") + ) + click.echo(result.model_dump_json(indent=2)) + + +@cli.command("select-crawler") +@click.option("--url", required=True, help="Target URL to analyze") +@click.option("--page-count", type=int, help="Expected page count") +@click.option("--json-output", is_flag=True, help="Output as JSON") +def select_crawler(url, page_count, json_output): + """Recommend the best crawler backend for a URL. + + Analyzes URL patterns and site characteristics to suggest: + - firecrawl: SPAs, JS-rendered content + - nodejs: Small static docs sites (<=50 pages) + - aiohttp: Medium technical docs (<=200 pages) + - scrapy: Large enterprise sites (>200 pages) + """ + parsed = urlparse(url) + domain = parsed.netloc.lower() + path = parsed.path.lower() + + recommendation = "firecrawl" # Default + reason = "Default recommendation for general crawling" + confidence = 0.7 + + # Check for SPA indicators in domain/path + url_lower = url.lower() + for framework, indicators in SPA_INDICATORS.items(): + if any(ind in url_lower or ind in domain for ind in indicators): + recommendation = "firecrawl" + reason = f"Detected {framework} SPA framework — needs JS rendering" + confidence = 0.9 + break + + # Check for static site indicators + for gen in STATIC_INDICATORS: + if gen in url_lower or gen in domain: + if page_count and page_count <= 50: + recommendation = "nodejs" + reason = f"Static site ({gen}), small page count — lightweight crawler sufficient" + else: + recommendation = "aiohttp" + reason = f"Static site ({gen}), moderate size — async crawler recommended" + confidence = 0.85 + break + + # Page count override (takes precedence over default, not over SPA detection) + spa_detected = confidence >= 0.9 and recommendation == "firecrawl" + if page_count and not spa_detected: + if page_count > 200: + recommendation = "scrapy" + reason = f"Large site ({page_count} pages) — enterprise crawler needed" + confidence = 0.9 + elif page_count <= 50: + recommendation = "nodejs" + reason = f"Small site ({page_count} pages) — lightweight crawler sufficient" + confidence = 0.8 + + # Known documentation platforms → firecrawl + doc_platforms = ["docs.", "developer.", "api.", "reference."] + if any(domain.startswith(p) for p in doc_platforms): + if recommendation == "firecrawl": + reason = "Documentation platform — Firecrawl handles dynamic docs well" + confidence = 0.85 + + result = { + "url": url, + "recommendation": recommendation, + "reason": reason, + "confidence": confidence, + "alternatives": _get_alternatives(recommendation), + } + + if json_output: + click.echo(json.dumps(result, indent=2)) + else: + console.print(f"\n[bold]Crawler Recommendation for:[/bold] {url}") + console.print(f" [green]Recommended:[/green] {recommendation}") + console.print(f" [dim]Reason:[/dim] {reason}") + console.print(f" [dim]Confidence:[/dim] {confidence:.0%}") + if result["alternatives"]: + console.print(f" [dim]Alternatives:[/dim] {', '.join(result['alternatives'])}") + + +@cli.command("list-crawls") +@click.option("--status", type=click.Choice(["pending", "completed", "failed", "stale"]), + help="Filter by crawl status") +@click.option("--limit", default=20, type=int, help="Max results") +def list_crawls(status, limit): + """List recent crawl records.""" + with db_session() as db: + if status: + rows = db.fetch_all( + """SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date + FROM documents WHERE crawl_status = %s + ORDER BY crawl_date DESC LIMIT %s""", + (status, limit), + ) + else: + rows = db.fetch_all( + """SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date + FROM documents ORDER BY crawl_date DESC LIMIT %s""", + (limit,), + ) + + table = Table(title="Crawl Records") + table.add_column("ID", style="cyan") + table.add_column("Title") + table.add_column("Crawler") + table.add_column("Status") + table.add_column("Date") + for r in rows: + table.add_row( + str(r.get("doc_id", "")), + str(r.get("title", ""))[:40], + str(r.get("crawl_method", "")), + str(r.get("crawl_status", "")), + str(r.get("crawl_date", ""))[:10], + ) + console.print(table) + + +def _extract_title(content: str, fallback: str) -> str: + """Extract title from markdown content.""" + for line in content.split("\n")[:10]: + line = line.strip() + if line.startswith("# ") and not line.startswith("##"): + return line[2:].strip() + return fallback.replace("-", " ").replace("_", " ").title() + + +def _extract_url_from_content(content: str) -> str | None: + """Extract source URL from markdown frontmatter or content.""" + # YAML frontmatter + if content.startswith("---"): + fm_end = content.find("---", 3) + if fm_end > 0: + fm = content[3:fm_end] + for line in fm.split("\n"): + if line.strip().startswith("url:") or line.strip().startswith("source:"): + url = line.split(":", 1)[1].strip().strip("'\"") + if url.startswith("http"): + return url + + # Source: URL pattern in content + m = re.search(r"\*\*Source:\*\*\s*(https?://\S+)", content[:500]) + if m: + return m.group(1) + + return None + + +def _get_alternatives(primary: str) -> list[str]: + """Get alternative crawler recommendations.""" + all_crawlers = ["firecrawl", "nodejs", "aiohttp", "scrapy"] + return [c for c in all_crawlers if c != primary][:2] + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/03-content-repository/code/CLAUDE.md b/custom-skills/90-reference-curator/03-content-repository/code/CLAUDE.md index 7c0ad07..1d3b58a 100644 --- a/custom-skills/90-reference-curator/03-content-repository/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/03-content-repository/code/CLAUDE.md @@ -1,46 +1,42 @@ # Content Repository -MySQL storage management for the reference library. Handles document storage, version control, deduplication, and retrieval. +MySQL storage management for the reference library. Handles document storage, version control, deduplication, and retrieval. Supports MySQL primary backend with JSON file fallback. ## Trigger Keywords "store content", "save to database", "check duplicates", "version tracking", "document retrieval", "reference library DB" ## Prerequisites -- MySQL 8.0+ with utf8mb4 charset -- Config file at `~/.config/reference-curator/db_config.yaml` -- Database `reference_library` initialized +- `refcurator` package installed (`uv pip install -e shared/lib/`) +- MySQL 8.0+ (optional — falls back to JSON file storage) +- Config at `~/.config/reference-curator/db_config.yaml` (optional) -## Database Setup +## Quick Start ```bash -# Initialize database -mysql -u root -p < references/schema.sql - -# Verify tables -mysql -u root -p reference_library -e "SHOW TABLES;" -``` - -## Core Scripts - -### Store Document -```bash -python scripts/store_document.py \ - --source-id 1 \ - --title "Prompt Engineering Guide" \ +# Store a document +uv run python scripts/repo.py store \ + --source-id 1 --title "Prompt Engineering Guide" \ --url "https://docs.anthropic.com/..." \ - --doc-type webpage \ - --raw-path ~/reference-library/raw/2025/01/abc123.md -``` + --doc-type webpage --raw-path ~/reference-library/raw/abc123.md -### Check Duplicate -```bash -python scripts/check_duplicate.py --url "https://docs.anthropic.com/..." -``` +# Check for duplicates +uv run python scripts/repo.py check-dup --url "https://docs.anthropic.com/..." -### Query by Topic -```bash -python scripts/query_topic.py --topic-slug prompt-engineering --min-quality 0.80 +# Query by topic +uv run python scripts/repo.py query-topic --topic-slug prompt-engineering --min-quality 0.80 + +# Get repository stats +uv run python scripts/repo.py stats + +# Find stale documents (older than 30 days) +uv run python scripts/repo.py find-stale --days 30 + +# Get pending reviews +uv run python scripts/repo.py pending-reviews --output pending.json + +# Get export-ready content +uv run python scripts/repo.py export-ready --min-score 0.85 ``` ## Table Quick Reference @@ -61,31 +57,17 @@ python scripts/query_topic.py --topic-slug prompt-engineering --min-quality 0.80 **review_status:** `pending` → `in_review` → `approved` | `needs_refactor` | `rejected` -## Common Queries - -### Find Stale Documents -```bash -python scripts/find_stale.py --output stale_docs.json -``` - -### Get Pending Reviews -```bash -python scripts/pending_reviews.py --output pending.json -``` - -### Export-Ready Content -```bash -python scripts/export_ready.py --min-score 0.85 --output ready.json -``` - ## Scripts -- `scripts/store_document.py` - Store new document -- `scripts/check_duplicate.py` - URL deduplication -- `scripts/query_topic.py` - Query by topic -- `scripts/find_stale.py` - Find stale documents -- `scripts/pending_reviews.py` - Get pending reviews -- `scripts/db_utils.py` - Database connection utilities +| Command | Purpose | +|---------|---------| +| `repo.py store` | Store a new document | +| `repo.py check-dup` | URL deduplication check | +| `repo.py query-topic` | Query documents by topic | +| `repo.py find-stale` | Find stale documents | +| `repo.py pending-reviews` | Get pending reviews | +| `repo.py export-ready` | Get approved content ready for export | +| `repo.py stats` | Show repository statistics | ## Integration diff --git a/custom-skills/90-reference-curator/03-content-repository/code/scripts/repo.py b/custom-skills/90-reference-curator/03-content-repository/code/scripts/repo.py new file mode 100644 index 0000000..fa92157 --- /dev/null +++ b/custom-skills/90-reference-curator/03-content-repository/code/scripts/repo.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Content Repository CLI — CRUD operations for the reference library. + +Usage: + python repo.py store --source-id 1 --title "Doc" --url "https://..." --doc-type webpage --raw-path /path/to/file + python repo.py check-dup --url "https://..." + python repo.py query-topic --topic-slug prompt-engineering [--min-quality 0.80] + python repo.py find-stale [--output stale.json] + python repo.py pending-reviews [--output pending.json] + python repo.py export-ready [--min-score 0.85] [--output ready.json] + python repo.py stats +""" + +import json +import sys +from datetime import datetime +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.utils import url_hash, normalize_url + +console = Console() + + +@click.group() +def cli(): + """Content Repository — manage documents in the reference library.""" + pass + + +@cli.command() +@click.option("--source-id", required=True, type=int, help="Source ID from sources table") +@click.option("--title", required=True, help="Document title") +@click.option("--url", required=True, help="Document URL") +@click.option("--doc-type", required=True, + type=click.Choice(["webpage", "pdf", "markdown", "api_spec", "code_sample"])) +@click.option("--raw-path", required=True, type=click.Path(), help="Path to raw content file") +@click.option("--crawl-method", default="firecrawl", + type=click.Choice(["firecrawl", "scrapy", "aiohttp", "nodejs", "manual", "api"])) +@click.option("--language", default="en", type=click.Choice(["en", "ko", "mixed"])) +def store(source_id, title, url, doc_type, raw_path, crawl_method, language): + """Store a new document in the repository.""" + raw = Path(raw_path) + if not raw.is_file(): + console.print(f"[red]Error:[/red] Raw file not found: {raw_path}") + sys.exit(1) + + content_size = raw.stat().st_size + + with db_session() as db: + # Check for duplicate + existing = db.fetch_one( + "SELECT doc_id, title FROM documents WHERE url_hash = %s", + (url_hash(url),), + ) + if existing: + console.print( + f"[yellow]Duplicate:[/yellow] URL already stored as doc_id={existing['doc_id']} " + f"({existing['title']})" + ) + sys.exit(0) + + doc_id = db.insert_returning_id( + """INSERT INTO documents + (source_id, title, url, doc_type, language, crawl_date, + crawl_method, crawl_status, raw_content_path, raw_content_size) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", + (source_id, title, url, doc_type, language, datetime.now().isoformat(), + crawl_method, "completed", str(raw.resolve()), content_size), + ) + + console.print(f"[green]Stored:[/green] doc_id={doc_id} — {title}") + click.echo(json.dumps({"doc_id": doc_id, "title": title, "url": url})) + + +@cli.command("check-dup") +@click.option("--url", required=True, help="URL to check for duplicates") +def check_dup(url): + """Check if a URL already exists in the repository.""" + h = url_hash(url) + + with db_session() as db: + existing = db.fetch_one( + "SELECT doc_id, title, url, crawl_status FROM documents WHERE url_hash = %s", + (h,), + ) + + if existing: + console.print(f"[yellow]Duplicate found:[/yellow] doc_id={existing['doc_id']}") + click.echo(json.dumps(existing, default=str)) + else: + console.print("[green]No duplicate found.[/green]") + click.echo(json.dumps({"duplicate": False, "url": url})) + + +@cli.command("query-topic") +@click.option("--topic-slug", required=True, help="Topic slug to query") +@click.option("--min-quality", default=0.0, type=float, help="Minimum quality score") +@click.option("--output", type=click.Path(), help="Output JSON file path") +def query_topic(topic_slug, min_quality, output): + """Query documents by topic.""" + with db_session() as db: + rows = db.fetch_all( + """SELECT d.doc_id, d.title, d.url, d.crawl_status, + dt.relevance_score, t.topic_name + FROM documents d + JOIN document_topics dt ON d.doc_id = dt.doc_id + JOIN topics t ON dt.topic_id = t.topic_id + WHERE t.topic_slug = %s + ORDER BY dt.relevance_score DESC""", + (topic_slug,), + ) + + if min_quality > 0: + # Filter by review score if available + rows = [r for r in rows if r.get("relevance_score", 0) >= min_quality] + + if output: + Path(output).write_text(json.dumps(rows, indent=2, default=str)) + console.print(f"[green]Wrote {len(rows)} results to {output}[/green]") + else: + _print_doc_table(rows, f"Topic: {topic_slug}") + + +@cli.command("find-stale") +@click.option("--days", default=30, type=int, help="Documents older than N days") +@click.option("--output", type=click.Path(), help="Output JSON file path") +def find_stale(days, output): + """Find documents that may be outdated.""" + with db_session() as db: + rows = db.fetch_all( + """SELECT doc_id, title, url, crawl_date, crawl_status + FROM documents + WHERE crawl_status = %s + ORDER BY crawl_date ASC""", + ("completed",), + ) + + # Filter by age + cutoff = datetime.now() + stale = [] + for r in rows: + crawl_date = r.get("crawl_date") + if crawl_date: + if isinstance(crawl_date, str): + crawl_date = datetime.fromisoformat(crawl_date) + age_days = (cutoff - crawl_date).days + if age_days >= days: + r["age_days"] = age_days + stale.append(r) + + if output: + Path(output).write_text(json.dumps(stale, indent=2, default=str)) + console.print(f"[green]Found {len(stale)} stale documents, wrote to {output}[/green]") + else: + _print_doc_table(stale, f"Stale documents (>{days} days)") + + +@cli.command("pending-reviews") +@click.option("--output", type=click.Path(), help="Output JSON file path") +def pending_reviews(output): + """Get documents pending quality review.""" + with db_session() as db: + rows = db.fetch_all( + """SELECT dc.distill_id, d.doc_id, d.title, d.url, + dc.token_count_distilled, dc.distill_date + FROM distilled_content dc + JOIN documents d ON dc.doc_id = d.doc_id + WHERE dc.review_status = %s + ORDER BY dc.distill_date ASC""", + ("pending",), + ) + + if output: + Path(output).write_text(json.dumps(rows, indent=2, default=str)) + console.print(f"[green]{len(rows)} pending reviews, wrote to {output}[/green]") + else: + table = Table(title="Pending Reviews") + table.add_column("distill_id", style="cyan") + table.add_column("doc_id") + table.add_column("Title") + table.add_column("Tokens", justify="right") + for r in rows: + table.add_row( + str(r.get("distill_id", "")), + str(r.get("doc_id", "")), + str(r.get("title", ""))[:50], + str(r.get("token_count_distilled", "")), + ) + console.print(table) + + +@cli.command("export-ready") +@click.option("--min-score", default=0.80, type=float, help="Minimum quality score") +@click.option("--output", type=click.Path(), help="Output JSON file path") +def export_ready(min_score, output): + """Get documents approved and ready for export.""" + with db_session() as db: + rows = db.fetch_all( + """SELECT d.doc_id, d.title, d.url, + dc.structured_content, dc.token_count_distilled, + rl.quality_score, rl.decision + FROM documents d + JOIN distilled_content dc ON d.doc_id = dc.doc_id + JOIN review_logs rl ON dc.distill_id = rl.distill_id + WHERE dc.review_status = %s + AND rl.decision = %s + AND rl.review_id = ( + SELECT MAX(rl2.review_id) + FROM review_logs rl2 + WHERE rl2.distill_id = dc.distill_id + ) + ORDER BY rl.quality_score DESC""", + ("approved", "approve"), + ) + + # Filter by min score + rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score] + + if output: + Path(output).write_text(json.dumps(rows, indent=2, default=str)) + console.print(f"[green]{len(rows)} export-ready documents, wrote to {output}[/green]") + else: + table = Table(title=f"Export-Ready (score >= {min_score})") + table.add_column("doc_id", style="cyan") + table.add_column("Title") + table.add_column("Score", justify="right") + table.add_column("Tokens", justify="right") + for r in rows: + table.add_row( + str(r.get("doc_id", "")), + str(r.get("title", ""))[:50], + f"{r.get('quality_score', 0):.2f}", + str(r.get("token_count_distilled", "")), + ) + console.print(table) + + +@cli.command() +def stats(): + """Show repository statistics.""" + with db_session() as db: + doc_count = db.fetch_one("SELECT COUNT(*) as cnt FROM documents") or {"cnt": 0} + source_count = db.fetch_one("SELECT COUNT(*) as cnt FROM sources") or {"cnt": 0} + + status_rows = db.fetch_all( + """SELECT crawl_status, COUNT(*) as cnt + FROM documents + GROUP BY crawl_status""" + ) + + review_rows = db.fetch_all( + """SELECT review_status, COUNT(*) as cnt + FROM distilled_content + GROUP BY review_status""" + ) + + topic_rows = db.fetch_all( + """SELECT t.topic_name, COUNT(dt.doc_id) as cnt + FROM topics t + LEFT JOIN document_topics dt ON t.topic_id = dt.topic_id + GROUP BY t.topic_id + ORDER BY cnt DESC""" + ) + + console.print() + console.print(f"[bold]Reference Library Statistics[/bold]") + console.print(f" Sources: {source_count['cnt']}") + console.print(f" Documents: {doc_count['cnt']}") + console.print() + + if status_rows: + table = Table(title="Documents by Crawl Status") + table.add_column("Status") + table.add_column("Count", justify="right") + for r in status_rows: + table.add_row(str(r["crawl_status"]), str(r["cnt"])) + console.print(table) + + if review_rows: + table = Table(title="Distilled Content by Review Status") + table.add_column("Status") + table.add_column("Count", justify="right") + for r in review_rows: + table.add_row(str(r["review_status"]), str(r["cnt"])) + console.print(table) + + if topic_rows: + table = Table(title="Documents by Topic") + table.add_column("Topic") + table.add_column("Documents", justify="right") + for r in topic_rows: + table.add_row(str(r["topic_name"]), str(r["cnt"])) + console.print(table) + + result = { + "sources": source_count["cnt"], + "documents": doc_count["cnt"], + "by_status": {r["crawl_status"]: r["cnt"] for r in status_rows}, + "by_review": {r["review_status"]: r["cnt"] for r in review_rows}, + "by_topic": {r["topic_name"]: r["cnt"] for r in topic_rows}, + } + click.echo(json.dumps(result, default=str)) + + +def _print_doc_table(rows: list[dict], title: str): + """Print a table of documents.""" + table = Table(title=title) + table.add_column("doc_id", style="cyan") + table.add_column("Title") + table.add_column("URL") + table.add_column("Status") + for r in rows: + table.add_row( + str(r.get("doc_id", "")), + str(r.get("title", ""))[:40], + str(r.get("url", ""))[:50], + str(r.get("crawl_status", "")), + ) + console.print(table) + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/04-content-distiller/code/CLAUDE.md b/custom-skills/90-reference-curator/04-content-distiller/code/CLAUDE.md index 4f33b85..620190f 100644 --- a/custom-skills/90-reference-curator/04-content-distiller/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/04-content-distiller/code/CLAUDE.md @@ -1,44 +1,58 @@ # Content Distiller -Analyzes and distills raw crawled content into concise reference materials. Extracts key concepts, code snippets, and creates structured summaries. +Analyzes and distills raw crawled content into concise reference materials. Claude performs the actual distillation (summarization, key concept extraction). Scripts handle data loading and storage. ## Trigger Keywords "distill content", "summarize document", "extract key concepts", "process raw content", "create reference summary" ## Goals -1. **Compress** - Reduce token count while preserving essential information -2. **Structure** - Organize content for easy retrieval -3. **Extract** - Pull out code snippets, key concepts, patterns -4. **Annotate** - Add metadata for searchability +1. **Compress** — Reduce token count while preserving essential information +2. **Structure** — Organize content for easy retrieval +3. **Extract** — Pull out code snippets, key concepts, patterns +4. **Annotate** — Add metadata for searchability ## Workflow -### Step 1: Load Raw Content +### Step 1: Load Pending Documents ```bash -python scripts/load_pending.py --output pending_docs.json +uv run python scripts/distiller.py load-pending --output pending.json ``` -### Step 2: Analyze Content Structure -Identify document characteristics: -- Has code blocks? -- Has headers? -- Has tables? -- Estimated tokens? - -### Step 3: Extract Key Components -```bash -python scripts/extract_components.py --doc-id 123 --output components.json -``` - -Extracts: -- Code snippets with language tags -- Key concepts and definitions -- Best practices +### Step 2: Analyze and Distill (Claude) +For each pending document, Claude reads the raw content and creates: +- Executive summary (2-3 sentences) +- Key concepts with definitions - Techniques and patterns +- Code examples +- Best practices + +### Step 3: Store Distilled Content +```bash +uv run python scripts/distiller.py store \ + --doc-id 123 \ + --content distilled.md \ + --summary summary.txt \ + --concepts concepts.json \ + --snippets snippets.json \ + --model claude-opus-4-6 +``` + +### Step 4: Handle Refactor Requests +When quality-reviewer returns `refactor`, load context for re-distillation: +```bash +uv run python scripts/distiller.py refactor --distill-id 456 --output context.json +``` + +This outputs a context bundle with the current distilled content, raw source, and all review feedback. + +### Step 5: View Distilled Content +```bash +uv run python scripts/distiller.py show --distill-id 456 +``` + +## Distilled Output Template -### Step 4: Create Structured Summary -Output template: ```markdown # {title} @@ -62,17 +76,6 @@ Output template: {actionable recommendations} ``` -### Step 5: Optimize for Tokens -Target: 25-35% of original token count -```bash -python scripts/optimize_content.py --doc-id 123 --target-ratio 0.30 -``` - -### Step 6: Store Distilled Content -```bash -python scripts/store_distilled.py --doc-id 123 --content distilled.md -``` - ## Quality Metrics | Metric | Target | @@ -82,20 +85,14 @@ python scripts/store_distilled.py --doc-id 123 --content distilled.md | Code Snippet Retention | 100% of relevant examples | | Readability | Clear, scannable structure | -## Handling Refactor Requests - -When `quality-reviewer` returns `refactor`: -```bash -python scripts/refactor_content.py --distill-id 456 --instructions "Add more examples" -``` - ## Scripts -- `scripts/load_pending.py` - Load documents pending distillation -- `scripts/extract_components.py` - Extract code, concepts, patterns -- `scripts/optimize_content.py` - Token optimization -- `scripts/store_distilled.py` - Save to database -- `scripts/refactor_content.py` - Handle refactor requests +| Command | Purpose | +|---------|---------| +| `distiller.py load-pending` | Load documents pending distillation | +| `distiller.py store` | Save distilled content to DB | +| `distiller.py refactor` | Load context for re-distillation | +| `distiller.py show` | Show distilled content details | ## Integration diff --git a/custom-skills/90-reference-curator/04-content-distiller/code/scripts/distiller.py b/custom-skills/90-reference-curator/04-content-distiller/code/scripts/distiller.py new file mode 100644 index 0000000..15e5aff --- /dev/null +++ b/custom-skills/90-reference-curator/04-content-distiller/code/scripts/distiller.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Content Distiller CLI — manage distillation data I/O. + +Claude performs the actual distillation (summarization, extraction). +This script handles loading raw content from the DB and storing distilled output. + +Usage: + python distiller.py load-pending [--output pending.json] + python distiller.py store --doc-id 123 --content distilled.md [--model claude-opus-4-6] + python distiller.py refactor --distill-id 456 [--output context.json] + python distiller.py show --distill-id 456 +""" + +import json +import sys +from datetime import datetime +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.utils import count_tokens + +console = Console() + + +@click.group() +def cli(): + """Content Distiller — manage distillation data.""" + pass + + +@cli.command("load-pending") +@click.option("--output", type=click.Path(), help="Output JSON file path") +@click.option("--limit", default=50, type=int, help="Max documents to load") +def load_pending(output, limit): + """Load documents pending distillation. + + Finds documents with crawl_status='completed' that have no distilled content yet. + """ + with db_session() as db: + rows = db.fetch_all( + """SELECT d.doc_id, d.title, d.url, d.raw_content_path, + d.raw_content_size, d.doc_type, + s.source_name, s.credibility_tier + FROM documents d + JOIN sources s ON d.source_id = s.source_id + LEFT JOIN distilled_content dc ON d.doc_id = dc.doc_id + WHERE d.crawl_status = %s AND dc.distill_id IS NULL + ORDER BY s.credibility_tier ASC, d.crawl_date ASC + LIMIT %s""", + ("completed", limit), + ) + + # Enrich with raw content preview + for row in rows: + raw_path = row.get("raw_content_path") + if raw_path and Path(raw_path).is_file(): + content = Path(raw_path).read_text(errors="replace") + row["token_count_estimate"] = count_tokens(content) + row["content_preview"] = content[:200] + "..." if len(content) > 200 else content + else: + row["token_count_estimate"] = 0 + row["content_preview"] = "[file not found]" + + if output: + Path(output).write_text(json.dumps(rows, indent=2, default=str)) + console.print(f"[green]{len(rows)} pending documents written to {output}[/green]") + else: + table = Table(title=f"Pending Distillation ({len(rows)} documents)") + table.add_column("doc_id", style="cyan") + table.add_column("Title") + table.add_column("Source") + table.add_column("Tier") + table.add_column("~Tokens", justify="right") + for r in rows: + table.add_row( + str(r.get("doc_id", "")), + str(r.get("title", ""))[:40], + str(r.get("source_name", ""))[:20], + str(r.get("credibility_tier", "")), + str(r.get("token_count_estimate", "")), + ) + console.print(table) + + click.echo(json.dumps({"count": len(rows)}, default=str)) + + +@cli.command() +@click.option("--doc-id", required=True, type=int, help="Document ID to store distilled content for") +@click.option("--content", required=True, type=click.Path(exists=True), + help="Path to distilled markdown content") +@click.option("--summary", type=click.Path(exists=True), help="Path to summary text file") +@click.option("--concepts", type=click.Path(exists=True), help="Path to key concepts JSON") +@click.option("--snippets", type=click.Path(exists=True), help="Path to code snippets JSON") +@click.option("--model", default="claude-opus-4-6", help="Model used for distillation") +def store(doc_id, content, summary, concepts, snippets, model): + """Store distilled content for a document.""" + structured = Path(content).read_text(errors="replace") + token_distilled = count_tokens(structured) + + summary_text = Path(summary).read_text() if summary else None + concepts_json = json.loads(Path(concepts).read_text()) if concepts else None + snippets_json = json.loads(Path(snippets).read_text()) if snippets else None + + with db_session() as db: + # Get original token count + doc = db.fetch_one( + "SELECT raw_content_path, raw_content_size FROM documents WHERE doc_id = %s", + (doc_id,), + ) + token_original = 0 + if doc and doc.get("raw_content_path"): + raw_path = Path(doc["raw_content_path"]) + if raw_path.is_file(): + token_original = count_tokens(raw_path.read_text(errors="replace")) + + # Check for existing distilled_content row (refactor case) + existing = db.fetch_one( + "SELECT distill_id, review_status FROM distilled_content WHERE doc_id = %s", + (doc_id,), + ) + + if existing and existing.get("review_status") in ("needs_refactor", "pending"): + # Update existing row instead of creating a new one + distill_id = existing["distill_id"] + db.execute( + """UPDATE distilled_content + SET summary = %s, key_concepts = %s, code_snippets = %s, + structured_content = %s, token_count_original = %s, + token_count_distilled = %s, distill_model = %s, + distill_date = %s, review_status = %s + WHERE distill_id = %s""", + (summary_text, + json.dumps(concepts_json) if concepts_json else None, + json.dumps(snippets_json) if snippets_json else None, + structured, token_original, token_distilled, model, + datetime.now().isoformat(), "pending", distill_id), + ) + action = "Updated" + else: + # First distillation for this document + distill_id = db.insert_returning_id( + """INSERT INTO distilled_content + (doc_id, summary, key_concepts, code_snippets, structured_content, + token_count_original, token_count_distilled, distill_model, distill_date, + review_status) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", + (doc_id, summary_text, + json.dumps(concepts_json) if concepts_json else None, + json.dumps(snippets_json) if snippets_json else None, + structured, token_original, token_distilled, model, + datetime.now().isoformat(), "pending"), + ) + action = "Stored" + + ratio = (token_distilled / token_original * 100) if token_original else 0 + console.print( + f"[green]{action}:[/green] distill_id={distill_id} for doc_id={doc_id} " + f"({token_original} → {token_distilled} tokens, {ratio:.0f}% compression)" + ) + click.echo(json.dumps({ + "distill_id": distill_id, + "doc_id": doc_id, + "token_original": token_original, + "token_distilled": token_distilled, + "compression_ratio": round(ratio, 2), + })) + + +@cli.command() +@click.option("--distill-id", required=True, type=int, help="Distilled content ID") +@click.option("--output", type=click.Path(), help="Output context JSON for re-distillation") +def refactor(distill_id, output): + """Load existing distilled content + review feedback for re-distillation. + + Outputs a context bundle that Claude can use to re-distill with improvements. + """ + with db_session() as db: + distilled = db.fetch_one( + """SELECT dc.*, d.title, d.url, d.raw_content_path + FROM distilled_content dc + JOIN documents d ON dc.doc_id = d.doc_id + WHERE dc.distill_id = %s""", + (distill_id,), + ) + + if not distilled: + console.print(f"[red]Error:[/red] distill_id={distill_id} not found") + sys.exit(1) + + # Get review feedback + reviews = db.fetch_all( + """SELECT review_round, quality_score, decision, feedback, refactor_instructions + FROM review_logs + WHERE distill_id = %s + ORDER BY review_round ASC""", + (distill_id,), + ) + + # Load raw content if available + raw_content = "" + raw_path = distilled.get("raw_content_path") + if raw_path and Path(raw_path).is_file(): + raw_content = Path(raw_path).read_text(errors="replace") + + context = { + "distill_id": distill_id, + "doc_id": distilled.get("doc_id"), + "title": distilled.get("title"), + "url": distilled.get("url"), + "current_distilled": distilled.get("structured_content"), + "raw_content": raw_content, + "review_history": [ + { + "round": r.get("review_round"), + "score": float(r["quality_score"]) if r.get("quality_score") else None, + "decision": r.get("decision"), + "feedback": r.get("feedback"), + "instructions": r.get("refactor_instructions"), + } + for r in reviews + ], + } + + if output: + Path(output).write_text(json.dumps(context, indent=2, default=str)) + console.print(f"[green]Refactor context written to {output}[/green]") + else: + click.echo(json.dumps(context, indent=2, default=str)) + + +@cli.command() +@click.option("--distill-id", required=True, type=int, help="Distilled content ID") +def show(distill_id): + """Show details of a distilled content record.""" + with db_session() as db: + row = db.fetch_one( + """SELECT dc.*, d.title, d.url + FROM distilled_content dc + JOIN documents d ON dc.doc_id = d.doc_id + WHERE dc.distill_id = %s""", + (distill_id,), + ) + + if not row: + console.print(f"[red]Not found:[/red] distill_id={distill_id}") + sys.exit(1) + + console.print(f"\n[bold]Distilled Content #{distill_id}[/bold]") + console.print(f" Document: {row.get('title')} (doc_id={row.get('doc_id')})") + console.print(f" URL: {row.get('url')}") + console.print(f" Review Status: {row.get('review_status')}") + console.print(f" Model: {row.get('distill_model')}") + console.print(f" Tokens: {row.get('token_count_original')} → {row.get('token_count_distilled')}") + if row.get("summary"): + console.print(f"\n[bold]Summary:[/bold]\n{row['summary'][:500]}") + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/05-quality-reviewer/code/CLAUDE.md b/custom-skills/90-reference-curator/05-quality-reviewer/code/CLAUDE.md index ab09278..2ba3856 100644 --- a/custom-skills/90-reference-curator/05-quality-reviewer/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/05-quality-reviewer/code/CLAUDE.md @@ -1,103 +1,93 @@ # Quality Reviewer -QA loop for reference library content. Scores distilled materials, routes decisions, and provides actionable feedback. +Pre-distillation quality gate using Gemini CLI as an independent evaluator. Assesses raw crawled content before distillation to filter out low-quality sources early. Also supports manual scoring and routing for edge cases. ## Trigger Keywords -"review content", "quality check", "QA review", "assess distilled content", "check reference quality" +"review content", "quality check", "QA review", "evaluate sources", "check reference quality" -## Decision Flow +## Primary Flow: Gemini Pre-Distillation Gate ``` -[Distilled Content] +[Raw Crawled Content] │ ▼ -┌─────────────────┐ -│ Score Criteria │ → accuracy, completeness, clarity, PE quality, usability -└─────────────────┘ +┌────────────────────┐ +│ Gemini CLI Eval │ → relevance, authority, completeness, freshness, distill_value +└────────────────────┘ │ - ├── ≥ 0.85 → APPROVE → markdown-exporter - ├── 0.60-0.84 → REFACTOR → content-distiller - ├── 0.40-0.59 → DEEP_RESEARCH → web-crawler - └── < 0.40 → REJECT → archive + ├── ≥ 0.75 → APPROVE → proceed to distillation + ├── 0.50-0.74 → DEEP_RESEARCH → re-crawl for better sources + └── < 0.50 → REJECT → skip distillation entirely ``` -## Scoring Criteria +## Evaluation Criteria (Gemini) -| Criterion | Weight | Checks | -|-----------|--------|--------| -| **Accuracy** | 0.25 | Factual correctness, up-to-date, attribution | -| **Completeness** | 0.20 | Key concepts, examples, edge cases | -| **Clarity** | 0.20 | Structure, concise language, logical flow | -| **PE Quality** | 0.25 | Techniques, before/after, explains why | -| **Usability** | 0.10 | Easy reference, searchable, appropriate length | +| Criterion | Weight | What It Checks | +|-----------|--------|----------------| +| **Relevance** | 0.25 | Does content match the curation topic? | +| **Authority** | 0.25 | Official docs / research paper, or blog spam? | +| **Completeness** | 0.20 | Full article, or nav fragment / error page / stub? | +| **Freshness** | 0.15 | Up-to-date or outdated information? | +| **Distill Value** | 0.15 | Unique info worth summarizing, or redundant? | ## Workflow -### Step 1: Load Pending Reviews +### Step 1: Evaluate Single Document ```bash -python scripts/load_pending_reviews.py --output pending.json +uv run python scripts/reviewer.py gemini-evaluate --doc-id 123 --topic "prompt engineering" + +# With auto-logging of decision: +uv run python scripts/reviewer.py gemini-evaluate --doc-id 123 --topic "prompt engineering" --auto-approve ``` -### Step 2: Score Content +### Step 2: Batch Evaluate All Pending ```bash -python scripts/score_content.py --distill-id 123 --output assessment.json +uv run python scripts/reviewer.py gemini-evaluate-pending --topic "prompt engineering" --auto-approve --limit 20 ``` -### Step 3: Calculate Final Score +### Step 3: Manual Review (Edge Cases) +For documents where Gemini evaluation fails or needs human judgment: ```bash -python scripts/calculate_score.py --assessment assessment.json +# Calculate score from manual assessment +uv run python scripts/reviewer.py calculate-score --assessment assessment.json + +# Route based on score +uv run python scripts/reviewer.py route --score 0.78 + +# Log review decision +uv run python scripts/reviewer.py log-review \ + --distill-id 123 --decision approve --score 0.85 \ + --feedback "Manually verified" ``` -### Step 4: Route Decision +### Step 4: Review History ```bash -python scripts/route_decision.py --distill-id 123 --score 0.78 +uv run python scripts/reviewer.py history --distill-id 123 ``` -Outputs: -- `approve` → Ready for export -- `refactor` → Return to distiller with instructions -- `deep_research` → Need more sources (queries generated) -- `reject` → Archive with reason +## Prerequisites -### Step 5: Log Review -```bash -python scripts/log_review.py --distill-id 123 --decision refactor --instructions "Add more examples" -``` - -## PE Quality Checklist - -When scoring `prompt_engineering_quality`: -- [ ] Demonstrates specific techniques (CoT, few-shot, etc.) -- [ ] Shows before/after examples -- [ ] Explains *why* techniques work -- [ ] Provides actionable patterns -- [ ] Includes edge cases and failure modes -- [ ] References authoritative sources - -## Auto-Approve Rules - -Tier 1 sources with score ≥ 0.80 may auto-approve: -```yaml -# In config -quality: - auto_approve_tier1_sources: true - auto_approve_min_score: 0.80 -``` +- Gemini CLI: `npm install -g @google/gemini-cli` +- Google auth: `gemini` (run once interactively to authenticate) +- `refcurator` package installed ## Scripts -- `scripts/load_pending_reviews.py` - Get pending reviews -- `scripts/score_content.py` - Multi-criteria scoring -- `scripts/calculate_score.py` - Weighted average calculation -- `scripts/route_decision.py` - Decision routing logic -- `scripts/log_review.py` - Log review to database -- `scripts/generate_feedback.py` - Generate refactor instructions +| Command | Purpose | +|---------|---------| +| `reviewer.py gemini-evaluate` | Evaluate single doc via Gemini CLI | +| `reviewer.py gemini-evaluate-pending` | Batch evaluate all pending docs | +| `reviewer.py calculate-score` | Manual weighted score calculation | +| `reviewer.py route` | Decision routing from score | +| `reviewer.py log-review` | Log review decision to DB | +| `reviewer.py load-pending` | Get pending reviews | +| `reviewer.py history` | Show review history | ## Integration | From | Action | To | |------|--------|-----| -| content-distiller | Distilled content | → | -| → | APPROVE | markdown-exporter | -| → | REFACTOR + instructions | content-distiller | -| → | DEEP_RESEARCH + queries | web-crawler-orchestrator | +| content-repository (raw docs) | Gemini evaluation | → | +| → | APPROVE | content-distiller | +| → | DEEP_RESEARCH | web-crawler-orchestrator | +| → | REJECT | archive (skip distillation) | diff --git a/custom-skills/90-reference-curator/05-quality-reviewer/code/scripts/reviewer.py b/custom-skills/90-reference-curator/05-quality-reviewer/code/scripts/reviewer.py new file mode 100644 index 0000000..a2e7f0f --- /dev/null +++ b/custom-skills/90-reference-curator/05-quality-reviewer/code/scripts/reviewer.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Quality Reviewer CLI — scoring, routing, and review logging. + +Claude performs qualitative assessment. This script handles: +- Loading pending reviews +- Calculating weighted quality scores +- Routing decisions based on thresholds +- Logging review decisions to the database + +Usage: + python reviewer.py load-pending [--output pending.json] + python reviewer.py calculate-score --assessment assessment.json + python reviewer.py route --score 0.78 + python reviewer.py log-review --distill-id 123 --decision refactor --score 0.78 [--feedback "..."] + python reviewer.py history --distill-id 123 +""" + +import json +import sys +from datetime import datetime +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.models import QAAssessment + +console = Console() + +# Thresholds from pipeline config +APPROVE_THRESHOLD = 0.85 +REFACTOR_THRESHOLD = 0.60 +DEEP_RESEARCH_THRESHOLD = 0.40 + + +@click.group() +def cli(): + """Quality Reviewer — scoring, routing, and review logging.""" + pass + + +@cli.command("load-pending") +@click.option("--output", type=click.Path(), help="Output JSON file path") +@click.option("--limit", default=50, type=int, help="Max results") +def load_pending(output, limit): + """Load distilled content pending quality review.""" + with db_session() as db: + rows = db.fetch_all( + """SELECT dc.distill_id, d.doc_id, d.title, d.url, + dc.token_count_distilled, dc.distill_date, + dc.summary, s.credibility_tier, s.vendor + FROM distilled_content dc + JOIN documents d ON dc.doc_id = d.doc_id + JOIN sources s ON d.source_id = s.source_id + WHERE dc.review_status = %s + ORDER BY s.credibility_tier ASC, dc.distill_date ASC + LIMIT %s""", + ("pending", limit), + ) + + if output: + Path(output).write_text(json.dumps(rows, indent=2, default=str)) + console.print(f"[green]{len(rows)} pending reviews written to {output}[/green]") + else: + table = Table(title=f"Pending Reviews ({len(rows)})") + table.add_column("distill_id", style="cyan") + table.add_column("Title") + table.add_column("Tier") + table.add_column("Tokens", justify="right") + for r in rows: + table.add_row( + str(r.get("distill_id", "")), + str(r.get("title", ""))[:40], + str(r.get("credibility_tier", "")), + str(r.get("token_count_distilled", "")), + ) + console.print(table) + + click.echo(json.dumps({"count": len(rows)}, default=str)) + + +@cli.command("calculate-score") +@click.option("--assessment", required=True, type=click.Path(exists=True), + help="Assessment JSON file with criteria scores") +@click.option("--json-output", is_flag=True, help="Output as JSON only") +def calculate_score(assessment, json_output): + """Calculate weighted quality score from assessment criteria. + + Input JSON format: + { + "accuracy": 0.90, + "completeness": 0.85, + "clarity": 0.95, + "prompt_engineering_quality": 0.88, + "usability": 0.82 + } + + Weights: accuracy 0.25, completeness 0.20, clarity 0.20, + prompt_engineering_quality 0.25, usability 0.10 + """ + data = json.loads(Path(assessment).read_text()) + qa = QAAssessment(**data) + + result = { + "criteria": data, + "weighted_score": qa.weighted_score, + "decision": _route_score(qa.weighted_score), + } + + if json_output: + click.echo(json.dumps(result, indent=2)) + else: + console.print(f"\n[bold]Quality Assessment[/bold]") + for criterion, score in data.items(): + bar = "█" * int(score * 20) + "░" * (20 - int(score * 20)) + console.print(f" {criterion:<30} {bar} {score:.2f}") + console.print(f"\n [bold]Weighted Score:[/bold] {qa.weighted_score:.4f}") + console.print(f" [bold]Decision:[/bold] {_route_score(qa.weighted_score)}") + + +@cli.command() +@click.option("--score", required=True, type=float, help="Quality score to route") +@click.option("--tier", type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]), + help="Source credibility tier (for auto-approve)") +@click.option("--auto-approve", is_flag=True, help="Enable auto-approve for tier1 sources") +def route(score, tier, auto_approve): + """Route a quality score to a decision. + + Thresholds: + >= 0.85 → approve + 0.60-0.84 → refactor + 0.40-0.59 → deep_research + < 0.40 → reject + """ + decision = _route_score(score) + + # Auto-approve tier1 sources with lower threshold + if auto_approve and tier == "tier1_official" and score >= 0.80: + decision = "approve" + + result = {"score": score, "decision": decision} + + if decision == "approve": + console.print(f"[green]APPROVE[/green] (score: {score:.2f})") + elif decision == "refactor": + console.print(f"[yellow]REFACTOR[/yellow] (score: {score:.2f})") + elif decision == "deep_research": + console.print(f"[blue]DEEP_RESEARCH[/blue] (score: {score:.2f})") + else: + console.print(f"[red]REJECT[/red] (score: {score:.2f})") + + click.echo(json.dumps(result)) + + +@cli.command("log-review") +@click.option("--distill-id", required=True, type=int, help="Distilled content ID") +@click.option("--decision", required=True, + type=click.Choice(["approve", "refactor", "deep_research", "reject"])) +@click.option("--score", required=True, type=float, help="Quality score") +@click.option("--assessment", type=click.Path(exists=True), help="Assessment JSON file") +@click.option("--feedback", help="Review feedback text") +@click.option("--instructions", help="Refactor instructions") +@click.option("--queries", type=click.Path(exists=True), help="Research queries JSON (for deep_research)") +@click.option("--reviewer", default="claude_review", + type=click.Choice(["auto_qa", "human", "claude_review"])) +def log_review(distill_id, decision, score, assessment, feedback, instructions, queries, reviewer): + """Log a review decision for distilled content.""" + assessment_json = json.loads(Path(assessment).read_text()) if assessment else None + queries_json = json.loads(Path(queries).read_text()) if queries else None + + with db_session() as db: + # Get current review round + last_review = db.fetch_one( + "SELECT MAX(review_round) as max_round FROM review_logs WHERE distill_id = %s", + (distill_id,), + ) + review_round = (last_review.get("max_round") or 0) + 1 if last_review else 1 + + review_id = db.insert_returning_id( + """INSERT INTO review_logs + (distill_id, review_round, reviewer_type, quality_score, assessment, + decision, feedback, refactor_instructions, research_queries, reviewed_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", + (distill_id, review_round, reviewer, score, + json.dumps(assessment_json) if assessment_json else None, + decision, feedback, instructions, + json.dumps(queries_json) if queries_json else None, + datetime.now().isoformat()), + ) + + # Update distilled_content review_status + status_map = { + "approve": "approved", + "refactor": "needs_refactor", + "deep_research": "needs_refactor", + "reject": "rejected", + } + db.execute( + "UPDATE distilled_content SET review_status = %s WHERE distill_id = %s", + (status_map[decision], distill_id), + ) + + console.print( + f"[green]Logged:[/green] review_id={review_id}, round={review_round}, " + f"decision={decision}, score={score:.2f}" + ) + click.echo(json.dumps({ + "review_id": review_id, + "distill_id": distill_id, + "review_round": review_round, + "decision": decision, + "score": score, + })) + + +@cli.command() +@click.option("--distill-id", required=True, type=int, help="Distilled content ID") +def history(distill_id): + """Show review history for a distilled content record.""" + with db_session() as db: + reviews = db.fetch_all( + """SELECT review_id, review_round, reviewer_type, quality_score, + decision, feedback, refactor_instructions, reviewed_at + FROM review_logs + WHERE distill_id = %s + ORDER BY review_round ASC""", + (distill_id,), + ) + + if not reviews: + console.print(f"[dim]No reviews found for distill_id={distill_id}[/dim]") + return + + table = Table(title=f"Review History — distill_id={distill_id}") + table.add_column("Round", style="cyan") + table.add_column("Score", justify="right") + table.add_column("Decision") + table.add_column("Reviewer") + table.add_column("Feedback") + for r in reviews: + decision = str(r.get("decision", "")) + style = {"approve": "green", "refactor": "yellow", + "deep_research": "blue", "reject": "red"}.get(decision, "") + table.add_row( + str(r.get("review_round", "")), + f"{float(r['quality_score']):.2f}" if r.get("quality_score") else "", + f"[{style}]{decision}[/{style}]" if style else decision, + str(r.get("reviewer_type", "")), + str(r.get("feedback", ""))[:40] if r.get("feedback") else "", + ) + console.print(table) + + +@cli.command("gemini-evaluate") +@click.option("--doc-id", required=True, type=int, help="Document ID to evaluate") +@click.option("--topic", required=True, help="Curation topic for relevance scoring") +@click.option("--auto-approve", is_flag=True, help="Auto-log decision based on Gemini verdict") +def gemini_evaluate(doc_id, topic, auto_approve): + """Evaluate raw crawled content using Gemini CLI (pre-distillation gate). + + Sends raw content to Gemini for independent quality assessment. + Scores: relevance, authority, completeness, freshness, distill_value. + """ + from refcurator.gemini import evaluate_content, is_available + + if not is_available(): + console.print("[red]Error:[/red] Gemini CLI not available. Install: npm install -g @google/gemini-cli") + sys.exit(1) + + with db_session() as db: + doc = db.fetch_one( + "SELECT doc_id, title, url, raw_content_path FROM documents WHERE doc_id = %s", + (doc_id,), + ) + + if not doc: + console.print(f"[red]Error:[/red] doc_id={doc_id} not found") + sys.exit(1) + + raw_path = doc.get("raw_content_path") + if not raw_path or not Path(raw_path).is_file(): + console.print(f"[red]Error:[/red] Raw content file not found: {raw_path}") + sys.exit(1) + + content = Path(raw_path).read_text(errors="replace") + url = doc.get("url", "") + + console.print(f"[dim]Evaluating doc_id={doc_id}: {doc.get('title', '')}[/dim]") + console.print(f"[dim]Sending {len(content):,} chars to Gemini...[/dim]") + + result = evaluate_content(content, topic, url) + + if result is None: + console.print("[yellow]Warning:[/yellow] Gemini evaluation failed — manual review needed") + click.echo(json.dumps({"doc_id": doc_id, "status": "gemini_failed"})) + return + + # Display results + console.print(f"\n[bold]Gemini Evaluation — doc_id={doc_id}[/bold]") + for criterion in ["relevance", "authority", "completeness", "freshness", "distill_value"]: + score = result.get(criterion, 0) + bar = "█" * int(score * 20) + "░" * (20 - int(score * 20)) + console.print(f" {criterion:<15} {bar} {score:.2f}") + + ws = result.get("weighted_score", 0) + verdict = result.get("verdict", "unknown") + reason = result.get("reason", "") + + console.print(f"\n [bold]Weighted Score:[/bold] {ws:.4f}") + verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white") + console.print(f" [bold]Verdict:[/bold] [{verdict_color}]{verdict}[/{verdict_color}]") + if reason: + console.print(f" [dim]Reason:[/dim] {reason}") + + if auto_approve: + # Map verdict to decision (no 'refactor' for raw content) + decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject" + + with db_session() as db: + # Log to review_logs (using doc_id context, not distill_id since pre-distillation) + # We create a placeholder distill_id = 0 entry or log against the document directly + review_id = db.insert_returning_id( + """INSERT INTO review_logs + (distill_id, review_round, reviewer_type, quality_score, assessment, + decision, feedback, reviewed_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""", + (0, 1, "auto_qa", ws, + json.dumps(result), + decision, reason, + datetime.now().isoformat()), + ) + + console.print(f"\n[green]Auto-logged:[/green] review_id={review_id}, decision={decision}") + + click.echo(json.dumps({ + "doc_id": doc_id, + "evaluation": result, + "auto_logged": auto_approve, + }, default=str)) + + +@cli.command("gemini-evaluate-pending") +@click.option("--topic", required=True, help="Curation topic for relevance scoring") +@click.option("--auto-approve", is_flag=True, help="Auto-log decisions") +@click.option("--limit", default=20, type=int, help="Max documents to evaluate") +def gemini_evaluate_pending(topic, auto_approve, limit): + """Evaluate all crawled-but-not-evaluated documents using Gemini.""" + from refcurator.gemini import evaluate_content, is_available + + if not is_available(): + console.print("[red]Error:[/red] Gemini CLI not available") + sys.exit(1) + + with db_session() as db: + rows = db.fetch_all( + """SELECT doc_id, title, url, raw_content_path + FROM documents + WHERE crawl_status = %s + ORDER BY doc_id ASC + LIMIT %s""", + ("completed", limit), + ) + + if not rows: + console.print("[dim]No pending documents to evaluate.[/dim]") + return + + console.print(f"[bold]Evaluating {len(rows)} documents with Gemini...[/bold]\n") + + stats = {"approve": 0, "reject": 0, "deep_research": 0, "failed": 0} + + for row in rows: + doc_id = row["doc_id"] + raw_path = row.get("raw_content_path") + + if not raw_path or not Path(raw_path).is_file(): + console.print(f" [yellow]Skip[/yellow] doc_id={doc_id}: raw file not found") + stats["failed"] += 1 + continue + + content = Path(raw_path).read_text(errors="replace") + result = evaluate_content(content, topic, row.get("url", "")) + + if result is None: + console.print(f" [yellow]Failed[/yellow] doc_id={doc_id}: Gemini returned no result") + stats["failed"] += 1 + continue + + verdict = result.get("verdict", "reject") + ws = result.get("weighted_score", 0) + verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white") + console.print( + f" [{verdict_color}]{verdict:>13}[/{verdict_color}] " + f"({ws:.2f}) doc_id={doc_id} — {row.get('title', '')[:40]}" + ) + + stats[verdict] = stats.get(verdict, 0) + 1 + + if auto_approve: + decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject" + with db_session() as db: + db.insert_returning_id( + """INSERT INTO review_logs + (distill_id, review_round, reviewer_type, quality_score, assessment, + decision, feedback, reviewed_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""", + (0, 1, "auto_qa", ws, + json.dumps(result), + decision, result.get("reason", ""), + datetime.now().isoformat()), + ) + + console.print(f"\n[bold]Results:[/bold]") + console.print(f" [green]Approved:[/green] {stats['approve']}") + console.print(f" [red]Rejected:[/red] {stats['reject']}") + console.print(f" [blue]Deep Research:[/blue] {stats['deep_research']}") + console.print(f" [yellow]Failed:[/yellow] {stats['failed']}") + click.echo(json.dumps(stats)) + + +def _route_score(score: float) -> str: + """Route a quality score to a decision based on thresholds.""" + if score >= APPROVE_THRESHOLD: + return "approve" + elif score >= REFACTOR_THRESHOLD: + return "refactor" + elif score >= DEEP_RESEARCH_THRESHOLD: + return "deep_research" + else: + return "reject" + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/06-markdown-exporter/code/CLAUDE.md b/custom-skills/90-reference-curator/06-markdown-exporter/code/CLAUDE.md index 5526c03..df84618 100644 --- a/custom-skills/90-reference-curator/06-markdown-exporter/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/06-markdown-exporter/code/CLAUDE.md @@ -9,57 +9,51 @@ Exports approved reference content as structured markdown files for project know | Type | Format | Use Case | |------|--------|----------| -| `project_files` | Nested markdown | Claude Projects knowledge | -| `fine_tuning` | JSONL | Model fine-tuning dataset | -| `knowledge_base` | Flat markdown | Documentation | +| `project` | Nested markdown | Claude Projects knowledge | +| `finetuning` | JSONL | Model fine-tuning dataset | ## Workflow -### Step 1: Query Approved Content +### Step 1: Export Project Files ```bash -python scripts/query_approved.py --min-score 0.80 --output approved.json +uv run python scripts/exporter.py project \ + --output ~/reference-library/exports/ \ + --min-score 0.80 \ + --structure nested_by_topic ``` -### Step 2: Organize by Structure - -**Nested by Topic (default):** +Output structure: ``` exports/ ├── INDEX.md ├── prompt-engineering/ │ ├── _index.md -│ ├── 01-chain-of-thought.md -│ └── 02-few-shot-prompting.md +│ ├── 00-chain-of-thought.md +│ └── 01-few-shot-prompting.md └── claude-models/ ├── _index.md - └── 01-model-comparison.md + └── 00-model-comparison.md ``` -**Flat Structure:** -``` -exports/ -├── INDEX.md -├── prompt-engineering-chain-of-thought.md -└── claude-models-comparison.md -``` - -### Step 3: Generate Files +### Step 2: Generate INDEX ```bash -python scripts/export_project.py \ - --structure nested_by_topic \ - --output ~/reference-library/exports/ \ - --include-metadata +uv run python scripts/exporter.py index --output ~/reference-library/exports/INDEX.md ``` -### Step 4: Generate INDEX +### Step 3: Add Cross-References ```bash -python scripts/generate_index.py --output ~/reference-library/exports/INDEX.md +uv run python scripts/exporter.py crossrefs --input ~/reference-library/exports/ +``` + +### Step 4: Verify Export +```bash +uv run python scripts/exporter.py verify --path ~/reference-library/exports/ ``` ### Step 5: Fine-tuning Export (Optional) ```bash -python scripts/export_finetuning.py \ - --output ~/reference-library/exports/fine_tuning.jsonl \ +uv run python scripts/exporter.py finetuning \ + --output ~/reference-library/exports/training.jsonl \ --max-tokens 4096 ``` @@ -71,62 +65,25 @@ JSONL format: {"role": "user", "content": "Explain {title}"}, {"role": "assistant", "content": "{structured_content}"} ], - "metadata": {"source": "{url}", "topic": "{topic_slug}", "quality_score": 0.92} + "metadata": {"source": "{url}", "quality_score": 0.92} } ``` ### Step 6: Log Export Job ```bash -python scripts/log_export.py --name "January 2025 Export" --type project_files --docs 45 -``` - -## Cross-Reference Generation -```bash -python scripts/add_crossrefs.py --input ~/reference-library/exports/ -``` - -Links related documents based on overlapping key concepts. - -## Output Verification - -After export, verify: -- [ ] All files readable and valid markdown -- [ ] INDEX.md links resolve correctly -- [ ] No broken cross-references -- [ ] Total token count matches expectation -- [ ] No duplicate content - -```bash -python scripts/verify_export.py --path ~/reference-library/exports/ +uv run python scripts/exporter.py log --name "April 2026 Export" --type project_files --docs 45 ``` ## Scripts -- `scripts/query_approved.py` - Get approved content from DB -- `scripts/export_project.py` - Main export for project files -- `scripts/export_finetuning.py` - JSONL export for fine-tuning -- `scripts/generate_index.py` - Generate INDEX.md -- `scripts/add_crossrefs.py` - Add cross-references -- `scripts/log_export.py` - Log export job to DB -- `scripts/verify_export.py` - Verify export integrity - -## Configuration - -```yaml -# ~/.config/reference-curator/export_config.yaml -output: - base_path: ~/reference-library/exports/ - project_files: - structure: nested_by_topic - index_file: INDEX.md - include_metadata: true - fine_tuning: - format: jsonl - max_tokens_per_sample: 4096 - -quality: - min_score_for_export: 0.80 -``` +| Command | Purpose | +|---------|---------| +| `exporter.py project` | Export as nested markdown files | +| `exporter.py finetuning` | Export as JSONL training dataset | +| `exporter.py index` | Generate INDEX.md table of contents | +| `exporter.py crossrefs` | Add cross-reference links | +| `exporter.py verify` | Verify export integrity | +| `exporter.py log` | Log export job to DB | ## Integration diff --git a/custom-skills/90-reference-curator/06-markdown-exporter/code/scripts/exporter.py b/custom-skills/90-reference-curator/06-markdown-exporter/code/scripts/exporter.py new file mode 100644 index 0000000..3bc1399 --- /dev/null +++ b/custom-skills/90-reference-curator/06-markdown-exporter/code/scripts/exporter.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Markdown Exporter CLI — export approved content as structured files. + +Usage: + python exporter.py project --output ~/reference-library/exports/ [--min-score 0.80] + python exporter.py finetuning --output ~/reference-library/exports/training.jsonl + python exporter.py index --output ~/reference-library/exports/INDEX.md + python exporter.py crossrefs --input ~/reference-library/exports/ + python exporter.py verify --path ~/reference-library/exports/ + python exporter.py log --name "Jan 2025 Export" --type project_files --docs 40 +""" + +import json +import re +import sys +from collections import defaultdict +from datetime import datetime +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.utils import count_tokens, slugify + +console = Console() + + +@click.group() +def cli(): + """Markdown Exporter — export approved references.""" + pass + + +@cli.command() +@click.option("--output", required=True, type=click.Path(), help="Output directory path") +@click.option("--min-score", default=0.80, type=float, help="Minimum quality score") +@click.option("--structure", default="nested_by_topic", + type=click.Choice(["nested_by_topic", "flat"])) +@click.option("--include-metadata", is_flag=True, default=True, help="Include source metadata") +def project(output, min_score, structure, include_metadata): + """Export approved content as markdown project files.""" + out_dir = Path(output).expanduser() + out_dir.mkdir(parents=True, exist_ok=True) + + with db_session() as db: + rows = db.fetch_all( + """SELECT d.doc_id, d.title, d.url, + dc.structured_content, dc.summary, dc.key_concepts, + dc.token_count_distilled, + rl.quality_score, + s.credibility_tier, s.vendor, s.source_name + FROM documents d + JOIN distilled_content dc ON d.doc_id = dc.doc_id + JOIN review_logs rl ON dc.distill_id = rl.distill_id + JOIN sources s ON d.source_id = s.source_id + WHERE dc.review_status = %s + AND rl.decision = %s + AND rl.review_id = ( + SELECT MAX(rl2.review_id) + FROM review_logs rl2 + WHERE rl2.distill_id = dc.distill_id + ) + ORDER BY rl.quality_score DESC""", + ("approved", "approve"), + ) + + # Filter by min score + rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score] + + if not rows: + console.print("[yellow]No approved documents found above minimum score.[/yellow]") + return + + # Get topic mappings + with db_session() as db: + topic_rows = db.fetch_all( + """SELECT dt.doc_id, t.topic_name, t.topic_slug + FROM document_topics dt + JOIN topics t ON dt.topic_id = t.topic_id""" + ) + + doc_topics = defaultdict(list) + for tr in topic_rows: + doc_topics[tr["doc_id"]].append(tr) + + exported = 0 + total_tokens = 0 + + if structure == "nested_by_topic": + exported, total_tokens = _export_nested(rows, doc_topics, out_dir, include_metadata) + else: + exported, total_tokens = _export_flat(rows, doc_topics, out_dir, include_metadata) + + console.print( + f"[green]Exported {exported} documents ({total_tokens:,} tokens) to {out_dir}[/green]" + ) + + +@cli.command() +@click.option("--output", required=True, type=click.Path(), help="Output JSONL file path") +@click.option("--min-score", default=0.80, type=float, help="Minimum quality score") +@click.option("--max-tokens", default=4096, type=int, help="Max tokens per sample") +@click.option("--system-prompt", default="You are an expert on AI and prompt engineering.", + help="System prompt for training samples") +def finetuning(output, min_score, max_tokens, system_prompt): + """Export approved content as JSONL fine-tuning dataset.""" + out_path = Path(output).expanduser() + out_path.parent.mkdir(parents=True, exist_ok=True) + + with db_session() as db: + rows = db.fetch_all( + """SELECT d.doc_id, d.title, d.url, + dc.structured_content, dc.summary, + dc.token_count_distilled, + rl.quality_score + FROM documents d + JOIN distilled_content dc ON d.doc_id = dc.doc_id + JOIN review_logs rl ON dc.distill_id = rl.distill_id + WHERE dc.review_status = %s + AND rl.decision = %s + AND rl.review_id = ( + SELECT MAX(rl2.review_id) + FROM review_logs rl2 + WHERE rl2.distill_id = dc.distill_id + ) + ORDER BY rl.quality_score DESC""", + ("approved", "approve"), + ) + + rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score] + + count = 0 + with open(out_path, "w") as f: + for row in rows: + content = row.get("structured_content", "") + if count_tokens(content) > max_tokens: + content = content[:max_tokens * 4] # Approximate truncation + + sample = { + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": f"Explain {row.get('title', 'this topic')}"}, + {"role": "assistant", "content": content}, + ], + "metadata": { + "source": row.get("url"), + "quality_score": float(row["quality_score"]) if row.get("quality_score") else None, + "doc_id": row.get("doc_id"), + }, + } + f.write(json.dumps(sample, ensure_ascii=False) + "\n") + count += 1 + + console.print(f"[green]Exported {count} samples to {out_path}[/green]") + + +@cli.command() +@click.option("--output", required=True, type=click.Path(), help="Output INDEX.md path") +@click.option("--exports-dir", type=click.Path(exists=True), + help="Exports directory to scan (defaults to parent of output)") +def index(output, exports_dir): + """Generate INDEX.md with table of contents.""" + out_path = Path(output).expanduser() + scan_dir = Path(exports_dir).expanduser() if exports_dir else out_path.parent + + lines = [ + "# Reference Library Index", + "", + f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}*", + "", + ] + + # Scan for topic directories + topic_dirs = sorted([d for d in scan_dir.iterdir() if d.is_dir() and not d.name.startswith("_")]) + + if topic_dirs: + lines.append("## Topics\n") + for topic_dir in topic_dirs: + md_files = sorted(topic_dir.glob("*.md")) + if md_files: + topic_name = topic_dir.name.replace("-", " ").title() + lines.append(f"### {topic_name}\n") + for md_file in md_files: + if md_file.name.startswith("_"): + continue + title = _extract_md_title(md_file) + rel_path = md_file.relative_to(scan_dir) + lines.append(f"- [{title}]({rel_path})") + lines.append("") + + # Scan for flat files + flat_files = sorted([f for f in scan_dir.glob("*.md") + if f.name not in ("INDEX.md", "_index.md")]) + if flat_files: + lines.append("## Documents\n") + for md_file in flat_files: + title = _extract_md_title(md_file) + lines.append(f"- [{title}]({md_file.name})") + lines.append("") + + out_path.write_text("\n".join(lines)) + console.print(f"[green]Generated INDEX.md at {out_path}[/green]") + + +@cli.command() +@click.option("--input", "input_dir", required=True, type=click.Path(exists=True), + help="Exports directory to add cross-references to") +def crossrefs(input_dir): + """Add cross-reference links between related documents.""" + scan_dir = Path(input_dir).expanduser() + md_files = list(scan_dir.rglob("*.md")) + + if not md_files: + console.print("[yellow]No markdown files found.[/yellow]") + return + + # Build concept index: concept → list of (file, title) + concept_index: dict[str, list[tuple[Path, str]]] = defaultdict(list) + file_concepts: dict[Path, set[str]] = {} + + for md_file in md_files: + if md_file.name in ("INDEX.md", "_index.md"): + continue + content = md_file.read_text(errors="replace") + title = _extract_md_title(md_file) + concepts = _extract_concepts(content) + file_concepts[md_file] = concepts + for concept in concepts: + concept_index[concept].append((md_file, title)) + + # Add cross-references + modified = 0 + for md_file in md_files: + if md_file.name in ("INDEX.md", "_index.md"): + continue + my_concepts = file_concepts.get(md_file, set()) + related: dict[str, str] = {} # title → relative path + + for concept in my_concepts: + for other_file, other_title in concept_index.get(concept, []): + if other_file != md_file and other_title not in related: + try: + rel = other_file.relative_to(scan_dir) + except ValueError: + rel = other_file + related[other_title] = str(rel) + + if related and len(related) <= 10: + content = md_file.read_text(errors="replace") + # Remove existing Related section if present + content = re.sub(r"\n## Related\n.*$", "", content, flags=re.DOTALL) + content = content.rstrip() + "\n\n## Related\n\n" + for title, path in sorted(related.items())[:5]: + content += f"- [{title}]({path})\n" + md_file.write_text(content) + modified += 1 + + console.print(f"[green]Added cross-references to {modified} files[/green]") + + +@cli.command() +@click.option("--path", required=True, type=click.Path(exists=True), help="Exports directory to verify") +def verify(path): + """Verify export integrity.""" + scan_dir = Path(path).expanduser() + md_files = list(scan_dir.rglob("*.md")) + + issues = [] + total_tokens = 0 + total_files = len(md_files) + + for md_file in md_files: + content = md_file.read_text(errors="replace") + tokens = count_tokens(content) + total_tokens += tokens + + # Check for empty files + if len(content.strip()) < 10: + issues.append(f"Empty or near-empty: {md_file.relative_to(scan_dir)}") + + # Check for broken internal links + for match in re.finditer(r"\[([^\]]+)\]\(([^)]+)\)", content): + link_path = match.group(2) + if link_path.startswith("http"): + continue + resolved = (md_file.parent / link_path).resolve() + if not resolved.exists(): + issues.append( + f"Broken link in {md_file.relative_to(scan_dir)}: {link_path}" + ) + + # Check INDEX.md exists + if not (scan_dir / "INDEX.md").is_file(): + issues.append("Missing INDEX.md") + + # Report + console.print(f"\n[bold]Export Verification: {scan_dir}[/bold]") + console.print(f" Files: {total_files}") + console.print(f" Total tokens: {total_tokens:,}") + + if issues: + console.print(f"\n[red]Issues ({len(issues)}):[/red]") + for issue in issues[:20]: + console.print(f" - {issue}") + else: + console.print(f"\n[green]All checks passed.[/green]") + + click.echo(json.dumps({ + "files": total_files, + "total_tokens": total_tokens, + "issues": len(issues), + "issue_details": issues[:20], + })) + + +@cli.command() +@click.option("--name", required=True, help="Export job name") +@click.option("--type", "export_type", required=True, + type=click.Choice(["project_files", "fine_tuning", "knowledge_base"])) +@click.option("--docs", required=True, type=int, help="Number of documents exported") +@click.option("--path", type=click.Path(), help="Export output path") +@click.option("--tokens", type=int, help="Total tokens exported") +def log(name, export_type, docs, path, tokens): + """Log an export job to the database.""" + with db_session() as db: + export_id = db.insert_returning_id( + """INSERT INTO export_jobs + (export_name, export_type, output_path, total_documents, total_tokens, + status, started_at, completed_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""", + (name, export_type, path, docs, tokens, + "completed", datetime.now().isoformat(), datetime.now().isoformat()), + ) + + console.print(f"[green]Logged export:[/green] export_id={export_id} — {name}") + + +# --- Helpers --- + +def _export_nested(rows, doc_topics, out_dir, include_metadata): + """Export documents in nested topic directory structure.""" + exported = 0 + total_tokens = 0 + topic_docs = defaultdict(list) + + for row in rows: + topics = doc_topics.get(row["doc_id"], []) + if topics: + for t in topics: + topic_docs[t["topic_slug"]].append(row) + else: + topic_docs["uncategorized"].append(row) + + for topic_slug, docs in sorted(topic_docs.items()): + topic_dir = out_dir / topic_slug + topic_dir.mkdir(parents=True, exist_ok=True) + + # Topic index + topic_name = topic_slug.replace("-", " ").title() + index_lines = [f"# {topic_name}\n"] + + for i, doc in enumerate(docs): + filename = f"{i:02d}-{slugify(doc.get('title', 'untitled'))}.md" + content = _format_document(doc, include_metadata) + (topic_dir / filename).write_text(content) + index_lines.append(f"- [{doc.get('title', 'Untitled')}]({filename})") + exported += 1 + total_tokens += doc.get("token_count_distilled") or count_tokens(content) + + (topic_dir / "_index.md").write_text("\n".join(index_lines) + "\n") + + return exported, total_tokens + + +def _export_flat(rows, doc_topics, out_dir, include_metadata): + """Export documents as flat files.""" + exported = 0 + total_tokens = 0 + + for row in rows: + topics = doc_topics.get(row["doc_id"], []) + topic_prefix = topics[0]["topic_slug"] + "-" if topics else "" + filename = f"{topic_prefix}{slugify(row.get('title', 'untitled'))}.md" + content = _format_document(row, include_metadata) + (out_dir / filename).write_text(content) + exported += 1 + total_tokens += row.get("token_count_distilled") or count_tokens(content) + + return exported, total_tokens + + +def _format_document(row: dict, include_metadata: bool) -> str: + """Format a document for export.""" + lines = [] + + if include_metadata: + lines.extend([ + f"# {row.get('title', 'Untitled')}", + "", + f"**Source:** {row.get('url', 'N/A')}", + f"**Tier:** {row.get('credibility_tier', 'N/A')} | " + f"**Vendor:** {row.get('vendor', 'N/A')} | " + f"**Score:** {float(row['quality_score']):.2f}" if row.get("quality_score") else "", + "", + "---", + "", + ]) + + content = row.get("structured_content", "") + if content: + lines.append(content) + + return "\n".join(lines) + + +def _extract_md_title(md_file: Path) -> str: + """Extract title from a markdown file.""" + try: + for line in md_file.read_text(errors="replace").split("\n")[:10]: + if line.startswith("# ") and not line.startswith("##"): + return line[2:].strip() + except Exception: + pass + return md_file.stem.replace("-", " ").title() + + +def _extract_concepts(content: str) -> set[str]: + """Extract key concepts from markdown content for cross-referencing.""" + concepts = set() + + # Extract from ## Key Concepts section + m = re.search(r"## Key Concepts?\n(.*?)(?=\n##|\Z)", content, re.DOTALL) + if m: + for line in m.group(1).split("\n"): + line = line.strip().lstrip("- *") + if line and ":" in line: + concept = line.split(":")[0].strip("*").strip() + if 2 < len(concept) < 50: + concepts.add(concept.lower()) + + # Extract bold terms + for m in re.finditer(r"\*\*([^*]{3,40})\*\*", content): + concepts.add(m.group(1).lower()) + + return concepts + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md b/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md index c6ff995..6b498ce 100644 --- a/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md @@ -1,6 +1,6 @@ # Pipeline Orchestrator -Coordinates the full 6-skill reference curation workflow with QA loop handling. +Coordinates the full 6-skill reference curation workflow with QA loop handling. Manages pipeline state (init, advance, pause, resume, complete) while Claude orchestrates the actual stage execution. ## Trigger Keywords "curate references", "full pipeline", "run curation", "reference-curator-pipeline" @@ -22,257 +22,144 @@ Coordinates the full 6-skill reference curation workflow with QA loop handling. crawler ─┘ ``` -## Input Detection +## Pipeline State Management -Parse input to determine mode: - -```python -def detect_input_mode(input_value): - if input_value.endswith('.json') and os.path.exists(input_value): - return 'manifest' - elif input_value.startswith('http://') or input_value.startswith('https://'): - return 'urls' - else: - return 'topic' +### Initialize a Run +```bash +uv run python scripts/pipeline.py init \ + --input "prompt engineering" --type topic \ + --options '{"max_sources": 10, "auto_approve": true}' ``` -## Pipeline Execution +### Advance to Next Stage +```bash +uv run python scripts/pipeline.py advance \ + --run-id 1 --stage crawling \ + --stats '{"sources_discovered": 8}' +``` -### Stage 1: Reference Discovery (Topic Mode Only) +### Pause on Error +```bash +uv run python scripts/pipeline.py pause \ + --run-id 1 --error "Crawl timeout on page 45" --stage crawling +``` + +### Resume from Pause +```bash +uv run python scripts/pipeline.py resume --run-id 1 +``` + +### Complete Pipeline +```bash +uv run python scripts/pipeline.py complete \ + --run-id 1 --export-path ~/reference-library/exports/ --export-count 40 +``` + +### Check Status +```bash +uv run python scripts/pipeline.py status --run-id 1 +uv run python scripts/pipeline.py status --all +``` + +## QA Loop Tracking ```bash -# Skip if input mode is 'urls' or 'manifest' -if mode == 'topic': - /reference-discovery "$TOPIC" --max-sources $MAX_SOURCES - # Output: manifest.json +# Track a refactor iteration for a document +uv run python scripts/pipeline.py track-iteration \ + --run-id 1 --doc-id 42 --action refactor + +# Track a deep research iteration +uv run python scripts/pipeline.py track-iteration \ + --run-id 1 --doc-id 42 --action deep_research +``` + +Returns one of: +- `re_distill` — proceed with refactor +- `re_crawl_and_distill` — proceed with deep research +- `needs_manual_review` — max iterations exceeded + +| Decision | Max Iterations | +|----------|----------------| +| REFACTOR | 3 | +| DEEP_RESEARCH | 2 | +| Combined total | 5 | + +## Pipeline Execution Flow + +### Stage 1: Reference Discovery (Topic Mode Only) +``` +If mode == 'topic': + → Claude runs WebSearch + → discover.py create-manifest + → discover.py dedup + → pipeline.py advance --stage crawling ``` ### Stage 2: Web Crawler - -```bash -# From manifest or URLs -/web-crawler $INPUT --max-pages $MAX_PAGES -# Output: crawled files in ~/reference-library/raw/ +``` +→ Claude uses Firecrawl MCP tools +→ crawl_mgr.py store-result +→ pipeline.py advance --stage storing ``` ### Stage 3: Content Repository - -```bash -/content-repository store -# Output: documents stored in MySQL or file-based storage +``` +→ repo.py store (for each crawled doc) +→ pipeline.py advance --stage evaluating ``` -### Stage 4: Content Distiller - -```bash -/content-distiller all-pending -# Output: distilled content records +### Stage 4: Gemini Quality Gate (Pre-Distillation) +``` +→ reviewer.py gemini-evaluate-pending --topic "$TOPIC" --auto-approve +→ APPROVE: proceed to distillation +→ DEEP_RESEARCH: pipeline.py track-iteration → crawler (re-crawl) +→ REJECT: skip document entirely +→ pipeline.py advance --stage distilling ``` -### Stage 5: Quality Reviewer - -```bash -if auto_approve: - /quality-reviewer all-pending --auto-approve --threshold $THRESHOLD -else: - /quality-reviewer all-pending +### Stage 5: Content Distiller (Approved Only) +``` +→ distiller.py load-pending +→ Claude distills each approved document +→ distiller.py store +→ pipeline.py advance --stage exporting ``` - -Handle QA decisions: -- **APPROVE**: Add to export queue -- **REFACTOR**: Re-run distiller with feedback (track iteration count) -- **DEEP_RESEARCH**: Run crawler for additional sources, then distill -- **REJECT**: Archive with reason ### Stage 6: Markdown Exporter - -```bash -/markdown-exporter $EXPORT_FORMAT -# Output: files in ~/reference-library/exports/ ``` - -## State Management - -### Initialize Pipeline State - -```python -def init_pipeline_state(run_id, input_value, options): - state = { - "run_id": run_id, - "run_type": detect_input_mode(input_value), - "input_value": input_value, - "status": "running", - "current_stage": "discovery", - "options": options, - "stats": { - "sources_discovered": 0, - "pages_crawled": 0, - "documents_stored": 0, - "documents_distilled": 0, - "approved": 0, - "refactored": 0, - "deep_researched": 0, - "rejected": 0, - "needs_manual_review": 0 - }, - "started_at": datetime.now().isoformat() - } - save_state(run_id, state) - return state -``` - -### MySQL State (Preferred) - -```sql -INSERT INTO pipeline_runs (run_type, input_value, options) -VALUES ('topic', 'Claude system prompts', '{"max_sources": 10}'); -``` - -### File-Based Fallback - -``` -~/reference-library/pipeline_state/run_XXX/ -├── state.json # Current stage and stats -├── manifest.json # Discovered sources -├── crawl_results.json # Crawled document paths -├── review_log.json # QA decisions per document -└── errors.log # Any errors encountered -``` - -## QA Loop Logic - -```python -MAX_REFACTOR_ITERATIONS = 3 -MAX_DEEP_RESEARCH_ITERATIONS = 2 -MAX_TOTAL_ITERATIONS = 5 - -def handle_qa_decision(doc_id, decision, iteration_counts): - refactor_count = iteration_counts.get('refactor', 0) - research_count = iteration_counts.get('deep_research', 0) - total = refactor_count + research_count - - if total >= MAX_TOTAL_ITERATIONS: - return 'needs_manual_review' - - if decision == 'refactor': - if refactor_count >= MAX_REFACTOR_ITERATIONS: - return 'needs_manual_review' - iteration_counts['refactor'] = refactor_count + 1 - return 're_distill' - - if decision == 'deep_research': - if research_count >= MAX_DEEP_RESEARCH_ITERATIONS: - return 'needs_manual_review' - iteration_counts['deep_research'] = research_count + 1 - return 're_crawl_and_distill' - - return decision # approve or reject +→ exporter.py project +→ exporter.py index +→ exporter.py crossrefs +→ exporter.py verify +→ pipeline.py complete ``` ## Checkpoint Strategy -Save checkpoint after each stage completes: - | Stage | Checkpoint | Resume Point | |-------|------------|--------------| -| discovery | `manifest.json` created | → crawler | -| crawl | `crawl_results.json` | → repository | -| store | DB records or file list | → distiller | +| discovery | manifest.json created | → crawler | +| crawl | crawl_result.json | → repository | +| store | DB records | → distiller | | distill | distilled_content records | → reviewer | | review | review_logs records | → exporter or loop | | export | final export complete | Done | -## Progress Reporting +## Scripts -Report progress to user at key checkpoints: - -``` -[Pipeline] Stage 1/6: Discovery - Found 8 sources -[Pipeline] Stage 2/6: Crawling - 45/50 pages complete -[Pipeline] Stage 3/6: Storing - 45 documents saved -[Pipeline] Stage 4/6: Distilling - 45 documents processed -[Pipeline] Stage 5/6: Reviewing - 40 approved, 3 refactored, 2 rejected -[Pipeline] Stage 6/6: Exporting - 40 documents exported -[Pipeline] Complete! See ~/reference-library/exports/ -``` - -## Error Handling - -```python -def handle_stage_error(stage, error, state): - state['status'] = 'paused' - state['error_message'] = str(error) - state['error_stage'] = stage - save_state(state['run_id'], state) - - # Log to errors.log - log_error(state['run_id'], stage, error) - - # Report to user - return f"Pipeline paused at {stage}: {error}. Resume with run_id {state['run_id']}" -``` - -## Resume Pipeline - -```python -def resume_pipeline(run_id): - state = load_state(run_id) - - if state['status'] != 'paused': - return f"Pipeline {run_id} is {state['status']}, cannot resume" - - stage = state['current_stage'] - state['status'] = 'running' - state['error_message'] = None - save_state(run_id, state) - - # Resume from failed stage - return execute_from_stage(stage, state) -``` - -## Output Summary - -On completion, generate summary: - -```json -{ - "run_id": 123, - "status": "completed", - "duration_minutes": 15, - "stats": { - "sources_discovered": 5, - "pages_crawled": 45, - "documents_stored": 45, - "documents_distilled": 45, - "approved": 40, - "refactored": 8, - "deep_researched": 2, - "rejected": 3, - "needs_manual_review": 2 - }, - "exports": { - "format": "project_files", - "path": "~/reference-library/exports/", - "document_count": 40 - }, - "errors": [] -} -``` - -## Integration Points - -| Skill | Called By | Provides | -|-------|-----------|----------| -| reference-discovery | Orchestrator | manifest.json | -| web-crawler | Orchestrator | Raw crawled files | -| content-repository | Orchestrator | Stored documents | -| content-distiller | Orchestrator, QA loop | Distilled content | -| quality-reviewer | Orchestrator | QA decisions | -| markdown-exporter | Orchestrator | Final exports | +| Command | Purpose | +|---------|---------| +| `pipeline.py init` | Initialize a new pipeline run | +| `pipeline.py advance` | Advance to next stage | +| `pipeline.py pause` | Pause on error | +| `pipeline.py resume` | Resume from pause | +| `pipeline.py complete` | Mark pipeline complete | +| `pipeline.py status` | Show run status | +| `pipeline.py track-iteration` | Track QA loop iterations | ## Configuration -Read from `~/.config/reference-curator/pipeline_config.yaml`: +Reads from `~/.config/reference-curator/pipeline_config.yaml`: ```yaml pipeline: @@ -288,9 +175,4 @@ qa_loop: export: default_format: project_files - include_rejected: false - -state: - backend: mysql # or 'file' - state_directory: ~/reference-library/pipeline_state/ ``` diff --git a/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/scripts/pipeline.py b/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/scripts/pipeline.py new file mode 100644 index 0000000..7c7da70 --- /dev/null +++ b/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/scripts/pipeline.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Pipeline Orchestrator CLI — manage pipeline runs and state. + +This script provides state management for the 6-stage pipeline. +Claude orchestrates the actual stages via slash commands. + +Usage: + python pipeline.py init --input "prompt engineering" --type topic [--options '{"max_sources": 10}'] + python pipeline.py advance --run-id 1 --stage crawling [--stats '{"pages_crawled": 45}'] + python pipeline.py pause --run-id 1 --error "Crawl timeout" --stage crawling + python pipeline.py resume --run-id 1 + python pipeline.py complete --run-id 1 [--export-path ~/reference-library/exports/] + python pipeline.py status [--run-id 1] + python pipeline.py track-iteration --run-id 1 --doc-id 42 --action refactor +""" + +import json +import sys +from datetime import datetime +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table + +from refcurator.db import db_session +from refcurator.config import get_pipeline_config + +console = Console() + +# QA loop limits +MAX_REFACTOR = 3 +MAX_DEEP_RESEARCH = 2 +MAX_TOTAL = 5 + + +@click.group() +def cli(): + """Pipeline Orchestrator — manage pipeline run state.""" + pass + + +@cli.command() +@click.option("--input", "input_value", required=True, help="Topic, URL(s), or manifest path") +@click.option("--type", "run_type", required=True, + type=click.Choice(["topic", "urls", "manifest"])) +@click.option("--options", type=str, help="Pipeline options as JSON string") +def init(input_value, run_type, options): + """Initialize a new pipeline run.""" + opts = json.loads(options) if options else {} + + # Merge with config defaults + try: + cfg = get_pipeline_config() + defaults = cfg.get("pipeline", {}) + for key in ("max_sources", "max_pages", "auto_approve", "approval_threshold"): + if key not in opts and key in defaults: + opts[key] = defaults[key] + except FileNotFoundError: + pass + + stats = { + "sources_discovered": 0, + "pages_crawled": 0, + "documents_stored": 0, + "documents_distilled": 0, + "approved": 0, + "refactored": 0, + "deep_researched": 0, + "rejected": 0, + "needs_manual_review": 0, + } + + # Determine starting stage + start_stage = "discovery" if run_type == "topic" else "crawling" + + with db_session() as db: + run_id = db.insert_returning_id( + """INSERT INTO pipeline_runs + (run_type, input_value, status, current_stage, options, stats, started_at) + VALUES (%s, %s, %s, %s, %s, %s, %s)""", + (run_type, input_value, "running", start_stage, + json.dumps(opts), json.dumps(stats), datetime.now().isoformat()), + ) + + console.print(f"[green]Pipeline initialized:[/green] run_id={run_id}") + console.print(f" Type: {run_type}") + console.print(f" Input: {input_value}") + console.print(f" Starting stage: {start_stage}") + click.echo(json.dumps({"run_id": run_id, "status": "running", "stage": start_stage})) + + +@cli.command() +@click.option("--run-id", required=True, type=int, help="Pipeline run ID") +@click.option("--stage", required=True, + type=click.Choice(["discovery", "crawling", "storing", "evaluating", + "distilling", "exporting"])) +@click.option("--stats", type=str, help="Stats update as JSON string (merged with existing)") +def advance(run_id, stage, stats): + """Advance pipeline to the next stage.""" + stats_update = json.loads(stats) if stats else {} + + with db_session() as db: + run = db.fetch_one( + "SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,) + ) + if not run: + console.print(f"[red]Error:[/red] run_id={run_id} not found") + sys.exit(1) + + if run["status"] != "running": + console.print(f"[red]Error:[/red] Pipeline is {run['status']}, not running") + sys.exit(1) + + # Merge stats + current_stats = json.loads(run["stats"]) if isinstance(run["stats"], str) else (run["stats"] or {}) + current_stats.update(stats_update) + + db.execute( + """UPDATE pipeline_runs + SET current_stage = %s, stats = %s + WHERE run_id = %s""", + (stage, json.dumps(current_stats), run_id), + ) + + stage_num = ["discovery", "crawling", "storing", "evaluating", "distilling", "exporting"].index(stage) + 1 + console.print(f"[green]Pipeline advanced:[/green] Stage {stage_num}/6 — {stage}") + click.echo(json.dumps({"run_id": run_id, "stage": stage, "stats": current_stats})) + + +@cli.command() +@click.option("--run-id", required=True, type=int, help="Pipeline run ID") +@click.option("--error", required=True, help="Error message") +@click.option("--stage", required=True, help="Stage where error occurred") +def pause(run_id, error, stage): + """Pause pipeline due to error.""" + with db_session() as db: + db.execute( + """UPDATE pipeline_runs + SET status = %s, error_message = %s, error_stage = %s + WHERE run_id = %s""", + ("paused", error, stage, run_id), + ) + + console.print(f"[yellow]Pipeline paused:[/yellow] run_id={run_id} at {stage}") + console.print(f" Error: {error}") + console.print(f" Resume with: python pipeline.py resume --run-id {run_id}") + + +@cli.command() +@click.option("--run-id", required=True, type=int, help="Pipeline run ID to resume") +def resume(run_id): + """Resume a paused pipeline.""" + with db_session() as db: + run = db.fetch_one( + "SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,) + ) + if not run: + console.print(f"[red]Error:[/red] run_id={run_id} not found") + sys.exit(1) + + if run["status"] != "paused": + console.print(f"[red]Error:[/red] Pipeline is {run['status']}, not paused") + sys.exit(1) + + db.execute( + """UPDATE pipeline_runs + SET status = %s, error_message = %s + WHERE run_id = %s""", + ("running", None, run_id), + ) + + stage = run["current_stage"] + console.print(f"[green]Pipeline resumed:[/green] run_id={run_id}, continuing from {stage}") + click.echo(json.dumps({"run_id": run_id, "status": "running", "resume_stage": stage})) + + +@cli.command() +@click.option("--run-id", required=True, type=int, help="Pipeline run ID") +@click.option("--export-path", type=click.Path(), help="Export output path") +@click.option("--export-count", type=int, help="Number of documents exported") +def complete(run_id, export_path, export_count): + """Mark pipeline as completed.""" + with db_session() as db: + db.execute( + """UPDATE pipeline_runs + SET status = %s, completed_at = %s, export_path = %s, export_document_count = %s + WHERE run_id = %s""", + ("completed", datetime.now().isoformat(), export_path, export_count, run_id), + ) + + run = db.fetch_one("SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,)) + + stats = json.loads(run["stats"]) if isinstance(run["stats"], str) else (run["stats"] or {}) + + console.print(f"\n[bold green]Pipeline Complete![/bold green] run_id={run_id}") + console.print(f" Sources: {stats.get('sources_discovered', 0)}") + console.print(f" Crawled: {stats.get('pages_crawled', 0)}") + console.print(f" Stored: {stats.get('documents_stored', 0)}") + console.print(f" Approved: {stats.get('approved', 0)}") + console.print(f" Rejected: {stats.get('rejected', 0)}") + if export_path: + console.print(f" Exports: {export_path}") + + click.echo(json.dumps({"run_id": run_id, "status": "completed", "stats": stats}, default=str)) + + +@cli.command() +@click.option("--run-id", type=int, help="Show specific run (or latest if omitted)") +@click.option("--all", "show_all", is_flag=True, help="Show all runs") +def status(run_id, show_all): + """Show pipeline run status.""" + with db_session() as db: + if run_id: + rows = db.fetch_all( + "SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,) + ) + elif show_all: + rows = db.fetch_all( + "SELECT * FROM pipeline_runs ORDER BY started_at DESC LIMIT %s", (20,) + ) + else: + rows = db.fetch_all( + "SELECT * FROM pipeline_runs ORDER BY started_at DESC LIMIT %s", (1,) + ) + + if not rows: + console.print("[dim]No pipeline runs found.[/dim]") + return + + for run in rows: + stats = json.loads(run["stats"]) if isinstance(run["stats"], str) else (run["stats"] or {}) + status_color = { + "running": "green", "completed": "blue", + "failed": "red", "paused": "yellow", + }.get(run["status"], "white") + + console.print(f"\n[bold]Pipeline Run #{run['run_id']}[/bold]") + console.print(f" Status: [{status_color}]{run['status']}[/{status_color}]") + console.print(f" Type: {run['run_type']}") + console.print(f" Input: {run['input_value']}") + console.print(f" Stage: {run['current_stage']}") + console.print(f" Started: {run['started_at']}") + + if run.get("error_message"): + console.print(f" [red]Error:[/red] {run['error_message']} (at {run.get('error_stage')})") + + if stats: + table = Table(title="Pipeline Stats", show_header=False) + table.add_column("Metric") + table.add_column("Value", justify="right") + for k, v in stats.items(): + if v: + table.add_row(k.replace("_", " ").title(), str(v)) + console.print(table) + + +@cli.command("track-iteration") +@click.option("--run-id", required=True, type=int, help="Pipeline run ID") +@click.option("--doc-id", required=True, type=int, help="Document ID") +@click.option("--action", required=True, type=click.Choice(["refactor", "deep_research"]), + help="QA loop action") +def track_iteration(run_id, doc_id, action): + """Track QA loop iteration for a document. + + Returns the routing decision: + - 're_distill': proceed with refactor + - 're_crawl_and_distill': proceed with deep research + - 'needs_manual_review': max iterations exceeded + """ + with db_session() as db: + tracker = db.fetch_one( + """SELECT * FROM pipeline_iteration_tracker + WHERE run_id = %s AND doc_id = %s""", + (run_id, doc_id), + ) + + if tracker: + refactor_count = tracker.get("refactor_count", 0) + deep_research_count = tracker.get("deep_research_count", 0) + else: + refactor_count = 0 + deep_research_count = 0 + + total = refactor_count + deep_research_count + + # Check limits + if total >= MAX_TOTAL: + result = "needs_manual_review" + elif action == "refactor" and refactor_count >= MAX_REFACTOR: + result = "needs_manual_review" + elif action == "deep_research" and deep_research_count >= MAX_DEEP_RESEARCH: + result = "needs_manual_review" + elif action == "refactor": + refactor_count += 1 + result = "re_distill" + else: + deep_research_count += 1 + result = "re_crawl_and_distill" + + # Upsert tracker + if tracker: + db.execute( + """UPDATE pipeline_iteration_tracker + SET refactor_count = %s, deep_research_count = %s, + final_decision = %s + WHERE run_id = %s AND doc_id = %s""", + (refactor_count, deep_research_count, + "needs_manual_review" if result == "needs_manual_review" else None, + run_id, doc_id), + ) + else: + db.insert_returning_id( + """INSERT INTO pipeline_iteration_tracker + (run_id, doc_id, refactor_count, deep_research_count, final_decision) + VALUES (%s, %s, %s, %s, %s)""", + (run_id, doc_id, refactor_count, deep_research_count, + "needs_manual_review" if result == "needs_manual_review" else None), + ) + + console.print( + f"[{'yellow' if result == 'needs_manual_review' else 'green'}]" + f"Iteration tracked:[/] doc_id={doc_id}, {action} " + f"(refactor: {refactor_count}/{MAX_REFACTOR}, " + f"research: {deep_research_count}/{MAX_DEEP_RESEARCH}) → {result}" + ) + click.echo(json.dumps({ + "run_id": run_id, + "doc_id": doc_id, + "action": action, + "result": result, + "refactor_count": refactor_count, + "deep_research_count": deep_research_count, + "total_iterations": refactor_count + deep_research_count, + })) + + +if __name__ == "__main__": + cli() diff --git a/custom-skills/90-reference-curator/install.sh b/custom-skills/90-reference-curator/install.sh index 632231e..ae8fb4f 100755 --- a/custom-skills/90-reference-curator/install.sh +++ b/custom-skills/90-reference-curator/install.sh @@ -51,6 +51,7 @@ COMMANDS=( "content-distiller" "quality-reviewer" "markdown-exporter" + "reference-curator-pipeline" ) # ============================================================================ @@ -541,6 +542,16 @@ check_status() { print_info "MySQL not configured or client not installed" fi + # Check Python package + print_substep "Python Package (refcurator)" + if uv run python3 -c "import refcurator; print(refcurator.__version__)" &>/dev/null; then + local pkg_version=$(uv run python3 -c "import refcurator; print(refcurator.__version__)" 2>/dev/null) + print_success "refcurator $pkg_version installed" + else + print_warning "refcurator package not installed — scripts will not work" + all_ok=false + fi + # Check Claude Code commands print_substep "Claude Code Commands ($CLAUDE_COMMANDS_DIR)" for cmd in "${COMMANDS[@]}"; do @@ -670,11 +681,43 @@ install() { setup_database fi + install_python_package register_commands register_skills post_install } +# ============================================================================ +# Install Python Package +# ============================================================================ + +install_python_package() { + print_step "Step 5b: Installing Python Package (refcurator)" + + print_substep "Checking for uv package manager" + if command -v uv &> /dev/null; then + print_success "uv found: $(uv --version 2>/dev/null)" + else + print_warning "uv not found — install with: curl -LsSf https://astral.sh/uv/install.sh | sh" + print_info "Skipping Python package installation" + return 0 + fi + + print_substep "Installing refcurator package" + local pkg_dir="$SCRIPT_DIR/shared/lib" + + if [[ -f "$pkg_dir/pyproject.toml" ]]; then + if uv pip install -e "$pkg_dir" 2>/dev/null; then + print_success "refcurator package installed (editable mode)" + else + print_warning "Failed to install refcurator — scripts may not work" + print_info "Try manually: uv pip install -e $pkg_dir" + fi + else + print_error "pyproject.toml not found at $pkg_dir" + fi +} + # ============================================================================ # Minimal Installation (Firecrawl only) # ============================================================================ @@ -712,6 +755,7 @@ EOF install_configs create_directories + install_python_package register_commands register_skills post_install diff --git a/custom-skills/90-reference-curator/shared/lib/pyproject.toml b/custom-skills/90-reference-curator/shared/lib/pyproject.toml new file mode 100644 index 0000000..09c314f --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "refcurator" +version = "1.0.0" +description = "Reference Curator — data management for the reference curation pipeline" +requires-python = ">=3.12" +license = "MIT" +authors = [ + { name = "Andrew Yim", email = "andrew@ourdigital.org" }, +] +dependencies = [ + "pymysql>=1.1.0", + "click>=8.0", + "pydantic>=2.0", + "pyyaml>=6.0", + "rich>=13.0", + "python-dotenv>=1.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "ruff>=0.4", +] + +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 100 +target-version = "py312" diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/__init__.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/__init__.py new file mode 100644 index 0000000..ddd5c07 --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/__init__.py @@ -0,0 +1,3 @@ +"""Reference Curator — data management for the reference curation pipeline.""" + +__version__ = "1.0.0" diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/config.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/config.py new file mode 100644 index 0000000..a2ea430 --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/config.py @@ -0,0 +1,133 @@ +"""Configuration loading for the reference curator pipeline. + +Loads YAML configs from ~/.config/reference-curator/ with env var substitution. +Falls back to bundled defaults in shared/config/. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +import yaml +from dotenv import load_dotenv + +# Load user env if present +_env_file = Path.home() / ".reference-curator.env" +if _env_file.is_file(): + load_dotenv(_env_file) + +# Config search paths (user override → bundled defaults) +USER_CONFIG_DIR = Path.home() / ".config" / "reference-curator" +BUNDLED_CONFIG_DIR = Path(__file__).resolve().parents[4] / "config" # shared/config/ + +# Default storage paths +DEFAULT_LIBRARY_PATH = Path( + os.environ.get("REFERENCE_LIBRARY_PATH", "~/Documents/reference-library") +).expanduser() +DEFAULT_STATE_DIR = DEFAULT_LIBRARY_PATH / "pipeline_state" + + +def _expand_env_vars(value: str) -> str: + """Expand ${VAR:-default} patterns in a string.""" + def _replace(match: re.Match) -> str: + var_expr = match.group(1) + if ":-" in var_expr: + var_name, default = var_expr.split(":-", 1) + return os.environ.get(var_name, default) + return os.environ.get(var_expr, match.group(0)) + + return re.sub(r"\$\{([^}]+)}", _replace, value) + + +def _expand_recursive(obj: Any) -> Any: + """Recursively expand env vars in a parsed YAML structure.""" + if isinstance(obj, str): + expanded = _expand_env_vars(obj) + # Expand ~ in path-like strings + if expanded.startswith("~"): + expanded = str(Path(expanded).expanduser()) + return expanded + if isinstance(obj, dict): + return {k: _expand_recursive(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_expand_recursive(item) for item in obj] + return obj + + +def load_config(name: str) -> dict: + """Load a YAML config file by name (without extension). + + Searches user config dir first, then bundled defaults. + Expands ${VAR:-default} env var patterns in all string values. + + Args: + name: Config file name without .yaml extension + (e.g., "db_config", "pipeline_config", "crawl_config", "export_config") + + Returns: + Parsed and expanded config dict. + + Raises: + FileNotFoundError: If config file not found in any search path. + """ + filename = f"{name}.yaml" + + for config_dir in [USER_CONFIG_DIR, BUNDLED_CONFIG_DIR]: + config_path = config_dir / filename + if config_path.is_file(): + with open(config_path) as f: + raw = yaml.safe_load(f) or {} + return _expand_recursive(raw) + + raise FileNotFoundError( + f"Config '{filename}' not found in {USER_CONFIG_DIR} or {BUNDLED_CONFIG_DIR}" + ) + + +def get_db_config() -> dict: + """Load database configuration.""" + return load_config("db_config") + + +def get_pipeline_config() -> dict: + """Load pipeline orchestrator configuration.""" + return load_config("pipeline_config") + + +def get_crawl_config() -> dict: + """Load crawler configuration.""" + return load_config("crawl_config") + + +def get_export_config() -> dict: + """Load export configuration.""" + return load_config("export_config") + + +def get_library_path() -> Path: + """Get the reference library base path.""" + return DEFAULT_LIBRARY_PATH + + +def get_state_dir() -> Path: + """Get the pipeline state directory.""" + try: + cfg = get_pipeline_config() + state_dir = cfg.get("state", {}).get("state_directory") + if state_dir: + return Path(state_dir).expanduser() + except FileNotFoundError: + pass + return DEFAULT_STATE_DIR + + +def get_state_backend() -> str: + """Get the state backend type: 'mysql' or 'file'.""" + try: + cfg = get_pipeline_config() + return cfg.get("state", {}).get("backend", "file") + except FileNotFoundError: + return "file" diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/db.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/db.py new file mode 100644 index 0000000..22072f2 --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/db.py @@ -0,0 +1,389 @@ +"""Database abstraction layer with MySQL and file-based backends. + +Usage: + from refcurator.db import get_backend + + db = get_backend() + rows = db.fetch_all("SELECT * FROM documents WHERE crawl_status = %s", ("pending",)) + doc_id = db.insert_returning_id("INSERT INTO documents (...) VALUES (...)", params) +""" + +from __future__ import annotations + +import json +import logging +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Optional, Protocol, Sequence + +from refcurator.config import get_db_config, get_state_backend, get_state_dir + +logger = logging.getLogger("refcurator.db") + + +class DatabaseBackend(Protocol): + """Protocol for database backends.""" + + def execute(self, sql: str, params: Sequence = ()) -> int: + """Execute a statement, return affected row count.""" + ... + + def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]: + """Fetch a single row as dict.""" + ... + + def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]: + """Fetch all rows as list of dicts.""" + ... + + def insert_returning_id(self, sql: str, params: Sequence = ()) -> int: + """Insert a row and return the auto-generated ID.""" + ... + + def close(self) -> None: + """Close the connection.""" + ... + + +class MySQLBackend: + """MySQL backend using PyMySQL.""" + + def __init__(self, config: dict | None = None): + import pymysql + import pymysql.cursors + + if config is None: + config = get_db_config().get("mysql", {}) + + self._conn = pymysql.connect( + host=config.get("host", "localhost"), + port=int(config.get("port", 3306)), + user=config.get("user", "root"), + password=config.get("password", ""), + database=config.get("database", "reference_library"), + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + autocommit=True, + ) + + def execute(self, sql: str, params: Sequence = ()) -> int: + with self._conn.cursor() as cur: + return cur.execute(sql, params) + + def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]: + with self._conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchone() + + def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]: + with self._conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchall() + + def insert_returning_id(self, sql: str, params: Sequence = ()) -> int: + with self._conn.cursor() as cur: + cur.execute(sql, params) + return cur.lastrowid + + def close(self) -> None: + self._conn.close() + + +class FileBackend: + """JSON file-based backend for use without MySQL. + + Stores data as JSON arrays in the state directory. + Supports basic CRUD but not complex queries or JOINs. + """ + + def __init__(self, state_dir: Path | None = None): + self._dir = state_dir or get_state_dir() + self._dir.mkdir(parents=True, exist_ok=True) + self._cache: dict[str, list[dict]] = {} + self._counters: dict[str, int] = {} + self._load_counters() + + def _table_path(self, table: str) -> Path: + return self._dir / f"{table}.json" + + def _load_table(self, table: str) -> list[dict]: + if table not in self._cache: + path = self._table_path(table) + if path.is_file(): + self._cache[table] = json.loads(path.read_text()) + else: + self._cache[table] = [] + return self._cache[table] + + def _save_table(self, table: str) -> None: + path = self._table_path(table) + path.write_text(json.dumps(self._cache.get(table, []), indent=2, default=str)) + + def _load_counters(self) -> None: + counter_path = self._dir / "_counters.json" + if counter_path.is_file(): + self._counters = json.loads(counter_path.read_text()) + + def _save_counters(self) -> None: + counter_path = self._dir / "_counters.json" + counter_path.write_text(json.dumps(self._counters, indent=2)) + + def _next_id(self, table: str) -> int: + current = self._counters.get(table, 0) + self._counters[table] = current + 1 + self._save_counters() + return current + 1 + + # --- Protocol methods --- + # These provide basic support for the most common operations. + # Complex SQL is not supported; use MySQL for full functionality. + + def execute(self, sql: str, params: Sequence = ()) -> int: + """Basic INSERT/UPDATE/DELETE support via SQL pattern matching.""" + sql_lower = sql.strip().lower() + table = _extract_table_name(sql) + + if sql_lower.startswith("insert"): + return self._handle_insert(table, sql, params) + elif sql_lower.startswith("update"): + return self._handle_update(table, sql, params) + elif sql_lower.startswith("delete"): + return self._handle_delete(table, sql, params) + + logger.warning("FileBackend: unsupported SQL operation: %s", sql[:60]) + return 0 + + def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]: + rows = self.fetch_all(sql, params) + return rows[0] if rows else None + + def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]: + table = _extract_table_name(sql) + if not table: + logger.warning("FileBackend: cannot extract table from: %s", sql[:60]) + return [] + + # Hard-fail on SQL patterns that FileBackend cannot handle correctly + _reject_unsupported_sql(sql) + + rows = self._load_table(table) + + # Basic WHERE clause filtering + conditions = _extract_where_conditions(sql, params) + if conditions: + rows = [r for r in rows if _matches_conditions(r, conditions)] + + # Basic ORDER BY + order_col = _extract_order_by(sql) + if order_col: + desc = "desc" in sql.lower().split("order by")[-1].lower() + rows = sorted(rows, key=lambda r: r.get(order_col, ""), reverse=desc) + + # Basic LIMIT + limit = _extract_limit(sql) + if limit is not None: + rows = rows[:limit] + + return rows + + def insert_returning_id(self, sql: str, params: Sequence = ()) -> int: + table = _extract_table_name(sql) + self._handle_insert(table, sql, params) + return self._counters.get(table, 0) + + def close(self) -> None: + pass # No connection to close + + # --- Internal handlers --- + + def _handle_insert(self, table: str, sql: str, params: Sequence) -> int: + columns = _extract_insert_columns(sql) + if not columns or len(columns) != len(params): + logger.warning("FileBackend: column/param mismatch for INSERT into %s", table) + return 0 + + row = dict(zip(columns, params)) + pk = _primary_key_for(table) + if pk and pk not in row: + row[pk] = self._next_id(table) + + rows = self._load_table(table) + rows.append(row) + self._cache[table] = rows + self._save_table(table) + return 1 + + def _handle_update(self, table: str, sql: str, params: Sequence) -> int: + rows = self._load_table(table) + set_cols = _extract_set_columns(sql) + conditions = _extract_where_conditions(sql, params[len(set_cols):]) + set_values = list(params[:len(set_cols)]) + + count = 0 + for row in rows: + if _matches_conditions(row, conditions): + for col, val in zip(set_cols, set_values): + row[col] = val + count += 1 + + if count > 0: + self._save_table(table) + return count + + def _handle_delete(self, table: str, sql: str, params: Sequence) -> int: + rows = self._load_table(table) + conditions = _extract_where_conditions(sql, params) + before = len(rows) + self._cache[table] = [r for r in rows if not _matches_conditions(r, conditions)] + self._save_table(table) + return before - len(self._cache[table]) + + +class UnsupportedQueryError(Exception): + """Raised when FileBackend encounters SQL it cannot handle correctly.""" + pass + + +def _reject_unsupported_sql(sql: str) -> None: + """Raise UnsupportedQueryError if the SQL uses patterns FileBackend cannot handle. + + FileBackend only supports single-table SELECT with simple WHERE col = %s. + JOINs, subqueries, aggregates, and GROUP BY would return wrong results silently. + """ + import re + sql_upper = sql.upper() + + unsupported = [] + if re.search(r"\bJOIN\b", sql_upper): + unsupported.append("JOIN") + if re.search(r"\bGROUP\s+BY\b", sql_upper): + unsupported.append("GROUP BY") + if re.search(r"\b(MAX|MIN|SUM|AVG|COUNT)\s*\(", sql_upper): + unsupported.append("aggregate functions") + if re.search(r"\bLEFT\s+JOIN\b", sql_upper): + unsupported.append("LEFT JOIN") + if re.search(r"\(\s*SELECT\b", sql_upper): + unsupported.append("subquery") + + if unsupported: + raise UnsupportedQueryError( + f"FileBackend cannot execute queries with {', '.join(unsupported)}. " + f"Configure MySQL or use 'backend: mysql' in pipeline_config.yaml. " + f"Query: {sql[:80]}..." + ) + + +# --- SQL parsing helpers (minimal, covers common patterns) --- + +def _extract_table_name(sql: str) -> str: + """Extract the primary table name from a SQL statement.""" + import re + sql_clean = sql.strip() + + # INSERT INTO table + m = re.search(r"(?:insert\s+into|update|delete\s+from|from)\s+(\w+)", sql_clean, re.I) + if m: + return m.group(1) + return "" + + +def _extract_insert_columns(sql: str) -> list[str]: + """Extract column names from INSERT INTO table (col1, col2, ...).""" + import re + m = re.search(r"\(([^)]+)\)\s*VALUES", sql, re.I) + if m: + return [c.strip() for c in m.group(1).split(",")] + return [] + + +def _extract_set_columns(sql: str) -> list[str]: + """Extract column names from UPDATE ... SET col1 = %s, col2 = %s.""" + import re + m = re.search(r"SET\s+(.+?)(?:\s+WHERE|$)", sql, re.I | re.S) + if m: + return [c.strip().split("=")[0].strip() for c in m.group(1).split(",")] + return [] + + +def _extract_where_conditions(sql: str, params: Sequence) -> list[tuple[str, Any]]: + """Extract simple col = %s conditions from WHERE clause.""" + import re + m = re.search(r"WHERE\s+(.+?)(?:\s+ORDER|\s+LIMIT|$)", sql, re.I | re.S) + if not m: + return [] + + where_clause = m.group(1) + cols = re.findall(r"(\w+)\s*=\s*%s", where_clause) + # Map to params (taking from the end of params for UPDATE, all for SELECT) + return list(zip(cols, params[-len(cols):] if cols else [])) + + +def _extract_order_by(sql: str) -> str: + """Extract the first ORDER BY column.""" + import re + m = re.search(r"ORDER\s+BY\s+(\w+)", sql, re.I) + return m.group(1) if m else "" + + +def _extract_limit(sql: str) -> int | None: + """Extract LIMIT value.""" + import re + m = re.search(r"LIMIT\s+(\d+)", sql, re.I) + return int(m.group(1)) if m else None + + +def _matches_conditions(row: dict, conditions: list[tuple[str, Any]]) -> bool: + """Check if a row matches all WHERE conditions.""" + return all(str(row.get(col)) == str(val) for col, val in conditions) + + +def _primary_key_for(table: str) -> str: + """Return the auto-increment primary key name for a table.""" + pk_map = { + "sources": "source_id", + "documents": "doc_id", + "distilled_content": "distill_id", + "review_logs": "review_id", + "topics": "topic_id", + "export_jobs": "export_id", + "pipeline_runs": "run_id", + "pipeline_iteration_tracker": "tracker_id", + "crawl_schedule": "schedule_id", + "change_detection": "change_id", + } + return pk_map.get(table, "") + + +# --- Factory --- + +def get_backend(backend_type: str | None = None) -> DatabaseBackend: + """Create and return the appropriate database backend. + + Args: + backend_type: 'mysql' or 'file'. If None, reads from pipeline config. + + Returns: + A DatabaseBackend instance. + """ + if backend_type is None: + backend_type = get_state_backend() + + if backend_type == "mysql": + return MySQLBackend() + + return FileBackend() + + +@contextmanager +def db_session(backend_type: str | None = None): + """Context manager for database sessions. + + Usage: + with db_session() as db: + rows = db.fetch_all("SELECT * FROM documents") + """ + db = get_backend(backend_type) + try: + yield db + finally: + db.close() diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/gemini.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/gemini.py new file mode 100644 index 0000000..eef52fd --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/gemini.py @@ -0,0 +1,182 @@ +"""Gemini CLI wrapper for independent content quality evaluation. + +Uses the Gemini CLI (google/gemini-cli) to evaluate raw crawled content +before distillation, providing third-party quality assessment. + +Requires: `npm install -g @google/gemini-cli` and Google auth configured. +""" + +from __future__ import annotations + +import json +import logging +import re +import subprocess +from typing import Optional + +logger = logging.getLogger("refcurator.gemini") + +GEMINI_CMD = "gemini" +TIMEOUT_SECONDS = 60 + +EVALUATION_PROMPT = """You are evaluating a raw reference document for inclusion in a curated knowledge base. + +Topic: {topic} +Source URL: {source_url} + +Score each criterion from 0.0 to 1.0: +- relevance: Does this content actually relate to the topic "{topic}"? +- authority: Is this from an authoritative, official source (official docs, research paper) or low-quality (scraped blog, forum post, SEO spam)? +- completeness: Is this a complete article with substance, or a navigation fragment, error page, stub, or boilerplate? +- freshness: Does the information appear current and not outdated? Look for version numbers, dates, deprecated APIs. +- distill_value: Does this contain unique, valuable information worth summarizing, or is it redundant with what official docs already cover? + +Return ONLY a valid JSON object with no markdown formatting, no code fences, no explanation: +{{"relevance": 0.0, "authority": 0.0, "completeness": 0.0, "freshness": 0.0, "distill_value": 0.0, "verdict": "approve", "reason": "brief explanation"}} + +The verdict must be one of: "approve", "reject", "deep_research" +- approve: source is worth distilling (score >= 0.75 typical) +- reject: not worth distilling (low quality, irrelevant, or fragment) +- deep_research: partially relevant but needs supplementary sources""" + + +def is_available() -> bool: + """Check if the Gemini CLI is installed and authenticated.""" + try: + result = subprocess.run( + [GEMINI_CMD, "--help"], + capture_output=True, timeout=10, + ) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +def evaluate_content( + content: str, + topic: str, + source_url: str = "", + timeout: int = TIMEOUT_SECONDS, +) -> Optional[dict]: + """Evaluate raw content using Gemini CLI. + + Args: + content: Raw document content (markdown/text) + topic: The curation topic for relevance scoring + source_url: Original URL of the content + timeout: Subprocess timeout in seconds + + Returns: + Parsed evaluation dict with scores, verdict, and reason. + Returns None if Gemini is unavailable or evaluation fails. + """ + # Truncate very long content to avoid overwhelming the model + max_chars = 50_000 + if len(content) > max_chars: + content = content[:max_chars] + "\n\n[... content truncated for evaluation ...]" + + prompt = EVALUATION_PROMPT.format(topic=topic, source_url=source_url) + + try: + result = subprocess.run( + [GEMINI_CMD, prompt], + input=content, + capture_output=True, + text=True, + timeout=timeout, + ) + + if result.returncode != 0: + logger.warning("Gemini CLI failed (exit %d): %s", result.returncode, result.stderr[:200]) + return None + + return _parse_response(result.stdout) + + except FileNotFoundError: + logger.warning("Gemini CLI not found. Install with: npm install -g @google/gemini-cli") + return None + except subprocess.TimeoutExpired: + logger.warning("Gemini CLI timed out after %ds", timeout) + return None + except Exception as e: + logger.warning("Gemini evaluation failed: %s", e) + return None + + +def _parse_response(output: str) -> Optional[dict]: + """Parse Gemini's response, handling markdown-wrapped JSON.""" + text = output.strip() + + # Try direct JSON parse first + try: + return _validate_evaluation(json.loads(text)) + except json.JSONDecodeError: + pass + + # Try extracting JSON from markdown code fences + m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL) + if m: + try: + return _validate_evaluation(json.loads(m.group(1).strip())) + except json.JSONDecodeError: + pass + + # Try finding a JSON object anywhere in the output + m = re.search(r"\{[^{}]*\"relevance\"[^{}]*\}", text, re.DOTALL) + if m: + try: + return _validate_evaluation(json.loads(m.group(0))) + except json.JSONDecodeError: + pass + + logger.warning("Could not parse Gemini response as JSON: %s", text[:200]) + return None + + +def _validate_evaluation(data: dict) -> Optional[dict]: + """Validate that the evaluation has required fields and reasonable values.""" + required_scores = ["relevance", "authority", "completeness", "freshness", "distill_value"] + + for field in required_scores: + if field not in data: + logger.warning("Missing required field: %s", field) + return None + score = data[field] + if not isinstance(score, (int, float)) or score < 0 or score > 1: + logger.warning("Invalid score for %s: %s", field, score) + return None + + if "verdict" not in data or data["verdict"] not in ("approve", "reject", "deep_research"): + data["verdict"] = _derive_verdict(data) + + if "reason" not in data: + data["reason"] = "" + + # Add weighted score + data["weighted_score"] = round( + data["relevance"] * 0.25 + + data["authority"] * 0.25 + + data["completeness"] * 0.20 + + data["freshness"] * 0.15 + + data["distill_value"] * 0.15, + 4, + ) + + return data + + +def _derive_verdict(data: dict) -> str: + """Derive verdict from scores if Gemini didn't provide one.""" + score = ( + data.get("relevance", 0) * 0.25 + + data.get("authority", 0) * 0.25 + + data.get("completeness", 0) * 0.20 + + data.get("freshness", 0) * 0.15 + + data.get("distill_value", 0) * 0.15 + ) + if score >= 0.75: + return "approve" + elif score >= 0.50: + return "deep_research" + else: + return "reject" diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/manifest.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/manifest.py new file mode 100644 index 0000000..4d86841 --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/manifest.py @@ -0,0 +1,90 @@ +"""Manifest I/O for reference discovery and crawl results.""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path + +from refcurator.models import CrawlResult, CrawlResultEntry, Manifest, ManifestURL +from refcurator.utils import normalize_url + + +def read_manifest(path: Path) -> Manifest: + """Read a manifest JSON file.""" + data = json.loads(path.read_text()) + return Manifest(**data) + + +def write_manifest(manifest: Manifest, path: Path) -> None: + """Write a manifest to a JSON file.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(manifest.model_dump_json(indent=2)) + + +def merge_manifests(manifests: list[Manifest]) -> Manifest: + """Merge multiple manifests, deduplicating URLs.""" + seen: dict[str, ManifestURL] = {} + topic_parts = [] + + for m in manifests: + if m.topic: + topic_parts.append(m.topic) + for url_entry in m.urls: + normalized = normalize_url(url_entry.url) + existing = seen.get(normalized) + if existing is None or ( + url_entry.credibility_score + and (existing.credibility_score or 0) < url_entry.credibility_score + ): + seen[normalized] = url_entry + + urls = list(seen.values()) + return Manifest( + discovery_date=datetime.now().isoformat(), + topic=" + ".join(topic_parts) if topic_parts else None, + total_urls=len(urls), + urls=urls, + ) + + +def dedup_manifest_urls(manifest: Manifest, existing_urls: set[str]) -> Manifest: + """Remove URLs already in the existing set (normalized comparison).""" + existing_normalized = {normalize_url(u) for u in existing_urls} + filtered = [u for u in manifest.urls if normalize_url(u.url) not in existing_normalized] + return Manifest( + discovery_date=manifest.discovery_date, + topic=manifest.topic, + total_urls=len(filtered), + urls=filtered, + ) + + +def read_crawl_result(path: Path) -> CrawlResult: + """Read a crawl result JSON file.""" + data = json.loads(path.read_text()) + return CrawlResult(**data) + + +def write_crawl_result(result: CrawlResult, path: Path) -> None: + """Write a crawl result to a JSON file.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(result.model_dump_json(indent=2)) + + +def create_crawl_result( + entries: list[dict], + crawler: str = "firecrawl", +) -> CrawlResult: + """Create a CrawlResult from a list of crawl entry dicts.""" + docs = [CrawlResultEntry(**e) for e in entries] + completed = [d for d in docs if d.status == "completed"] + failed = [d for d in docs if d.status != "completed"] + + return CrawlResult( + crawl_date=datetime.now().isoformat(), + crawler_used=crawler, + total_crawled=len(completed), + total_failed=len(failed), + documents=docs, + ) diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/models.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/models.py new file mode 100644 index 0000000..c21b5f2 --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/models.py @@ -0,0 +1,355 @@ +"""Pydantic v2 models matching the reference_library MySQL schema.""" + +from __future__ import annotations + +from datetime import date, datetime +from enum import Enum +from typing import Any, Optional + +from pydantic import BaseModel, Field, computed_field + + +# --- Enums matching MySQL ENUMs --- + +class SourceType(str, Enum): + official_docs = "official_docs" + engineering_blog = "engineering_blog" + research_paper = "research_paper" + github_repo = "github_repo" + community_guide = "community_guide" + pdf_document = "pdf_document" + api_reference = "api_reference" + + +class CredibilityTier(str, Enum): + tier1_official = "tier1_official" + tier2_verified = "tier2_verified" + tier3_community = "tier3_community" + + +class DocType(str, Enum): + webpage = "webpage" + pdf = "pdf" + markdown = "markdown" + api_spec = "api_spec" + code_sample = "code_sample" + + +class Language(str, Enum): + en = "en" + ko = "ko" + mixed = "mixed" + + +class CrawlMethod(str, Enum): + firecrawl = "firecrawl" + scrapy = "scrapy" + aiohttp = "aiohttp" + nodejs = "nodejs" + manual = "manual" + api = "api" + + +class CrawlStatus(str, Enum): + pending = "pending" + completed = "completed" + failed = "failed" + stale = "stale" + + +class ReviewStatus(str, Enum): + pending = "pending" + in_review = "in_review" + approved = "approved" + needs_refactor = "needs_refactor" + rejected = "rejected" + + +class ReviewerType(str, Enum): + auto_qa = "auto_qa" + human = "human" + claude_review = "claude_review" + gemini_review = "gemini_review" + + +class Decision(str, Enum): + approve = "approve" + refactor = "refactor" + deep_research = "deep_research" + reject = "reject" + + +class PipelineStatus(str, Enum): + running = "running" + completed = "completed" + failed = "failed" + paused = "paused" + + +class PipelineStage(str, Enum): + discovery = "discovery" + crawling = "crawling" + storing = "storing" + evaluating = "evaluating" + distilling = "distilling" + exporting = "exporting" + + +class RunType(str, Enum): + topic = "topic" + urls = "urls" + manifest = "manifest" + + +class ExportType(str, Enum): + project_files = "project_files" + fine_tuning = "fine_tuning" + training_dataset = "training_dataset" + knowledge_base = "knowledge_base" + + +class OutputFormat(str, Enum): + markdown = "markdown" + jsonl = "jsonl" + parquet = "parquet" + sqlite = "sqlite" + + +class Frequency(str, Enum): + daily = "daily" + weekly = "weekly" + biweekly = "biweekly" + monthly = "monthly" + on_demand = "on_demand" + + +class ChangeType(str, Enum): + content_updated = "content_updated" + url_moved = "url_moved" + deleted = "deleted" + new_version = "new_version" + + +class FinalDecision(str, Enum): + approved = "approved" + rejected = "rejected" + needs_manual_review = "needs_manual_review" + + +# --- Core Table Models --- + +class Source(BaseModel): + source_id: Optional[int] = None + source_name: str + source_type: SourceType + base_url: Optional[str] = None + credibility_tier: CredibilityTier = CredibilityTier.tier3_community + vendor: Optional[str] = None + is_active: bool = True + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +class Document(BaseModel): + doc_id: Optional[int] = None + source_id: int + title: str + url: Optional[str] = None + url_hash: Optional[str] = None # Generated column in MySQL + doc_type: DocType + language: Language = Language.en + original_publish_date: Optional[date] = None + last_modified_date: Optional[date] = None + crawl_date: Optional[datetime] = None + crawl_method: CrawlMethod = CrawlMethod.firecrawl + crawl_status: CrawlStatus = CrawlStatus.pending + raw_content_path: Optional[str] = None + raw_content_size: Optional[int] = None + version: int = 1 + previous_version_id: Optional[int] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +class DistilledContent(BaseModel): + distill_id: Optional[int] = None + doc_id: int + summary: Optional[str] = None + key_concepts: Optional[list[dict[str, Any]]] = None + code_snippets: Optional[list[dict[str, Any]]] = None + structured_content: Optional[str] = None + token_count_original: Optional[int] = None + token_count_distilled: Optional[int] = None + distill_model: Optional[str] = None + distill_date: Optional[datetime] = None + review_status: ReviewStatus = ReviewStatus.pending + + @computed_field + @property + def compression_ratio(self) -> Optional[float]: + if self.token_count_original and self.token_count_distilled: + return round(self.token_count_distilled / self.token_count_original * 100, 2) + return None + + +class ReviewLog(BaseModel): + review_id: Optional[int] = None + distill_id: int + review_round: int = 1 + reviewer_type: ReviewerType + quality_score: Optional[float] = None + assessment: Optional[dict[str, float]] = None + decision: Decision + feedback: Optional[str] = None + refactor_instructions: Optional[str] = None + research_queries: Optional[list[str]] = None + reviewed_at: Optional[datetime] = None + + +class Topic(BaseModel): + topic_id: Optional[int] = None + topic_name: str + topic_slug: str + parent_topic_id: Optional[int] = None + description: Optional[str] = None + + +class DocumentTopic(BaseModel): + doc_id: int + topic_id: int + relevance_score: float = 1.0 + + +class ExportJob(BaseModel): + export_id: Optional[int] = None + export_name: str + export_type: ExportType + output_format: OutputFormat = OutputFormat.markdown + topic_filter: Optional[list[int]] = None + date_range_start: Optional[date] = None + date_range_end: Optional[date] = None + min_quality_score: float = 0.80 + output_path: Optional[str] = None + total_documents: Optional[int] = None + total_tokens: Optional[int] = None + status: str = "pending" + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + created_at: Optional[datetime] = None + + +class PipelineRun(BaseModel): + run_id: Optional[int] = None + run_type: RunType + input_value: str + status: PipelineStatus = PipelineStatus.running + current_stage: PipelineStage = PipelineStage.discovery + options: Optional[dict[str, Any]] = None + stats: Optional[dict[str, int]] = Field(default_factory=lambda: { + "sources_discovered": 0, + "pages_crawled": 0, + "documents_stored": 0, + "documents_distilled": 0, + "approved": 0, + "refactored": 0, + "deep_researched": 0, + "rejected": 0, + "needs_manual_review": 0, + }) + export_path: Optional[str] = None + export_document_count: Optional[int] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + error_stage: Optional[str] = None + + +class PipelineIterationTracker(BaseModel): + tracker_id: Optional[int] = None + run_id: int + doc_id: int + refactor_count: int = 0 + deep_research_count: int = 0 + final_decision: Optional[FinalDecision] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +# --- Non-DB Models (manifest/crawl/assessment) --- + +class ManifestURL(BaseModel): + url: str + title: Optional[str] = None + credibility_tier: Optional[str] = None + credibility_score: Optional[float] = None + source_type: Optional[str] = None + vendor: Optional[str] = None + + +class Manifest(BaseModel): + discovery_date: Optional[str] = None + topic: Optional[str] = None + total_urls: int = 0 + urls: list[ManifestURL] = Field(default_factory=list) + + +class CrawlResultEntry(BaseModel): + url: str + title: Optional[str] = None + raw_path: str + content_size: int = 0 + status: str = "completed" + error: Optional[str] = None + + +class CrawlResult(BaseModel): + crawl_date: Optional[str] = None + crawler_used: str = "firecrawl" + total_crawled: int = 0 + total_failed: int = 0 + documents: list[CrawlResultEntry] = Field(default_factory=list) + + +class QAAssessment(BaseModel): + """Legacy model for post-distillation Claude self-review (deprecated).""" + accuracy: float = 0.0 + completeness: float = 0.0 + clarity: float = 0.0 + prompt_engineering_quality: float = 0.0 + usability: float = 0.0 + + @computed_field + @property + def weighted_score(self) -> float: + return round( + self.accuracy * 0.25 + + self.completeness * 0.20 + + self.clarity * 0.20 + + self.prompt_engineering_quality * 0.25 + + self.usability * 0.10, + 4, + ) + + +class SourceQAAssessment(BaseModel): + """Pre-distillation source quality assessment via Gemini.""" + relevance: float = 0.0 + authority: float = 0.0 + completeness: float = 0.0 + freshness: float = 0.0 + distill_value: float = 0.0 + verdict: str = "" + reason: str = "" + + @computed_field + @property + def weighted_score(self) -> float: + return round( + self.relevance * 0.25 + + self.authority * 0.25 + + self.completeness * 0.20 + + self.freshness * 0.15 + + self.distill_value * 0.15, + 4, + ) diff --git a/custom-skills/90-reference-curator/shared/lib/src/refcurator/utils.py b/custom-skills/90-reference-curator/shared/lib/src/refcurator/utils.py new file mode 100644 index 0000000..0cfc1a4 --- /dev/null +++ b/custom-skills/90-reference-curator/shared/lib/src/refcurator/utils.py @@ -0,0 +1,75 @@ +"""Common utilities for the reference curator pipeline.""" + +from __future__ import annotations + +import hashlib +import logging +import re +import unicodedata +from urllib.parse import urlparse, urlunparse, parse_qs, urlencode + + +def normalize_url(url: str) -> str: + """Normalize a URL for deduplication. + + - Lowercase scheme and host + - Remove trailing slashes + - Sort query parameters + - Remove common tracking params (utm_*, ref, fbclid) + - Remove fragment + """ + parsed = urlparse(url) + scheme = parsed.scheme.lower() + netloc = parsed.netloc.lower() + path = parsed.path.rstrip("/") or "/" + + # Sort query params, removing tracking params + tracking_params = {"utm_source", "utm_medium", "utm_campaign", "utm_content", + "utm_term", "ref", "fbclid", "gclid", "mc_cid", "mc_eid"} + params = parse_qs(parsed.query, keep_blank_values=True) + filtered = {k: v for k, v in sorted(params.items()) if k not in tracking_params} + query = urlencode(filtered, doseq=True) + + return urlunparse((scheme, netloc, path, "", query, "")) + + +def url_hash(url: str) -> str: + """SHA-256 hash of normalized URL. Matches the url_hash column in schema.sql.""" + return hashlib.sha256(normalize_url(url).encode()).hexdigest() + + +def slugify(text: str) -> str: + """Convert text to a URL/folder-friendly slug. + + >>> slugify("Prompt Engineering Best Practices") + 'prompt-engineering-best-practices' + """ + text = unicodedata.normalize("NFKD", text) + text = text.encode("ascii", "ignore").decode() + text = text.lower() + text = re.sub(r"[^a-z0-9]+", "-", text) + text = text.strip("-") + return text or "untitled" + + +def count_tokens(text: str) -> int: + """Approximate token count using chars/4 heuristic. + + Good enough for compression ratio calculations without requiring tiktoken. + """ + return max(1, len(text) // 4) + + +def setup_logging(level: str = "INFO", run_id: int | None = None) -> logging.Logger: + """Configure and return a logger for the reference curator pipeline.""" + logger = logging.getLogger("refcurator") + if not logger.handlers: + handler = logging.StreamHandler() + fmt = "[refcurator]" + if run_id: + fmt += f" [run:{run_id}]" + fmt += " %(levelname)s: %(message)s" + handler.setFormatter(logging.Formatter(fmt)) + logger.addHandler(handler) + logger.setLevel(getattr(logging, level.upper(), logging.INFO)) + return logger