feat(reference-curator): implement Python scripts + Gemini quality gate
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -36,8 +36,25 @@ Apply credibility scoring:
|
||||
- Relevance signals (0.15)
|
||||
|
||||
### Step 4: Output URL Manifest
|
||||
Generate JSON manifest for the crawler skill:
|
||||
Save discovered URLs as a manifest JSON, deduplicating against the existing repository:
|
||||
|
||||
```bash
|
||||
# Create manifest from discovered URLs
|
||||
uv run python scripts/discover.py create-manifest --topic "prompt engineering" --output manifest.json < urls.json
|
||||
|
||||
# Deduplicate against existing DB
|
||||
uv run python scripts/discover.py dedup --manifest manifest.json
|
||||
|
||||
# Register a new source
|
||||
uv run python scripts/discover.py register-source \
|
||||
--name "Anthropic Docs" --type official_docs \
|
||||
--url "https://docs.anthropic.com" --tier tier1_official --vendor anthropic
|
||||
|
||||
# List registered sources
|
||||
uv run python scripts/discover.py list-sources --vendor anthropic
|
||||
```
|
||||
|
||||
Manifest format:
|
||||
```json
|
||||
{
|
||||
"discovery_date": "2025-01-28T10:30:00",
|
||||
@@ -56,20 +73,31 @@ Generate JSON manifest for the crawler skill:
|
||||
}
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
### `discover_sources.py`
|
||||
Main discovery script. Usage:
|
||||
```bash
|
||||
python scripts/discover_sources.py --topic "prompt engineering" --vendors anthropic,openai --output manifest.json
|
||||
```
|
||||
|
||||
## Output
|
||||
- `manifest.json` → Handoff to `02-web-crawler-orchestrator`
|
||||
- Register new sources in `sources` table via `03-content-repository`
|
||||
- New sources registered in `sources` table via `register-source`
|
||||
|
||||
## Deduplication
|
||||
Before outputting:
|
||||
- Normalize URLs (remove trailing slashes, query params)
|
||||
- Check against existing `documents` table
|
||||
- Merge duplicates, keeping highest credibility score
|
||||
|
||||
## Scripts
|
||||
|
||||
All scripts require the `refcurator` package. Run with `uv run python` from the skill directory.
|
||||
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `discover.py create-manifest` | Create manifest from URL entries JSON |
|
||||
| `discover.py dedup` | Deduplicate manifest against DB |
|
||||
| `discover.py register-source` | Register a new source |
|
||||
| `discover.py list-sources` | List registered sources |
|
||||
|
||||
## Integration
|
||||
|
||||
| From | To |
|
||||
|------|-----|
|
||||
| WebSearch results | → manifest.json |
|
||||
| → manifest.json | web-crawler-orchestrator |
|
||||
| → register-source | content-repository (sources table) |
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reference Discovery CLI — manage source discovery and manifests.
|
||||
|
||||
Claude performs the actual web search. This script handles:
|
||||
- Receiving search results and creating manifests
|
||||
- Deduplicating URLs against the existing repository
|
||||
- Registering new sources in the database
|
||||
|
||||
Usage:
|
||||
python discover.py create-manifest --topic "prompt engineering" --output manifest.json < urls.json
|
||||
python discover.py dedup --manifest manifest.json [--output deduped.json]
|
||||
python discover.py register-source --name "Anthropic Docs" --type official_docs --url "https://docs.anthropic.com" --tier tier1_official --vendor anthropic
|
||||
python discover.py list-sources [--vendor anthropic]
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.manifest import write_manifest, read_manifest, dedup_manifest_urls
|
||||
from refcurator.models import Manifest, ManifestURL
|
||||
from refcurator.utils import normalize_url
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Reference Discovery — manage source manifests."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("create-manifest")
|
||||
@click.option("--topic", required=True, help="Discovery topic")
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output manifest path")
|
||||
@click.option("--input", "input_file", type=click.Path(exists=True),
|
||||
help="Input JSON file with URL entries (or pipe via stdin)")
|
||||
def create_manifest(topic, output, input_file):
|
||||
"""Create a manifest from discovered URLs.
|
||||
|
||||
Input format (JSON array):
|
||||
[
|
||||
{"url": "https://...", "title": "...", "credibility_score": 0.85, "source_type": "official_docs"},
|
||||
...
|
||||
]
|
||||
"""
|
||||
if input_file:
|
||||
data = json.loads(Path(input_file).read_text())
|
||||
elif not sys.stdin.isatty():
|
||||
data = json.load(sys.stdin)
|
||||
else:
|
||||
console.print("[red]Error:[/red] Provide URLs via --input file or stdin")
|
||||
sys.exit(1)
|
||||
|
||||
urls = [ManifestURL(**entry) for entry in data]
|
||||
|
||||
# Normalize and deduplicate
|
||||
seen = {}
|
||||
for u in urls:
|
||||
norm = normalize_url(u.url)
|
||||
if norm not in seen or (u.credibility_score or 0) > (seen[norm].credibility_score or 0):
|
||||
seen[norm] = u
|
||||
|
||||
unique_urls = list(seen.values())
|
||||
|
||||
manifest = Manifest(
|
||||
discovery_date=datetime.now().isoformat(),
|
||||
topic=topic,
|
||||
total_urls=len(unique_urls),
|
||||
urls=unique_urls,
|
||||
)
|
||||
|
||||
out_path = Path(output)
|
||||
write_manifest(manifest, out_path)
|
||||
console.print(f"[green]Created manifest:[/green] {len(unique_urls)} URLs → {output}")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--manifest", required=True, type=click.Path(exists=True), help="Manifest to dedup")
|
||||
@click.option("--output", type=click.Path(), help="Output path (defaults to overwriting input)")
|
||||
def dedup(manifest, output):
|
||||
"""Deduplicate manifest URLs against existing repository."""
|
||||
m = read_manifest(Path(manifest))
|
||||
original_count = len(m.urls)
|
||||
|
||||
with db_session() as db:
|
||||
existing = db.fetch_all("SELECT url FROM documents WHERE url IS NOT NULL")
|
||||
existing_urls = {r["url"] for r in existing}
|
||||
|
||||
deduped = dedup_manifest_urls(m, existing_urls)
|
||||
removed = original_count - len(deduped.urls)
|
||||
|
||||
out_path = Path(output) if output else Path(manifest)
|
||||
write_manifest(deduped, out_path)
|
||||
|
||||
console.print(
|
||||
f"[green]Deduped:[/green] {original_count} → {len(deduped.urls)} URLs "
|
||||
f"({removed} duplicates removed)"
|
||||
)
|
||||
|
||||
|
||||
@cli.command("register-source")
|
||||
@click.option("--name", required=True, help="Source name")
|
||||
@click.option("--type", "source_type", required=True,
|
||||
type=click.Choice(["official_docs", "engineering_blog", "research_paper",
|
||||
"github_repo", "community_guide", "pdf_document", "api_reference"]))
|
||||
@click.option("--url", required=True, help="Base URL")
|
||||
@click.option("--tier", default="tier3_community",
|
||||
type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]))
|
||||
@click.option("--vendor", help="Vendor name (e.g., anthropic, openai)")
|
||||
def register_source(name, source_type, url, tier, vendor):
|
||||
"""Register a new source in the repository."""
|
||||
with db_session() as db:
|
||||
# Check for existing source with same base_url
|
||||
existing = db.fetch_one(
|
||||
"SELECT source_id, source_name FROM sources WHERE base_url = %s",
|
||||
(url,),
|
||||
)
|
||||
if existing:
|
||||
console.print(
|
||||
f"[yellow]Already registered:[/yellow] source_id={existing['source_id']} "
|
||||
f"({existing['source_name']})"
|
||||
)
|
||||
return
|
||||
|
||||
source_id = db.insert_returning_id(
|
||||
"""INSERT INTO sources (source_name, source_type, base_url, credibility_tier, vendor)
|
||||
VALUES (%s, %s, %s, %s, %s)""",
|
||||
(name, source_type, url, tier, vendor),
|
||||
)
|
||||
|
||||
console.print(f"[green]Registered:[/green] source_id={source_id} — {name}")
|
||||
click.echo(json.dumps({"source_id": source_id, "name": name, "url": url}))
|
||||
|
||||
|
||||
@cli.command("list-sources")
|
||||
@click.option("--vendor", help="Filter by vendor")
|
||||
@click.option("--tier", type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]),
|
||||
help="Filter by credibility tier")
|
||||
def list_sources(vendor, tier):
|
||||
"""List registered sources."""
|
||||
with db_session() as db:
|
||||
if vendor:
|
||||
rows = db.fetch_all(
|
||||
"SELECT * FROM sources WHERE vendor = %s ORDER BY credibility_tier",
|
||||
(vendor,),
|
||||
)
|
||||
elif tier:
|
||||
rows = db.fetch_all(
|
||||
"SELECT * FROM sources WHERE credibility_tier = %s ORDER BY vendor",
|
||||
(tier,),
|
||||
)
|
||||
else:
|
||||
rows = db.fetch_all("SELECT * FROM sources ORDER BY credibility_tier, vendor")
|
||||
|
||||
table = Table(title="Registered Sources")
|
||||
table.add_column("ID", style="cyan")
|
||||
table.add_column("Name")
|
||||
table.add_column("Type")
|
||||
table.add_column("Tier")
|
||||
table.add_column("Vendor")
|
||||
table.add_column("URL")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("source_id", "")),
|
||||
str(r.get("source_name", "")),
|
||||
str(r.get("source_type", "")),
|
||||
str(r.get("credibility_tier", "")),
|
||||
str(r.get("vendor", "")),
|
||||
str(r.get("base_url", ""))[:40],
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -1,115 +1,48 @@
|
||||
# Web Crawler Orchestrator
|
||||
|
||||
Orchestrates web crawling with intelligent backend selection. Automatically chooses the best crawler based on site characteristics.
|
||||
Orchestrates web crawling with intelligent backend selection. Claude performs actual crawling via Firecrawl MCP tools. This skill manages crawl results, selects crawlers, and tracks crawl metadata.
|
||||
|
||||
## Trigger Keywords
|
||||
"crawl URLs", "fetch documents", "scrape pages", "download references"
|
||||
|
||||
## Intelligent Crawler Selection
|
||||
|
||||
Claude automatically selects the optimal crawler based on the request:
|
||||
```bash
|
||||
# Get crawler recommendation for a URL
|
||||
uv run python scripts/crawl_mgr.py select-crawler --url "https://docs.anthropic.com"
|
||||
```
|
||||
|
||||
| Crawler | Best For | Auto-Selected When |
|
||||
|---------|----------|-------------------|
|
||||
| **Node.js** (default) | Small docs sites | ≤50 pages, static content |
|
||||
| **Firecrawl MCP** (default) | Dynamic sites, SPAs | React/Vue/Angular, JS-rendered |
|
||||
| **Node.js** | Small docs sites | ≤50 pages, static content |
|
||||
| **Python aiohttp** | Technical docs | ≤200 pages, needs SEO data |
|
||||
| **Scrapy** | Enterprise crawls | >200 pages, multi-domain |
|
||||
| **Firecrawl MCP** | Dynamic sites | SPAs, JS-rendered content |
|
||||
|
||||
### Decision Flow
|
||||
|
||||
```
|
||||
[Crawl Request]
|
||||
│
|
||||
├─ Is it SPA/React/Vue/Angular? → Firecrawl MCP
|
||||
│
|
||||
├─ >200 pages or multi-domain? → Scrapy
|
||||
│
|
||||
├─ Needs SEO extraction? → Python aiohttp
|
||||
│
|
||||
└─ Default (small site) → Node.js
|
||||
```
|
||||
|
||||
## Crawler Backends
|
||||
|
||||
### Node.js (Default)
|
||||
Fast, lightweight crawler for small documentation sites.
|
||||
```bash
|
||||
cd ~/Project/our-seo-agent/util/js-crawler
|
||||
node src/crawler.js <URL> --max-pages 50
|
||||
```
|
||||
|
||||
### Python aiohttp
|
||||
Async crawler with full SEO extraction.
|
||||
```bash
|
||||
cd ~/Project/our-seo-agent
|
||||
python -m seo_agent.crawler --url <URL> --max-pages 100
|
||||
```
|
||||
|
||||
### Scrapy
|
||||
Enterprise-grade crawler with pipelines.
|
||||
```bash
|
||||
cd ~/Project/our-seo-agent
|
||||
scrapy crawl seo_spider -a start_url=<URL> -a max_pages=500
|
||||
```
|
||||
|
||||
### Firecrawl MCP
|
||||
Use MCP tools for JavaScript-heavy sites:
|
||||
```
|
||||
firecrawl_scrape(url, formats=["markdown"], only_main_content=true)
|
||||
firecrawl_crawl(url, max_depth=2, limit=50)
|
||||
firecrawl_map(url, limit=100) # Discover URLs first
|
||||
```
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Analyze Target Site
|
||||
Determine site characteristics:
|
||||
- Is it a SPA? (React, Vue, Angular, Next.js)
|
||||
- How many pages expected?
|
||||
- Does it need JavaScript rendering?
|
||||
- Is SEO data extraction needed?
|
||||
Run `select-crawler` to determine site characteristics and get a recommendation.
|
||||
|
||||
### Step 2: Select Crawler
|
||||
Based on analysis, select the appropriate backend.
|
||||
### Step 2: Execute Crawl
|
||||
Use Firecrawl MCP tools directly:
|
||||
```
|
||||
firecrawl_map(url, limit=100) # Discover URLs
|
||||
firecrawl_scrape(url, formats=["markdown"], only_main_content=true)
|
||||
firecrawl_crawl(url, max_depth=2, limit=50)
|
||||
```
|
||||
|
||||
### Step 3: Load URL Manifest
|
||||
### Step 3: Store Crawl Results
|
||||
```bash
|
||||
# From reference-discovery output
|
||||
cat manifest.json | jq '.urls[].url'
|
||||
```
|
||||
# Store crawled files and create result manifest
|
||||
uv run python scripts/crawl_mgr.py store-result \
|
||||
--raw-dir ~/Documents/reference-library/raw/ \
|
||||
--crawler firecrawl \
|
||||
--source-id 1 \
|
||||
--output crawl_result.json
|
||||
|
||||
### Step 4: Execute Crawl
|
||||
|
||||
**For Node.js:**
|
||||
```bash
|
||||
cd ~/Project/our-seo-agent/util/js-crawler
|
||||
for url in $(cat urls.txt); do
|
||||
node src/crawler.js "$url" --max-pages 50
|
||||
sleep 2
|
||||
done
|
||||
```
|
||||
|
||||
**For Firecrawl MCP (Claude Desktop/Code):**
|
||||
Use the firecrawl MCP tools directly in conversation.
|
||||
|
||||
### Step 5: Save Raw Content
|
||||
```
|
||||
~/reference-library/raw/
|
||||
└── 2025/01/
|
||||
├── a1b2c3d4.md
|
||||
└── b2c3d4e5.md
|
||||
```
|
||||
|
||||
### Step 6: Generate Crawl Manifest
|
||||
```json
|
||||
{
|
||||
"crawl_date": "2025-01-28T12:00:00",
|
||||
"crawler_used": "nodejs",
|
||||
"total_crawled": 45,
|
||||
"total_failed": 5,
|
||||
"documents": [...]
|
||||
}
|
||||
# List recent crawls
|
||||
uv run python scripts/crawl_mgr.py list-crawls --status completed
|
||||
```
|
||||
|
||||
## Rate Limiting
|
||||
@@ -129,31 +62,19 @@ All crawlers respect these limits:
|
||||
| Access denied (403) | Log, mark as `failed` |
|
||||
| JS rendering needed | Switch to Firecrawl |
|
||||
|
||||
## Site Type Detection
|
||||
|
||||
Indicators for automatic routing:
|
||||
|
||||
**SPA (→ Firecrawl):**
|
||||
- URL contains `#/` or uses hash routing
|
||||
- Page source shows React/Vue/Angular markers
|
||||
- Content loads dynamically after initial load
|
||||
|
||||
**Static docs (→ Node.js/aiohttp):**
|
||||
- Built with Hugo, Jekyll, MkDocs, Docusaurus, GitBook
|
||||
- Clean HTML structure
|
||||
- Server-side rendered
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/select_crawler.py` - Intelligent crawler selection
|
||||
- `scripts/crawl_with_nodejs.sh` - Node.js wrapper
|
||||
- `scripts/crawl_with_aiohttp.sh` - Python wrapper
|
||||
- `scripts/crawl_with_firecrawl.py` - Firecrawl MCP wrapper
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `crawl_mgr.py select-crawler` | Recommend optimal crawler for a URL |
|
||||
| `crawl_mgr.py store-result` | Store crawl results and create manifest |
|
||||
| `crawl_mgr.py list-crawls` | List recent crawl records |
|
||||
|
||||
## Integration
|
||||
|
||||
| From | To |
|
||||
|------|-----|
|
||||
| reference-discovery | URL manifest input |
|
||||
| Firecrawl MCP | Raw crawled files |
|
||||
| → | content-repository (crawl manifest + raw files) |
|
||||
| quality-reviewer (deep_research) | Additional crawl requests |
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Web Crawler Manager CLI — manage crawl results and crawler selection.
|
||||
|
||||
Claude performs actual crawling via Firecrawl MCP. This script handles:
|
||||
- Storing crawl results metadata
|
||||
- Selecting the optimal crawler backend
|
||||
- Generating crawl result manifests
|
||||
|
||||
Usage:
|
||||
python crawl_mgr.py store-result --manifest manifest.json --raw-dir ~/reference-library/raw/
|
||||
python crawl_mgr.py select-crawler --url "https://docs.anthropic.com"
|
||||
python crawl_mgr.py list-crawls [--status completed]
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.manifest import create_crawl_result, write_crawl_result, read_manifest
|
||||
from refcurator.utils import count_tokens
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
# Known SPA frameworks and their indicators
|
||||
SPA_INDICATORS = {
|
||||
"react": ["react", "next.js", "nextjs", "gatsby", "remix"],
|
||||
"vue": ["vue", "nuxt"],
|
||||
"angular": ["angular"],
|
||||
"svelte": ["svelte", "sveltekit"],
|
||||
}
|
||||
|
||||
# Known static site generators
|
||||
STATIC_INDICATORS = ["hugo", "jekyll", "mkdocs", "docusaurus", "gitbook", "sphinx",
|
||||
"readthedocs", "vuepress", "docsify"]
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Web Crawler Manager — manage crawl results and backend selection."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("store-result")
|
||||
@click.option("--raw-dir", required=True, type=click.Path(exists=True),
|
||||
help="Directory containing crawled raw files")
|
||||
@click.option("--crawler", default="firecrawl",
|
||||
type=click.Choice(["firecrawl", "nodejs", "aiohttp", "scrapy"]))
|
||||
@click.option("--source-id", type=int, help="Source ID to associate documents with")
|
||||
@click.option("--output", type=click.Path(), help="Output crawl result JSON path")
|
||||
def store_result(raw_dir, crawler, source_id, output):
|
||||
"""Store crawl results from raw files into the repository.
|
||||
|
||||
Scans raw_dir for .md files, creates document records, and generates a crawl result manifest.
|
||||
"""
|
||||
raw_path = Path(raw_dir)
|
||||
md_files = sorted(raw_path.glob("**/*.md"))
|
||||
|
||||
if not md_files:
|
||||
console.print(f"[yellow]No .md files found in {raw_dir}[/yellow]")
|
||||
sys.exit(0)
|
||||
|
||||
entries = []
|
||||
stored = 0
|
||||
|
||||
with db_session() as db:
|
||||
for md_file in md_files:
|
||||
content = md_file.read_text(errors="replace")
|
||||
content_size = md_file.stat().st_size
|
||||
|
||||
# Extract title from first heading or filename
|
||||
title = _extract_title(content, md_file.stem)
|
||||
|
||||
# Extract URL from frontmatter if available
|
||||
url = _extract_url_from_content(content)
|
||||
|
||||
entry = {
|
||||
"url": url or f"file://{md_file.resolve()}",
|
||||
"title": title,
|
||||
"raw_path": str(md_file.resolve()),
|
||||
"content_size": content_size,
|
||||
"status": "completed",
|
||||
}
|
||||
entries.append(entry)
|
||||
|
||||
if source_id:
|
||||
db.insert_returning_id(
|
||||
"""INSERT INTO documents
|
||||
(source_id, title, url, doc_type, crawl_date,
|
||||
crawl_method, crawl_status, raw_content_path, raw_content_size)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(source_id, title, url, "markdown", datetime.now().isoformat(),
|
||||
crawler, "completed", str(md_file.resolve()), content_size),
|
||||
)
|
||||
stored += 1
|
||||
|
||||
result = create_crawl_result(entries, crawler)
|
||||
|
||||
if output:
|
||||
write_crawl_result(result, Path(output))
|
||||
console.print(f"[green]Crawl result written to {output}[/green]")
|
||||
|
||||
console.print(
|
||||
f"[green]Processed {len(entries)} files[/green]"
|
||||
+ (f", stored {stored} documents" if stored else "")
|
||||
)
|
||||
click.echo(result.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@cli.command("select-crawler")
|
||||
@click.option("--url", required=True, help="Target URL to analyze")
|
||||
@click.option("--page-count", type=int, help="Expected page count")
|
||||
@click.option("--json-output", is_flag=True, help="Output as JSON")
|
||||
def select_crawler(url, page_count, json_output):
|
||||
"""Recommend the best crawler backend for a URL.
|
||||
|
||||
Analyzes URL patterns and site characteristics to suggest:
|
||||
- firecrawl: SPAs, JS-rendered content
|
||||
- nodejs: Small static docs sites (<=50 pages)
|
||||
- aiohttp: Medium technical docs (<=200 pages)
|
||||
- scrapy: Large enterprise sites (>200 pages)
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.netloc.lower()
|
||||
path = parsed.path.lower()
|
||||
|
||||
recommendation = "firecrawl" # Default
|
||||
reason = "Default recommendation for general crawling"
|
||||
confidence = 0.7
|
||||
|
||||
# Check for SPA indicators in domain/path
|
||||
url_lower = url.lower()
|
||||
for framework, indicators in SPA_INDICATORS.items():
|
||||
if any(ind in url_lower or ind in domain for ind in indicators):
|
||||
recommendation = "firecrawl"
|
||||
reason = f"Detected {framework} SPA framework — needs JS rendering"
|
||||
confidence = 0.9
|
||||
break
|
||||
|
||||
# Check for static site indicators
|
||||
for gen in STATIC_INDICATORS:
|
||||
if gen in url_lower or gen in domain:
|
||||
if page_count and page_count <= 50:
|
||||
recommendation = "nodejs"
|
||||
reason = f"Static site ({gen}), small page count — lightweight crawler sufficient"
|
||||
else:
|
||||
recommendation = "aiohttp"
|
||||
reason = f"Static site ({gen}), moderate size — async crawler recommended"
|
||||
confidence = 0.85
|
||||
break
|
||||
|
||||
# Page count override (takes precedence over default, not over SPA detection)
|
||||
spa_detected = confidence >= 0.9 and recommendation == "firecrawl"
|
||||
if page_count and not spa_detected:
|
||||
if page_count > 200:
|
||||
recommendation = "scrapy"
|
||||
reason = f"Large site ({page_count} pages) — enterprise crawler needed"
|
||||
confidence = 0.9
|
||||
elif page_count <= 50:
|
||||
recommendation = "nodejs"
|
||||
reason = f"Small site ({page_count} pages) — lightweight crawler sufficient"
|
||||
confidence = 0.8
|
||||
|
||||
# Known documentation platforms → firecrawl
|
||||
doc_platforms = ["docs.", "developer.", "api.", "reference."]
|
||||
if any(domain.startswith(p) for p in doc_platforms):
|
||||
if recommendation == "firecrawl":
|
||||
reason = "Documentation platform — Firecrawl handles dynamic docs well"
|
||||
confidence = 0.85
|
||||
|
||||
result = {
|
||||
"url": url,
|
||||
"recommendation": recommendation,
|
||||
"reason": reason,
|
||||
"confidence": confidence,
|
||||
"alternatives": _get_alternatives(recommendation),
|
||||
}
|
||||
|
||||
if json_output:
|
||||
click.echo(json.dumps(result, indent=2))
|
||||
else:
|
||||
console.print(f"\n[bold]Crawler Recommendation for:[/bold] {url}")
|
||||
console.print(f" [green]Recommended:[/green] {recommendation}")
|
||||
console.print(f" [dim]Reason:[/dim] {reason}")
|
||||
console.print(f" [dim]Confidence:[/dim] {confidence:.0%}")
|
||||
if result["alternatives"]:
|
||||
console.print(f" [dim]Alternatives:[/dim] {', '.join(result['alternatives'])}")
|
||||
|
||||
|
||||
@cli.command("list-crawls")
|
||||
@click.option("--status", type=click.Choice(["pending", "completed", "failed", "stale"]),
|
||||
help="Filter by crawl status")
|
||||
@click.option("--limit", default=20, type=int, help="Max results")
|
||||
def list_crawls(status, limit):
|
||||
"""List recent crawl records."""
|
||||
with db_session() as db:
|
||||
if status:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date
|
||||
FROM documents WHERE crawl_status = %s
|
||||
ORDER BY crawl_date DESC LIMIT %s""",
|
||||
(status, limit),
|
||||
)
|
||||
else:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, crawl_method, crawl_status, crawl_date
|
||||
FROM documents ORDER BY crawl_date DESC LIMIT %s""",
|
||||
(limit,),
|
||||
)
|
||||
|
||||
table = Table(title="Crawl Records")
|
||||
table.add_column("ID", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("Crawler")
|
||||
table.add_column("Status")
|
||||
table.add_column("Date")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:40],
|
||||
str(r.get("crawl_method", "")),
|
||||
str(r.get("crawl_status", "")),
|
||||
str(r.get("crawl_date", ""))[:10],
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
def _extract_title(content: str, fallback: str) -> str:
|
||||
"""Extract title from markdown content."""
|
||||
for line in content.split("\n")[:10]:
|
||||
line = line.strip()
|
||||
if line.startswith("# ") and not line.startswith("##"):
|
||||
return line[2:].strip()
|
||||
return fallback.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
|
||||
def _extract_url_from_content(content: str) -> str | None:
|
||||
"""Extract source URL from markdown frontmatter or content."""
|
||||
# YAML frontmatter
|
||||
if content.startswith("---"):
|
||||
fm_end = content.find("---", 3)
|
||||
if fm_end > 0:
|
||||
fm = content[3:fm_end]
|
||||
for line in fm.split("\n"):
|
||||
if line.strip().startswith("url:") or line.strip().startswith("source:"):
|
||||
url = line.split(":", 1)[1].strip().strip("'\"")
|
||||
if url.startswith("http"):
|
||||
return url
|
||||
|
||||
# Source: URL pattern in content
|
||||
m = re.search(r"\*\*Source:\*\*\s*(https?://\S+)", content[:500])
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _get_alternatives(primary: str) -> list[str]:
|
||||
"""Get alternative crawler recommendations."""
|
||||
all_crawlers = ["firecrawl", "nodejs", "aiohttp", "scrapy"]
|
||||
return [c for c in all_crawlers if c != primary][:2]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -1,46 +1,42 @@
|
||||
# Content Repository
|
||||
|
||||
MySQL storage management for the reference library. Handles document storage, version control, deduplication, and retrieval.
|
||||
MySQL storage management for the reference library. Handles document storage, version control, deduplication, and retrieval. Supports MySQL primary backend with JSON file fallback.
|
||||
|
||||
## Trigger Keywords
|
||||
"store content", "save to database", "check duplicates", "version tracking", "document retrieval", "reference library DB"
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- MySQL 8.0+ with utf8mb4 charset
|
||||
- Config file at `~/.config/reference-curator/db_config.yaml`
|
||||
- Database `reference_library` initialized
|
||||
- `refcurator` package installed (`uv pip install -e shared/lib/`)
|
||||
- MySQL 8.0+ (optional — falls back to JSON file storage)
|
||||
- Config at `~/.config/reference-curator/db_config.yaml` (optional)
|
||||
|
||||
## Database Setup
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Initialize database
|
||||
mysql -u root -p < references/schema.sql
|
||||
|
||||
# Verify tables
|
||||
mysql -u root -p reference_library -e "SHOW TABLES;"
|
||||
```
|
||||
|
||||
## Core Scripts
|
||||
|
||||
### Store Document
|
||||
```bash
|
||||
python scripts/store_document.py \
|
||||
--source-id 1 \
|
||||
--title "Prompt Engineering Guide" \
|
||||
# Store a document
|
||||
uv run python scripts/repo.py store \
|
||||
--source-id 1 --title "Prompt Engineering Guide" \
|
||||
--url "https://docs.anthropic.com/..." \
|
||||
--doc-type webpage \
|
||||
--raw-path ~/reference-library/raw/2025/01/abc123.md
|
||||
```
|
||||
--doc-type webpage --raw-path ~/reference-library/raw/abc123.md
|
||||
|
||||
### Check Duplicate
|
||||
```bash
|
||||
python scripts/check_duplicate.py --url "https://docs.anthropic.com/..."
|
||||
```
|
||||
# Check for duplicates
|
||||
uv run python scripts/repo.py check-dup --url "https://docs.anthropic.com/..."
|
||||
|
||||
### Query by Topic
|
||||
```bash
|
||||
python scripts/query_topic.py --topic-slug prompt-engineering --min-quality 0.80
|
||||
# Query by topic
|
||||
uv run python scripts/repo.py query-topic --topic-slug prompt-engineering --min-quality 0.80
|
||||
|
||||
# Get repository stats
|
||||
uv run python scripts/repo.py stats
|
||||
|
||||
# Find stale documents (older than 30 days)
|
||||
uv run python scripts/repo.py find-stale --days 30
|
||||
|
||||
# Get pending reviews
|
||||
uv run python scripts/repo.py pending-reviews --output pending.json
|
||||
|
||||
# Get export-ready content
|
||||
uv run python scripts/repo.py export-ready --min-score 0.85
|
||||
```
|
||||
|
||||
## Table Quick Reference
|
||||
@@ -61,31 +57,17 @@ python scripts/query_topic.py --topic-slug prompt-engineering --min-quality 0.80
|
||||
|
||||
**review_status:** `pending` → `in_review` → `approved` | `needs_refactor` | `rejected`
|
||||
|
||||
## Common Queries
|
||||
|
||||
### Find Stale Documents
|
||||
```bash
|
||||
python scripts/find_stale.py --output stale_docs.json
|
||||
```
|
||||
|
||||
### Get Pending Reviews
|
||||
```bash
|
||||
python scripts/pending_reviews.py --output pending.json
|
||||
```
|
||||
|
||||
### Export-Ready Content
|
||||
```bash
|
||||
python scripts/export_ready.py --min-score 0.85 --output ready.json
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/store_document.py` - Store new document
|
||||
- `scripts/check_duplicate.py` - URL deduplication
|
||||
- `scripts/query_topic.py` - Query by topic
|
||||
- `scripts/find_stale.py` - Find stale documents
|
||||
- `scripts/pending_reviews.py` - Get pending reviews
|
||||
- `scripts/db_utils.py` - Database connection utilities
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `repo.py store` | Store a new document |
|
||||
| `repo.py check-dup` | URL deduplication check |
|
||||
| `repo.py query-topic` | Query documents by topic |
|
||||
| `repo.py find-stale` | Find stale documents |
|
||||
| `repo.py pending-reviews` | Get pending reviews |
|
||||
| `repo.py export-ready` | Get approved content ready for export |
|
||||
| `repo.py stats` | Show repository statistics |
|
||||
|
||||
## Integration
|
||||
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Content Repository CLI — CRUD operations for the reference library.
|
||||
|
||||
Usage:
|
||||
python repo.py store --source-id 1 --title "Doc" --url "https://..." --doc-type webpage --raw-path /path/to/file
|
||||
python repo.py check-dup --url "https://..."
|
||||
python repo.py query-topic --topic-slug prompt-engineering [--min-quality 0.80]
|
||||
python repo.py find-stale [--output stale.json]
|
||||
python repo.py pending-reviews [--output pending.json]
|
||||
python repo.py export-ready [--min-score 0.85] [--output ready.json]
|
||||
python repo.py stats
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.utils import url_hash, normalize_url
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Content Repository — manage documents in the reference library."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--source-id", required=True, type=int, help="Source ID from sources table")
|
||||
@click.option("--title", required=True, help="Document title")
|
||||
@click.option("--url", required=True, help="Document URL")
|
||||
@click.option("--doc-type", required=True,
|
||||
type=click.Choice(["webpage", "pdf", "markdown", "api_spec", "code_sample"]))
|
||||
@click.option("--raw-path", required=True, type=click.Path(), help="Path to raw content file")
|
||||
@click.option("--crawl-method", default="firecrawl",
|
||||
type=click.Choice(["firecrawl", "scrapy", "aiohttp", "nodejs", "manual", "api"]))
|
||||
@click.option("--language", default="en", type=click.Choice(["en", "ko", "mixed"]))
|
||||
def store(source_id, title, url, doc_type, raw_path, crawl_method, language):
|
||||
"""Store a new document in the repository."""
|
||||
raw = Path(raw_path)
|
||||
if not raw.is_file():
|
||||
console.print(f"[red]Error:[/red] Raw file not found: {raw_path}")
|
||||
sys.exit(1)
|
||||
|
||||
content_size = raw.stat().st_size
|
||||
|
||||
with db_session() as db:
|
||||
# Check for duplicate
|
||||
existing = db.fetch_one(
|
||||
"SELECT doc_id, title FROM documents WHERE url_hash = %s",
|
||||
(url_hash(url),),
|
||||
)
|
||||
if existing:
|
||||
console.print(
|
||||
f"[yellow]Duplicate:[/yellow] URL already stored as doc_id={existing['doc_id']} "
|
||||
f"({existing['title']})"
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
doc_id = db.insert_returning_id(
|
||||
"""INSERT INTO documents
|
||||
(source_id, title, url, doc_type, language, crawl_date,
|
||||
crawl_method, crawl_status, raw_content_path, raw_content_size)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(source_id, title, url, doc_type, language, datetime.now().isoformat(),
|
||||
crawl_method, "completed", str(raw.resolve()), content_size),
|
||||
)
|
||||
|
||||
console.print(f"[green]Stored:[/green] doc_id={doc_id} — {title}")
|
||||
click.echo(json.dumps({"doc_id": doc_id, "title": title, "url": url}))
|
||||
|
||||
|
||||
@cli.command("check-dup")
|
||||
@click.option("--url", required=True, help="URL to check for duplicates")
|
||||
def check_dup(url):
|
||||
"""Check if a URL already exists in the repository."""
|
||||
h = url_hash(url)
|
||||
|
||||
with db_session() as db:
|
||||
existing = db.fetch_one(
|
||||
"SELECT doc_id, title, url, crawl_status FROM documents WHERE url_hash = %s",
|
||||
(h,),
|
||||
)
|
||||
|
||||
if existing:
|
||||
console.print(f"[yellow]Duplicate found:[/yellow] doc_id={existing['doc_id']}")
|
||||
click.echo(json.dumps(existing, default=str))
|
||||
else:
|
||||
console.print("[green]No duplicate found.[/green]")
|
||||
click.echo(json.dumps({"duplicate": False, "url": url}))
|
||||
|
||||
|
||||
@cli.command("query-topic")
|
||||
@click.option("--topic-slug", required=True, help="Topic slug to query")
|
||||
@click.option("--min-quality", default=0.0, type=float, help="Minimum quality score")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def query_topic(topic_slug, min_quality, output):
|
||||
"""Query documents by topic."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url, d.crawl_status,
|
||||
dt.relevance_score, t.topic_name
|
||||
FROM documents d
|
||||
JOIN document_topics dt ON d.doc_id = dt.doc_id
|
||||
JOIN topics t ON dt.topic_id = t.topic_id
|
||||
WHERE t.topic_slug = %s
|
||||
ORDER BY dt.relevance_score DESC""",
|
||||
(topic_slug,),
|
||||
)
|
||||
|
||||
if min_quality > 0:
|
||||
# Filter by review score if available
|
||||
rows = [r for r in rows if r.get("relevance_score", 0) >= min_quality]
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]Wrote {len(rows)} results to {output}[/green]")
|
||||
else:
|
||||
_print_doc_table(rows, f"Topic: {topic_slug}")
|
||||
|
||||
|
||||
@cli.command("find-stale")
|
||||
@click.option("--days", default=30, type=int, help="Documents older than N days")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def find_stale(days, output):
|
||||
"""Find documents that may be outdated."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, crawl_date, crawl_status
|
||||
FROM documents
|
||||
WHERE crawl_status = %s
|
||||
ORDER BY crawl_date ASC""",
|
||||
("completed",),
|
||||
)
|
||||
|
||||
# Filter by age
|
||||
cutoff = datetime.now()
|
||||
stale = []
|
||||
for r in rows:
|
||||
crawl_date = r.get("crawl_date")
|
||||
if crawl_date:
|
||||
if isinstance(crawl_date, str):
|
||||
crawl_date = datetime.fromisoformat(crawl_date)
|
||||
age_days = (cutoff - crawl_date).days
|
||||
if age_days >= days:
|
||||
r["age_days"] = age_days
|
||||
stale.append(r)
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(stale, indent=2, default=str))
|
||||
console.print(f"[green]Found {len(stale)} stale documents, wrote to {output}[/green]")
|
||||
else:
|
||||
_print_doc_table(stale, f"Stale documents (>{days} days)")
|
||||
|
||||
|
||||
@cli.command("pending-reviews")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def pending_reviews(output):
|
||||
"""Get documents pending quality review."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT dc.distill_id, d.doc_id, d.title, d.url,
|
||||
dc.token_count_distilled, dc.distill_date
|
||||
FROM distilled_content dc
|
||||
JOIN documents d ON dc.doc_id = d.doc_id
|
||||
WHERE dc.review_status = %s
|
||||
ORDER BY dc.distill_date ASC""",
|
||||
("pending",),
|
||||
)
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]{len(rows)} pending reviews, wrote to {output}[/green]")
|
||||
else:
|
||||
table = Table(title="Pending Reviews")
|
||||
table.add_column("distill_id", style="cyan")
|
||||
table.add_column("doc_id")
|
||||
table.add_column("Title")
|
||||
table.add_column("Tokens", justify="right")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("distill_id", "")),
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:50],
|
||||
str(r.get("token_count_distilled", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
@cli.command("export-ready")
|
||||
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def export_ready(min_score, output):
|
||||
"""Get documents approved and ready for export."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url,
|
||||
dc.structured_content, dc.token_count_distilled,
|
||||
rl.quality_score, rl.decision
|
||||
FROM documents d
|
||||
JOIN distilled_content dc ON d.doc_id = dc.doc_id
|
||||
JOIN review_logs rl ON dc.distill_id = rl.distill_id
|
||||
WHERE dc.review_status = %s
|
||||
AND rl.decision = %s
|
||||
AND rl.review_id = (
|
||||
SELECT MAX(rl2.review_id)
|
||||
FROM review_logs rl2
|
||||
WHERE rl2.distill_id = dc.distill_id
|
||||
)
|
||||
ORDER BY rl.quality_score DESC""",
|
||||
("approved", "approve"),
|
||||
)
|
||||
|
||||
# Filter by min score
|
||||
rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]{len(rows)} export-ready documents, wrote to {output}[/green]")
|
||||
else:
|
||||
table = Table(title=f"Export-Ready (score >= {min_score})")
|
||||
table.add_column("doc_id", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("Score", justify="right")
|
||||
table.add_column("Tokens", justify="right")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:50],
|
||||
f"{r.get('quality_score', 0):.2f}",
|
||||
str(r.get("token_count_distilled", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def stats():
|
||||
"""Show repository statistics."""
|
||||
with db_session() as db:
|
||||
doc_count = db.fetch_one("SELECT COUNT(*) as cnt FROM documents") or {"cnt": 0}
|
||||
source_count = db.fetch_one("SELECT COUNT(*) as cnt FROM sources") or {"cnt": 0}
|
||||
|
||||
status_rows = db.fetch_all(
|
||||
"""SELECT crawl_status, COUNT(*) as cnt
|
||||
FROM documents
|
||||
GROUP BY crawl_status"""
|
||||
)
|
||||
|
||||
review_rows = db.fetch_all(
|
||||
"""SELECT review_status, COUNT(*) as cnt
|
||||
FROM distilled_content
|
||||
GROUP BY review_status"""
|
||||
)
|
||||
|
||||
topic_rows = db.fetch_all(
|
||||
"""SELECT t.topic_name, COUNT(dt.doc_id) as cnt
|
||||
FROM topics t
|
||||
LEFT JOIN document_topics dt ON t.topic_id = dt.topic_id
|
||||
GROUP BY t.topic_id
|
||||
ORDER BY cnt DESC"""
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(f"[bold]Reference Library Statistics[/bold]")
|
||||
console.print(f" Sources: {source_count['cnt']}")
|
||||
console.print(f" Documents: {doc_count['cnt']}")
|
||||
console.print()
|
||||
|
||||
if status_rows:
|
||||
table = Table(title="Documents by Crawl Status")
|
||||
table.add_column("Status")
|
||||
table.add_column("Count", justify="right")
|
||||
for r in status_rows:
|
||||
table.add_row(str(r["crawl_status"]), str(r["cnt"]))
|
||||
console.print(table)
|
||||
|
||||
if review_rows:
|
||||
table = Table(title="Distilled Content by Review Status")
|
||||
table.add_column("Status")
|
||||
table.add_column("Count", justify="right")
|
||||
for r in review_rows:
|
||||
table.add_row(str(r["review_status"]), str(r["cnt"]))
|
||||
console.print(table)
|
||||
|
||||
if topic_rows:
|
||||
table = Table(title="Documents by Topic")
|
||||
table.add_column("Topic")
|
||||
table.add_column("Documents", justify="right")
|
||||
for r in topic_rows:
|
||||
table.add_row(str(r["topic_name"]), str(r["cnt"]))
|
||||
console.print(table)
|
||||
|
||||
result = {
|
||||
"sources": source_count["cnt"],
|
||||
"documents": doc_count["cnt"],
|
||||
"by_status": {r["crawl_status"]: r["cnt"] for r in status_rows},
|
||||
"by_review": {r["review_status"]: r["cnt"] for r in review_rows},
|
||||
"by_topic": {r["topic_name"]: r["cnt"] for r in topic_rows},
|
||||
}
|
||||
click.echo(json.dumps(result, default=str))
|
||||
|
||||
|
||||
def _print_doc_table(rows: list[dict], title: str):
|
||||
"""Print a table of documents."""
|
||||
table = Table(title=title)
|
||||
table.add_column("doc_id", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("URL")
|
||||
table.add_column("Status")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:40],
|
||||
str(r.get("url", ""))[:50],
|
||||
str(r.get("crawl_status", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -1,44 +1,58 @@
|
||||
# Content Distiller
|
||||
|
||||
Analyzes and distills raw crawled content into concise reference materials. Extracts key concepts, code snippets, and creates structured summaries.
|
||||
Analyzes and distills raw crawled content into concise reference materials. Claude performs the actual distillation (summarization, key concept extraction). Scripts handle data loading and storage.
|
||||
|
||||
## Trigger Keywords
|
||||
"distill content", "summarize document", "extract key concepts", "process raw content", "create reference summary"
|
||||
|
||||
## Goals
|
||||
|
||||
1. **Compress** - Reduce token count while preserving essential information
|
||||
2. **Structure** - Organize content for easy retrieval
|
||||
3. **Extract** - Pull out code snippets, key concepts, patterns
|
||||
4. **Annotate** - Add metadata for searchability
|
||||
1. **Compress** — Reduce token count while preserving essential information
|
||||
2. **Structure** — Organize content for easy retrieval
|
||||
3. **Extract** — Pull out code snippets, key concepts, patterns
|
||||
4. **Annotate** — Add metadata for searchability
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Load Raw Content
|
||||
### Step 1: Load Pending Documents
|
||||
```bash
|
||||
python scripts/load_pending.py --output pending_docs.json
|
||||
uv run python scripts/distiller.py load-pending --output pending.json
|
||||
```
|
||||
|
||||
### Step 2: Analyze Content Structure
|
||||
Identify document characteristics:
|
||||
- Has code blocks?
|
||||
- Has headers?
|
||||
- Has tables?
|
||||
- Estimated tokens?
|
||||
|
||||
### Step 3: Extract Key Components
|
||||
```bash
|
||||
python scripts/extract_components.py --doc-id 123 --output components.json
|
||||
```
|
||||
|
||||
Extracts:
|
||||
- Code snippets with language tags
|
||||
- Key concepts and definitions
|
||||
- Best practices
|
||||
### Step 2: Analyze and Distill (Claude)
|
||||
For each pending document, Claude reads the raw content and creates:
|
||||
- Executive summary (2-3 sentences)
|
||||
- Key concepts with definitions
|
||||
- Techniques and patterns
|
||||
- Code examples
|
||||
- Best practices
|
||||
|
||||
### Step 3: Store Distilled Content
|
||||
```bash
|
||||
uv run python scripts/distiller.py store \
|
||||
--doc-id 123 \
|
||||
--content distilled.md \
|
||||
--summary summary.txt \
|
||||
--concepts concepts.json \
|
||||
--snippets snippets.json \
|
||||
--model claude-opus-4-6
|
||||
```
|
||||
|
||||
### Step 4: Handle Refactor Requests
|
||||
When quality-reviewer returns `refactor`, load context for re-distillation:
|
||||
```bash
|
||||
uv run python scripts/distiller.py refactor --distill-id 456 --output context.json
|
||||
```
|
||||
|
||||
This outputs a context bundle with the current distilled content, raw source, and all review feedback.
|
||||
|
||||
### Step 5: View Distilled Content
|
||||
```bash
|
||||
uv run python scripts/distiller.py show --distill-id 456
|
||||
```
|
||||
|
||||
## Distilled Output Template
|
||||
|
||||
### Step 4: Create Structured Summary
|
||||
Output template:
|
||||
```markdown
|
||||
# {title}
|
||||
|
||||
@@ -62,17 +76,6 @@ Output template:
|
||||
{actionable recommendations}
|
||||
```
|
||||
|
||||
### Step 5: Optimize for Tokens
|
||||
Target: 25-35% of original token count
|
||||
```bash
|
||||
python scripts/optimize_content.py --doc-id 123 --target-ratio 0.30
|
||||
```
|
||||
|
||||
### Step 6: Store Distilled Content
|
||||
```bash
|
||||
python scripts/store_distilled.py --doc-id 123 --content distilled.md
|
||||
```
|
||||
|
||||
## Quality Metrics
|
||||
|
||||
| Metric | Target |
|
||||
@@ -82,20 +85,14 @@ python scripts/store_distilled.py --doc-id 123 --content distilled.md
|
||||
| Code Snippet Retention | 100% of relevant examples |
|
||||
| Readability | Clear, scannable structure |
|
||||
|
||||
## Handling Refactor Requests
|
||||
|
||||
When `quality-reviewer` returns `refactor`:
|
||||
```bash
|
||||
python scripts/refactor_content.py --distill-id 456 --instructions "Add more examples"
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/load_pending.py` - Load documents pending distillation
|
||||
- `scripts/extract_components.py` - Extract code, concepts, patterns
|
||||
- `scripts/optimize_content.py` - Token optimization
|
||||
- `scripts/store_distilled.py` - Save to database
|
||||
- `scripts/refactor_content.py` - Handle refactor requests
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `distiller.py load-pending` | Load documents pending distillation |
|
||||
| `distiller.py store` | Save distilled content to DB |
|
||||
| `distiller.py refactor` | Load context for re-distillation |
|
||||
| `distiller.py show` | Show distilled content details |
|
||||
|
||||
## Integration
|
||||
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Content Distiller CLI — manage distillation data I/O.
|
||||
|
||||
Claude performs the actual distillation (summarization, extraction).
|
||||
This script handles loading raw content from the DB and storing distilled output.
|
||||
|
||||
Usage:
|
||||
python distiller.py load-pending [--output pending.json]
|
||||
python distiller.py store --doc-id 123 --content distilled.md [--model claude-opus-4-6]
|
||||
python distiller.py refactor --distill-id 456 [--output context.json]
|
||||
python distiller.py show --distill-id 456
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.utils import count_tokens
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Content Distiller — manage distillation data."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("load-pending")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
@click.option("--limit", default=50, type=int, help="Max documents to load")
|
||||
def load_pending(output, limit):
|
||||
"""Load documents pending distillation.
|
||||
|
||||
Finds documents with crawl_status='completed' that have no distilled content yet.
|
||||
"""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url, d.raw_content_path,
|
||||
d.raw_content_size, d.doc_type,
|
||||
s.source_name, s.credibility_tier
|
||||
FROM documents d
|
||||
JOIN sources s ON d.source_id = s.source_id
|
||||
LEFT JOIN distilled_content dc ON d.doc_id = dc.doc_id
|
||||
WHERE d.crawl_status = %s AND dc.distill_id IS NULL
|
||||
ORDER BY s.credibility_tier ASC, d.crawl_date ASC
|
||||
LIMIT %s""",
|
||||
("completed", limit),
|
||||
)
|
||||
|
||||
# Enrich with raw content preview
|
||||
for row in rows:
|
||||
raw_path = row.get("raw_content_path")
|
||||
if raw_path and Path(raw_path).is_file():
|
||||
content = Path(raw_path).read_text(errors="replace")
|
||||
row["token_count_estimate"] = count_tokens(content)
|
||||
row["content_preview"] = content[:200] + "..." if len(content) > 200 else content
|
||||
else:
|
||||
row["token_count_estimate"] = 0
|
||||
row["content_preview"] = "[file not found]"
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]{len(rows)} pending documents written to {output}[/green]")
|
||||
else:
|
||||
table = Table(title=f"Pending Distillation ({len(rows)} documents)")
|
||||
table.add_column("doc_id", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("Source")
|
||||
table.add_column("Tier")
|
||||
table.add_column("~Tokens", justify="right")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:40],
|
||||
str(r.get("source_name", ""))[:20],
|
||||
str(r.get("credibility_tier", "")),
|
||||
str(r.get("token_count_estimate", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
click.echo(json.dumps({"count": len(rows)}, default=str))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--doc-id", required=True, type=int, help="Document ID to store distilled content for")
|
||||
@click.option("--content", required=True, type=click.Path(exists=True),
|
||||
help="Path to distilled markdown content")
|
||||
@click.option("--summary", type=click.Path(exists=True), help="Path to summary text file")
|
||||
@click.option("--concepts", type=click.Path(exists=True), help="Path to key concepts JSON")
|
||||
@click.option("--snippets", type=click.Path(exists=True), help="Path to code snippets JSON")
|
||||
@click.option("--model", default="claude-opus-4-6", help="Model used for distillation")
|
||||
def store(doc_id, content, summary, concepts, snippets, model):
|
||||
"""Store distilled content for a document."""
|
||||
structured = Path(content).read_text(errors="replace")
|
||||
token_distilled = count_tokens(structured)
|
||||
|
||||
summary_text = Path(summary).read_text() if summary else None
|
||||
concepts_json = json.loads(Path(concepts).read_text()) if concepts else None
|
||||
snippets_json = json.loads(Path(snippets).read_text()) if snippets else None
|
||||
|
||||
with db_session() as db:
|
||||
# Get original token count
|
||||
doc = db.fetch_one(
|
||||
"SELECT raw_content_path, raw_content_size FROM documents WHERE doc_id = %s",
|
||||
(doc_id,),
|
||||
)
|
||||
token_original = 0
|
||||
if doc and doc.get("raw_content_path"):
|
||||
raw_path = Path(doc["raw_content_path"])
|
||||
if raw_path.is_file():
|
||||
token_original = count_tokens(raw_path.read_text(errors="replace"))
|
||||
|
||||
# Check for existing distilled_content row (refactor case)
|
||||
existing = db.fetch_one(
|
||||
"SELECT distill_id, review_status FROM distilled_content WHERE doc_id = %s",
|
||||
(doc_id,),
|
||||
)
|
||||
|
||||
if existing and existing.get("review_status") in ("needs_refactor", "pending"):
|
||||
# Update existing row instead of creating a new one
|
||||
distill_id = existing["distill_id"]
|
||||
db.execute(
|
||||
"""UPDATE distilled_content
|
||||
SET summary = %s, key_concepts = %s, code_snippets = %s,
|
||||
structured_content = %s, token_count_original = %s,
|
||||
token_count_distilled = %s, distill_model = %s,
|
||||
distill_date = %s, review_status = %s
|
||||
WHERE distill_id = %s""",
|
||||
(summary_text,
|
||||
json.dumps(concepts_json) if concepts_json else None,
|
||||
json.dumps(snippets_json) if snippets_json else None,
|
||||
structured, token_original, token_distilled, model,
|
||||
datetime.now().isoformat(), "pending", distill_id),
|
||||
)
|
||||
action = "Updated"
|
||||
else:
|
||||
# First distillation for this document
|
||||
distill_id = db.insert_returning_id(
|
||||
"""INSERT INTO distilled_content
|
||||
(doc_id, summary, key_concepts, code_snippets, structured_content,
|
||||
token_count_original, token_count_distilled, distill_model, distill_date,
|
||||
review_status)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(doc_id, summary_text,
|
||||
json.dumps(concepts_json) if concepts_json else None,
|
||||
json.dumps(snippets_json) if snippets_json else None,
|
||||
structured, token_original, token_distilled, model,
|
||||
datetime.now().isoformat(), "pending"),
|
||||
)
|
||||
action = "Stored"
|
||||
|
||||
ratio = (token_distilled / token_original * 100) if token_original else 0
|
||||
console.print(
|
||||
f"[green]{action}:[/green] distill_id={distill_id} for doc_id={doc_id} "
|
||||
f"({token_original} → {token_distilled} tokens, {ratio:.0f}% compression)"
|
||||
)
|
||||
click.echo(json.dumps({
|
||||
"distill_id": distill_id,
|
||||
"doc_id": doc_id,
|
||||
"token_original": token_original,
|
||||
"token_distilled": token_distilled,
|
||||
"compression_ratio": round(ratio, 2),
|
||||
}))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
|
||||
@click.option("--output", type=click.Path(), help="Output context JSON for re-distillation")
|
||||
def refactor(distill_id, output):
|
||||
"""Load existing distilled content + review feedback for re-distillation.
|
||||
|
||||
Outputs a context bundle that Claude can use to re-distill with improvements.
|
||||
"""
|
||||
with db_session() as db:
|
||||
distilled = db.fetch_one(
|
||||
"""SELECT dc.*, d.title, d.url, d.raw_content_path
|
||||
FROM distilled_content dc
|
||||
JOIN documents d ON dc.doc_id = d.doc_id
|
||||
WHERE dc.distill_id = %s""",
|
||||
(distill_id,),
|
||||
)
|
||||
|
||||
if not distilled:
|
||||
console.print(f"[red]Error:[/red] distill_id={distill_id} not found")
|
||||
sys.exit(1)
|
||||
|
||||
# Get review feedback
|
||||
reviews = db.fetch_all(
|
||||
"""SELECT review_round, quality_score, decision, feedback, refactor_instructions
|
||||
FROM review_logs
|
||||
WHERE distill_id = %s
|
||||
ORDER BY review_round ASC""",
|
||||
(distill_id,),
|
||||
)
|
||||
|
||||
# Load raw content if available
|
||||
raw_content = ""
|
||||
raw_path = distilled.get("raw_content_path")
|
||||
if raw_path and Path(raw_path).is_file():
|
||||
raw_content = Path(raw_path).read_text(errors="replace")
|
||||
|
||||
context = {
|
||||
"distill_id": distill_id,
|
||||
"doc_id": distilled.get("doc_id"),
|
||||
"title": distilled.get("title"),
|
||||
"url": distilled.get("url"),
|
||||
"current_distilled": distilled.get("structured_content"),
|
||||
"raw_content": raw_content,
|
||||
"review_history": [
|
||||
{
|
||||
"round": r.get("review_round"),
|
||||
"score": float(r["quality_score"]) if r.get("quality_score") else None,
|
||||
"decision": r.get("decision"),
|
||||
"feedback": r.get("feedback"),
|
||||
"instructions": r.get("refactor_instructions"),
|
||||
}
|
||||
for r in reviews
|
||||
],
|
||||
}
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(context, indent=2, default=str))
|
||||
console.print(f"[green]Refactor context written to {output}[/green]")
|
||||
else:
|
||||
click.echo(json.dumps(context, indent=2, default=str))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
|
||||
def show(distill_id):
|
||||
"""Show details of a distilled content record."""
|
||||
with db_session() as db:
|
||||
row = db.fetch_one(
|
||||
"""SELECT dc.*, d.title, d.url
|
||||
FROM distilled_content dc
|
||||
JOIN documents d ON dc.doc_id = d.doc_id
|
||||
WHERE dc.distill_id = %s""",
|
||||
(distill_id,),
|
||||
)
|
||||
|
||||
if not row:
|
||||
console.print(f"[red]Not found:[/red] distill_id={distill_id}")
|
||||
sys.exit(1)
|
||||
|
||||
console.print(f"\n[bold]Distilled Content #{distill_id}[/bold]")
|
||||
console.print(f" Document: {row.get('title')} (doc_id={row.get('doc_id')})")
|
||||
console.print(f" URL: {row.get('url')}")
|
||||
console.print(f" Review Status: {row.get('review_status')}")
|
||||
console.print(f" Model: {row.get('distill_model')}")
|
||||
console.print(f" Tokens: {row.get('token_count_original')} → {row.get('token_count_distilled')}")
|
||||
if row.get("summary"):
|
||||
console.print(f"\n[bold]Summary:[/bold]\n{row['summary'][:500]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -1,103 +1,93 @@
|
||||
# Quality Reviewer
|
||||
|
||||
QA loop for reference library content. Scores distilled materials, routes decisions, and provides actionable feedback.
|
||||
Pre-distillation quality gate using Gemini CLI as an independent evaluator. Assesses raw crawled content before distillation to filter out low-quality sources early. Also supports manual scoring and routing for edge cases.
|
||||
|
||||
## Trigger Keywords
|
||||
"review content", "quality check", "QA review", "assess distilled content", "check reference quality"
|
||||
"review content", "quality check", "QA review", "evaluate sources", "check reference quality"
|
||||
|
||||
## Decision Flow
|
||||
## Primary Flow: Gemini Pre-Distillation Gate
|
||||
|
||||
```
|
||||
[Distilled Content]
|
||||
[Raw Crawled Content]
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Score Criteria │ → accuracy, completeness, clarity, PE quality, usability
|
||||
└─────────────────┘
|
||||
┌────────────────────┐
|
||||
│ Gemini CLI Eval │ → relevance, authority, completeness, freshness, distill_value
|
||||
└────────────────────┘
|
||||
│
|
||||
├── ≥ 0.85 → APPROVE → markdown-exporter
|
||||
├── 0.60-0.84 → REFACTOR → content-distiller
|
||||
├── 0.40-0.59 → DEEP_RESEARCH → web-crawler
|
||||
└── < 0.40 → REJECT → archive
|
||||
├── ≥ 0.75 → APPROVE → proceed to distillation
|
||||
├── 0.50-0.74 → DEEP_RESEARCH → re-crawl for better sources
|
||||
└── < 0.50 → REJECT → skip distillation entirely
|
||||
```
|
||||
|
||||
## Scoring Criteria
|
||||
## Evaluation Criteria (Gemini)
|
||||
|
||||
| Criterion | Weight | Checks |
|
||||
|-----------|--------|--------|
|
||||
| **Accuracy** | 0.25 | Factual correctness, up-to-date, attribution |
|
||||
| **Completeness** | 0.20 | Key concepts, examples, edge cases |
|
||||
| **Clarity** | 0.20 | Structure, concise language, logical flow |
|
||||
| **PE Quality** | 0.25 | Techniques, before/after, explains why |
|
||||
| **Usability** | 0.10 | Easy reference, searchable, appropriate length |
|
||||
| Criterion | Weight | What It Checks |
|
||||
|-----------|--------|----------------|
|
||||
| **Relevance** | 0.25 | Does content match the curation topic? |
|
||||
| **Authority** | 0.25 | Official docs / research paper, or blog spam? |
|
||||
| **Completeness** | 0.20 | Full article, or nav fragment / error page / stub? |
|
||||
| **Freshness** | 0.15 | Up-to-date or outdated information? |
|
||||
| **Distill Value** | 0.15 | Unique info worth summarizing, or redundant? |
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Load Pending Reviews
|
||||
### Step 1: Evaluate Single Document
|
||||
```bash
|
||||
python scripts/load_pending_reviews.py --output pending.json
|
||||
uv run python scripts/reviewer.py gemini-evaluate --doc-id 123 --topic "prompt engineering"
|
||||
|
||||
# With auto-logging of decision:
|
||||
uv run python scripts/reviewer.py gemini-evaluate --doc-id 123 --topic "prompt engineering" --auto-approve
|
||||
```
|
||||
|
||||
### Step 2: Score Content
|
||||
### Step 2: Batch Evaluate All Pending
|
||||
```bash
|
||||
python scripts/score_content.py --distill-id 123 --output assessment.json
|
||||
uv run python scripts/reviewer.py gemini-evaluate-pending --topic "prompt engineering" --auto-approve --limit 20
|
||||
```
|
||||
|
||||
### Step 3: Calculate Final Score
|
||||
### Step 3: Manual Review (Edge Cases)
|
||||
For documents where Gemini evaluation fails or needs human judgment:
|
||||
```bash
|
||||
python scripts/calculate_score.py --assessment assessment.json
|
||||
# Calculate score from manual assessment
|
||||
uv run python scripts/reviewer.py calculate-score --assessment assessment.json
|
||||
|
||||
# Route based on score
|
||||
uv run python scripts/reviewer.py route --score 0.78
|
||||
|
||||
# Log review decision
|
||||
uv run python scripts/reviewer.py log-review \
|
||||
--distill-id 123 --decision approve --score 0.85 \
|
||||
--feedback "Manually verified"
|
||||
```
|
||||
|
||||
### Step 4: Route Decision
|
||||
### Step 4: Review History
|
||||
```bash
|
||||
python scripts/route_decision.py --distill-id 123 --score 0.78
|
||||
uv run python scripts/reviewer.py history --distill-id 123
|
||||
```
|
||||
|
||||
Outputs:
|
||||
- `approve` → Ready for export
|
||||
- `refactor` → Return to distiller with instructions
|
||||
- `deep_research` → Need more sources (queries generated)
|
||||
- `reject` → Archive with reason
|
||||
## Prerequisites
|
||||
|
||||
### Step 5: Log Review
|
||||
```bash
|
||||
python scripts/log_review.py --distill-id 123 --decision refactor --instructions "Add more examples"
|
||||
```
|
||||
|
||||
## PE Quality Checklist
|
||||
|
||||
When scoring `prompt_engineering_quality`:
|
||||
- [ ] Demonstrates specific techniques (CoT, few-shot, etc.)
|
||||
- [ ] Shows before/after examples
|
||||
- [ ] Explains *why* techniques work
|
||||
- [ ] Provides actionable patterns
|
||||
- [ ] Includes edge cases and failure modes
|
||||
- [ ] References authoritative sources
|
||||
|
||||
## Auto-Approve Rules
|
||||
|
||||
Tier 1 sources with score ≥ 0.80 may auto-approve:
|
||||
```yaml
|
||||
# In config
|
||||
quality:
|
||||
auto_approve_tier1_sources: true
|
||||
auto_approve_min_score: 0.80
|
||||
```
|
||||
- Gemini CLI: `npm install -g @google/gemini-cli`
|
||||
- Google auth: `gemini` (run once interactively to authenticate)
|
||||
- `refcurator` package installed
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/load_pending_reviews.py` - Get pending reviews
|
||||
- `scripts/score_content.py` - Multi-criteria scoring
|
||||
- `scripts/calculate_score.py` - Weighted average calculation
|
||||
- `scripts/route_decision.py` - Decision routing logic
|
||||
- `scripts/log_review.py` - Log review to database
|
||||
- `scripts/generate_feedback.py` - Generate refactor instructions
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `reviewer.py gemini-evaluate` | Evaluate single doc via Gemini CLI |
|
||||
| `reviewer.py gemini-evaluate-pending` | Batch evaluate all pending docs |
|
||||
| `reviewer.py calculate-score` | Manual weighted score calculation |
|
||||
| `reviewer.py route` | Decision routing from score |
|
||||
| `reviewer.py log-review` | Log review decision to DB |
|
||||
| `reviewer.py load-pending` | Get pending reviews |
|
||||
| `reviewer.py history` | Show review history |
|
||||
|
||||
## Integration
|
||||
|
||||
| From | Action | To |
|
||||
|------|--------|-----|
|
||||
| content-distiller | Distilled content | → |
|
||||
| → | APPROVE | markdown-exporter |
|
||||
| → | REFACTOR + instructions | content-distiller |
|
||||
| → | DEEP_RESEARCH + queries | web-crawler-orchestrator |
|
||||
| content-repository (raw docs) | Gemini evaluation | → |
|
||||
| → | APPROVE | content-distiller |
|
||||
| → | DEEP_RESEARCH | web-crawler-orchestrator |
|
||||
| → | REJECT | archive (skip distillation) |
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quality Reviewer CLI — scoring, routing, and review logging.
|
||||
|
||||
Claude performs qualitative assessment. This script handles:
|
||||
- Loading pending reviews
|
||||
- Calculating weighted quality scores
|
||||
- Routing decisions based on thresholds
|
||||
- Logging review decisions to the database
|
||||
|
||||
Usage:
|
||||
python reviewer.py load-pending [--output pending.json]
|
||||
python reviewer.py calculate-score --assessment assessment.json
|
||||
python reviewer.py route --score 0.78
|
||||
python reviewer.py log-review --distill-id 123 --decision refactor --score 0.78 [--feedback "..."]
|
||||
python reviewer.py history --distill-id 123
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.models import QAAssessment
|
||||
|
||||
console = Console()
|
||||
|
||||
# Thresholds from pipeline config
|
||||
APPROVE_THRESHOLD = 0.85
|
||||
REFACTOR_THRESHOLD = 0.60
|
||||
DEEP_RESEARCH_THRESHOLD = 0.40
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Quality Reviewer — scoring, routing, and review logging."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("load-pending")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
@click.option("--limit", default=50, type=int, help="Max results")
|
||||
def load_pending(output, limit):
|
||||
"""Load distilled content pending quality review."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT dc.distill_id, d.doc_id, d.title, d.url,
|
||||
dc.token_count_distilled, dc.distill_date,
|
||||
dc.summary, s.credibility_tier, s.vendor
|
||||
FROM distilled_content dc
|
||||
JOIN documents d ON dc.doc_id = d.doc_id
|
||||
JOIN sources s ON d.source_id = s.source_id
|
||||
WHERE dc.review_status = %s
|
||||
ORDER BY s.credibility_tier ASC, dc.distill_date ASC
|
||||
LIMIT %s""",
|
||||
("pending", limit),
|
||||
)
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]{len(rows)} pending reviews written to {output}[/green]")
|
||||
else:
|
||||
table = Table(title=f"Pending Reviews ({len(rows)})")
|
||||
table.add_column("distill_id", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("Tier")
|
||||
table.add_column("Tokens", justify="right")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("distill_id", "")),
|
||||
str(r.get("title", ""))[:40],
|
||||
str(r.get("credibility_tier", "")),
|
||||
str(r.get("token_count_distilled", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
click.echo(json.dumps({"count": len(rows)}, default=str))
|
||||
|
||||
|
||||
@cli.command("calculate-score")
|
||||
@click.option("--assessment", required=True, type=click.Path(exists=True),
|
||||
help="Assessment JSON file with criteria scores")
|
||||
@click.option("--json-output", is_flag=True, help="Output as JSON only")
|
||||
def calculate_score(assessment, json_output):
|
||||
"""Calculate weighted quality score from assessment criteria.
|
||||
|
||||
Input JSON format:
|
||||
{
|
||||
"accuracy": 0.90,
|
||||
"completeness": 0.85,
|
||||
"clarity": 0.95,
|
||||
"prompt_engineering_quality": 0.88,
|
||||
"usability": 0.82
|
||||
}
|
||||
|
||||
Weights: accuracy 0.25, completeness 0.20, clarity 0.20,
|
||||
prompt_engineering_quality 0.25, usability 0.10
|
||||
"""
|
||||
data = json.loads(Path(assessment).read_text())
|
||||
qa = QAAssessment(**data)
|
||||
|
||||
result = {
|
||||
"criteria": data,
|
||||
"weighted_score": qa.weighted_score,
|
||||
"decision": _route_score(qa.weighted_score),
|
||||
}
|
||||
|
||||
if json_output:
|
||||
click.echo(json.dumps(result, indent=2))
|
||||
else:
|
||||
console.print(f"\n[bold]Quality Assessment[/bold]")
|
||||
for criterion, score in data.items():
|
||||
bar = "█" * int(score * 20) + "░" * (20 - int(score * 20))
|
||||
console.print(f" {criterion:<30} {bar} {score:.2f}")
|
||||
console.print(f"\n [bold]Weighted Score:[/bold] {qa.weighted_score:.4f}")
|
||||
console.print(f" [bold]Decision:[/bold] {_route_score(qa.weighted_score)}")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--score", required=True, type=float, help="Quality score to route")
|
||||
@click.option("--tier", type=click.Choice(["tier1_official", "tier2_verified", "tier3_community"]),
|
||||
help="Source credibility tier (for auto-approve)")
|
||||
@click.option("--auto-approve", is_flag=True, help="Enable auto-approve for tier1 sources")
|
||||
def route(score, tier, auto_approve):
|
||||
"""Route a quality score to a decision.
|
||||
|
||||
Thresholds:
|
||||
>= 0.85 → approve
|
||||
0.60-0.84 → refactor
|
||||
0.40-0.59 → deep_research
|
||||
< 0.40 → reject
|
||||
"""
|
||||
decision = _route_score(score)
|
||||
|
||||
# Auto-approve tier1 sources with lower threshold
|
||||
if auto_approve and tier == "tier1_official" and score >= 0.80:
|
||||
decision = "approve"
|
||||
|
||||
result = {"score": score, "decision": decision}
|
||||
|
||||
if decision == "approve":
|
||||
console.print(f"[green]APPROVE[/green] (score: {score:.2f})")
|
||||
elif decision == "refactor":
|
||||
console.print(f"[yellow]REFACTOR[/yellow] (score: {score:.2f})")
|
||||
elif decision == "deep_research":
|
||||
console.print(f"[blue]DEEP_RESEARCH[/blue] (score: {score:.2f})")
|
||||
else:
|
||||
console.print(f"[red]REJECT[/red] (score: {score:.2f})")
|
||||
|
||||
click.echo(json.dumps(result))
|
||||
|
||||
|
||||
@cli.command("log-review")
|
||||
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
|
||||
@click.option("--decision", required=True,
|
||||
type=click.Choice(["approve", "refactor", "deep_research", "reject"]))
|
||||
@click.option("--score", required=True, type=float, help="Quality score")
|
||||
@click.option("--assessment", type=click.Path(exists=True), help="Assessment JSON file")
|
||||
@click.option("--feedback", help="Review feedback text")
|
||||
@click.option("--instructions", help="Refactor instructions")
|
||||
@click.option("--queries", type=click.Path(exists=True), help="Research queries JSON (for deep_research)")
|
||||
@click.option("--reviewer", default="claude_review",
|
||||
type=click.Choice(["auto_qa", "human", "claude_review"]))
|
||||
def log_review(distill_id, decision, score, assessment, feedback, instructions, queries, reviewer):
|
||||
"""Log a review decision for distilled content."""
|
||||
assessment_json = json.loads(Path(assessment).read_text()) if assessment else None
|
||||
queries_json = json.loads(Path(queries).read_text()) if queries else None
|
||||
|
||||
with db_session() as db:
|
||||
# Get current review round
|
||||
last_review = db.fetch_one(
|
||||
"SELECT MAX(review_round) as max_round FROM review_logs WHERE distill_id = %s",
|
||||
(distill_id,),
|
||||
)
|
||||
review_round = (last_review.get("max_round") or 0) + 1 if last_review else 1
|
||||
|
||||
review_id = db.insert_returning_id(
|
||||
"""INSERT INTO review_logs
|
||||
(distill_id, review_round, reviewer_type, quality_score, assessment,
|
||||
decision, feedback, refactor_instructions, research_queries, reviewed_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(distill_id, review_round, reviewer, score,
|
||||
json.dumps(assessment_json) if assessment_json else None,
|
||||
decision, feedback, instructions,
|
||||
json.dumps(queries_json) if queries_json else None,
|
||||
datetime.now().isoformat()),
|
||||
)
|
||||
|
||||
# Update distilled_content review_status
|
||||
status_map = {
|
||||
"approve": "approved",
|
||||
"refactor": "needs_refactor",
|
||||
"deep_research": "needs_refactor",
|
||||
"reject": "rejected",
|
||||
}
|
||||
db.execute(
|
||||
"UPDATE distilled_content SET review_status = %s WHERE distill_id = %s",
|
||||
(status_map[decision], distill_id),
|
||||
)
|
||||
|
||||
console.print(
|
||||
f"[green]Logged:[/green] review_id={review_id}, round={review_round}, "
|
||||
f"decision={decision}, score={score:.2f}"
|
||||
)
|
||||
click.echo(json.dumps({
|
||||
"review_id": review_id,
|
||||
"distill_id": distill_id,
|
||||
"review_round": review_round,
|
||||
"decision": decision,
|
||||
"score": score,
|
||||
}))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--distill-id", required=True, type=int, help="Distilled content ID")
|
||||
def history(distill_id):
|
||||
"""Show review history for a distilled content record."""
|
||||
with db_session() as db:
|
||||
reviews = db.fetch_all(
|
||||
"""SELECT review_id, review_round, reviewer_type, quality_score,
|
||||
decision, feedback, refactor_instructions, reviewed_at
|
||||
FROM review_logs
|
||||
WHERE distill_id = %s
|
||||
ORDER BY review_round ASC""",
|
||||
(distill_id,),
|
||||
)
|
||||
|
||||
if not reviews:
|
||||
console.print(f"[dim]No reviews found for distill_id={distill_id}[/dim]")
|
||||
return
|
||||
|
||||
table = Table(title=f"Review History — distill_id={distill_id}")
|
||||
table.add_column("Round", style="cyan")
|
||||
table.add_column("Score", justify="right")
|
||||
table.add_column("Decision")
|
||||
table.add_column("Reviewer")
|
||||
table.add_column("Feedback")
|
||||
for r in reviews:
|
||||
decision = str(r.get("decision", ""))
|
||||
style = {"approve": "green", "refactor": "yellow",
|
||||
"deep_research": "blue", "reject": "red"}.get(decision, "")
|
||||
table.add_row(
|
||||
str(r.get("review_round", "")),
|
||||
f"{float(r['quality_score']):.2f}" if r.get("quality_score") else "",
|
||||
f"[{style}]{decision}[/{style}]" if style else decision,
|
||||
str(r.get("reviewer_type", "")),
|
||||
str(r.get("feedback", ""))[:40] if r.get("feedback") else "",
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
@cli.command("gemini-evaluate")
|
||||
@click.option("--doc-id", required=True, type=int, help="Document ID to evaluate")
|
||||
@click.option("--topic", required=True, help="Curation topic for relevance scoring")
|
||||
@click.option("--auto-approve", is_flag=True, help="Auto-log decision based on Gemini verdict")
|
||||
def gemini_evaluate(doc_id, topic, auto_approve):
|
||||
"""Evaluate raw crawled content using Gemini CLI (pre-distillation gate).
|
||||
|
||||
Sends raw content to Gemini for independent quality assessment.
|
||||
Scores: relevance, authority, completeness, freshness, distill_value.
|
||||
"""
|
||||
from refcurator.gemini import evaluate_content, is_available
|
||||
|
||||
if not is_available():
|
||||
console.print("[red]Error:[/red] Gemini CLI not available. Install: npm install -g @google/gemini-cli")
|
||||
sys.exit(1)
|
||||
|
||||
with db_session() as db:
|
||||
doc = db.fetch_one(
|
||||
"SELECT doc_id, title, url, raw_content_path FROM documents WHERE doc_id = %s",
|
||||
(doc_id,),
|
||||
)
|
||||
|
||||
if not doc:
|
||||
console.print(f"[red]Error:[/red] doc_id={doc_id} not found")
|
||||
sys.exit(1)
|
||||
|
||||
raw_path = doc.get("raw_content_path")
|
||||
if not raw_path or not Path(raw_path).is_file():
|
||||
console.print(f"[red]Error:[/red] Raw content file not found: {raw_path}")
|
||||
sys.exit(1)
|
||||
|
||||
content = Path(raw_path).read_text(errors="replace")
|
||||
url = doc.get("url", "")
|
||||
|
||||
console.print(f"[dim]Evaluating doc_id={doc_id}: {doc.get('title', '')}[/dim]")
|
||||
console.print(f"[dim]Sending {len(content):,} chars to Gemini...[/dim]")
|
||||
|
||||
result = evaluate_content(content, topic, url)
|
||||
|
||||
if result is None:
|
||||
console.print("[yellow]Warning:[/yellow] Gemini evaluation failed — manual review needed")
|
||||
click.echo(json.dumps({"doc_id": doc_id, "status": "gemini_failed"}))
|
||||
return
|
||||
|
||||
# Display results
|
||||
console.print(f"\n[bold]Gemini Evaluation — doc_id={doc_id}[/bold]")
|
||||
for criterion in ["relevance", "authority", "completeness", "freshness", "distill_value"]:
|
||||
score = result.get(criterion, 0)
|
||||
bar = "█" * int(score * 20) + "░" * (20 - int(score * 20))
|
||||
console.print(f" {criterion:<15} {bar} {score:.2f}")
|
||||
|
||||
ws = result.get("weighted_score", 0)
|
||||
verdict = result.get("verdict", "unknown")
|
||||
reason = result.get("reason", "")
|
||||
|
||||
console.print(f"\n [bold]Weighted Score:[/bold] {ws:.4f}")
|
||||
verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white")
|
||||
console.print(f" [bold]Verdict:[/bold] [{verdict_color}]{verdict}[/{verdict_color}]")
|
||||
if reason:
|
||||
console.print(f" [dim]Reason:[/dim] {reason}")
|
||||
|
||||
if auto_approve:
|
||||
# Map verdict to decision (no 'refactor' for raw content)
|
||||
decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject"
|
||||
|
||||
with db_session() as db:
|
||||
# Log to review_logs (using doc_id context, not distill_id since pre-distillation)
|
||||
# We create a placeholder distill_id = 0 entry or log against the document directly
|
||||
review_id = db.insert_returning_id(
|
||||
"""INSERT INTO review_logs
|
||||
(distill_id, review_round, reviewer_type, quality_score, assessment,
|
||||
decision, feedback, reviewed_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(0, 1, "auto_qa", ws,
|
||||
json.dumps(result),
|
||||
decision, reason,
|
||||
datetime.now().isoformat()),
|
||||
)
|
||||
|
||||
console.print(f"\n[green]Auto-logged:[/green] review_id={review_id}, decision={decision}")
|
||||
|
||||
click.echo(json.dumps({
|
||||
"doc_id": doc_id,
|
||||
"evaluation": result,
|
||||
"auto_logged": auto_approve,
|
||||
}, default=str))
|
||||
|
||||
|
||||
@cli.command("gemini-evaluate-pending")
|
||||
@click.option("--topic", required=True, help="Curation topic for relevance scoring")
|
||||
@click.option("--auto-approve", is_flag=True, help="Auto-log decisions")
|
||||
@click.option("--limit", default=20, type=int, help="Max documents to evaluate")
|
||||
def gemini_evaluate_pending(topic, auto_approve, limit):
|
||||
"""Evaluate all crawled-but-not-evaluated documents using Gemini."""
|
||||
from refcurator.gemini import evaluate_content, is_available
|
||||
|
||||
if not is_available():
|
||||
console.print("[red]Error:[/red] Gemini CLI not available")
|
||||
sys.exit(1)
|
||||
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, raw_content_path
|
||||
FROM documents
|
||||
WHERE crawl_status = %s
|
||||
ORDER BY doc_id ASC
|
||||
LIMIT %s""",
|
||||
("completed", limit),
|
||||
)
|
||||
|
||||
if not rows:
|
||||
console.print("[dim]No pending documents to evaluate.[/dim]")
|
||||
return
|
||||
|
||||
console.print(f"[bold]Evaluating {len(rows)} documents with Gemini...[/bold]\n")
|
||||
|
||||
stats = {"approve": 0, "reject": 0, "deep_research": 0, "failed": 0}
|
||||
|
||||
for row in rows:
|
||||
doc_id = row["doc_id"]
|
||||
raw_path = row.get("raw_content_path")
|
||||
|
||||
if not raw_path or not Path(raw_path).is_file():
|
||||
console.print(f" [yellow]Skip[/yellow] doc_id={doc_id}: raw file not found")
|
||||
stats["failed"] += 1
|
||||
continue
|
||||
|
||||
content = Path(raw_path).read_text(errors="replace")
|
||||
result = evaluate_content(content, topic, row.get("url", ""))
|
||||
|
||||
if result is None:
|
||||
console.print(f" [yellow]Failed[/yellow] doc_id={doc_id}: Gemini returned no result")
|
||||
stats["failed"] += 1
|
||||
continue
|
||||
|
||||
verdict = result.get("verdict", "reject")
|
||||
ws = result.get("weighted_score", 0)
|
||||
verdict_color = {"approve": "green", "reject": "red", "deep_research": "blue"}.get(verdict, "white")
|
||||
console.print(
|
||||
f" [{verdict_color}]{verdict:>13}[/{verdict_color}] "
|
||||
f"({ws:.2f}) doc_id={doc_id} — {row.get('title', '')[:40]}"
|
||||
)
|
||||
|
||||
stats[verdict] = stats.get(verdict, 0) + 1
|
||||
|
||||
if auto_approve:
|
||||
decision = verdict if verdict in ("approve", "reject", "deep_research") else "reject"
|
||||
with db_session() as db:
|
||||
db.insert_returning_id(
|
||||
"""INSERT INTO review_logs
|
||||
(distill_id, review_round, reviewer_type, quality_score, assessment,
|
||||
decision, feedback, reviewed_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(0, 1, "auto_qa", ws,
|
||||
json.dumps(result),
|
||||
decision, result.get("reason", ""),
|
||||
datetime.now().isoformat()),
|
||||
)
|
||||
|
||||
console.print(f"\n[bold]Results:[/bold]")
|
||||
console.print(f" [green]Approved:[/green] {stats['approve']}")
|
||||
console.print(f" [red]Rejected:[/red] {stats['reject']}")
|
||||
console.print(f" [blue]Deep Research:[/blue] {stats['deep_research']}")
|
||||
console.print(f" [yellow]Failed:[/yellow] {stats['failed']}")
|
||||
click.echo(json.dumps(stats))
|
||||
|
||||
|
||||
def _route_score(score: float) -> str:
|
||||
"""Route a quality score to a decision based on thresholds."""
|
||||
if score >= APPROVE_THRESHOLD:
|
||||
return "approve"
|
||||
elif score >= REFACTOR_THRESHOLD:
|
||||
return "refactor"
|
||||
elif score >= DEEP_RESEARCH_THRESHOLD:
|
||||
return "deep_research"
|
||||
else:
|
||||
return "reject"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -9,57 +9,51 @@ Exports approved reference content as structured markdown files for project know
|
||||
|
||||
| Type | Format | Use Case |
|
||||
|------|--------|----------|
|
||||
| `project_files` | Nested markdown | Claude Projects knowledge |
|
||||
| `fine_tuning` | JSONL | Model fine-tuning dataset |
|
||||
| `knowledge_base` | Flat markdown | Documentation |
|
||||
| `project` | Nested markdown | Claude Projects knowledge |
|
||||
| `finetuning` | JSONL | Model fine-tuning dataset |
|
||||
|
||||
## Workflow
|
||||
|
||||
### Step 1: Query Approved Content
|
||||
### Step 1: Export Project Files
|
||||
```bash
|
||||
python scripts/query_approved.py --min-score 0.80 --output approved.json
|
||||
uv run python scripts/exporter.py project \
|
||||
--output ~/reference-library/exports/ \
|
||||
--min-score 0.80 \
|
||||
--structure nested_by_topic
|
||||
```
|
||||
|
||||
### Step 2: Organize by Structure
|
||||
|
||||
**Nested by Topic (default):**
|
||||
Output structure:
|
||||
```
|
||||
exports/
|
||||
├── INDEX.md
|
||||
├── prompt-engineering/
|
||||
│ ├── _index.md
|
||||
│ ├── 01-chain-of-thought.md
|
||||
│ └── 02-few-shot-prompting.md
|
||||
│ ├── 00-chain-of-thought.md
|
||||
│ └── 01-few-shot-prompting.md
|
||||
└── claude-models/
|
||||
├── _index.md
|
||||
└── 01-model-comparison.md
|
||||
└── 00-model-comparison.md
|
||||
```
|
||||
|
||||
**Flat Structure:**
|
||||
```
|
||||
exports/
|
||||
├── INDEX.md
|
||||
├── prompt-engineering-chain-of-thought.md
|
||||
└── claude-models-comparison.md
|
||||
```
|
||||
|
||||
### Step 3: Generate Files
|
||||
### Step 2: Generate INDEX
|
||||
```bash
|
||||
python scripts/export_project.py \
|
||||
--structure nested_by_topic \
|
||||
--output ~/reference-library/exports/ \
|
||||
--include-metadata
|
||||
uv run python scripts/exporter.py index --output ~/reference-library/exports/INDEX.md
|
||||
```
|
||||
|
||||
### Step 4: Generate INDEX
|
||||
### Step 3: Add Cross-References
|
||||
```bash
|
||||
python scripts/generate_index.py --output ~/reference-library/exports/INDEX.md
|
||||
uv run python scripts/exporter.py crossrefs --input ~/reference-library/exports/
|
||||
```
|
||||
|
||||
### Step 4: Verify Export
|
||||
```bash
|
||||
uv run python scripts/exporter.py verify --path ~/reference-library/exports/
|
||||
```
|
||||
|
||||
### Step 5: Fine-tuning Export (Optional)
|
||||
```bash
|
||||
python scripts/export_finetuning.py \
|
||||
--output ~/reference-library/exports/fine_tuning.jsonl \
|
||||
uv run python scripts/exporter.py finetuning \
|
||||
--output ~/reference-library/exports/training.jsonl \
|
||||
--max-tokens 4096
|
||||
```
|
||||
|
||||
@@ -71,62 +65,25 @@ JSONL format:
|
||||
{"role": "user", "content": "Explain {title}"},
|
||||
{"role": "assistant", "content": "{structured_content}"}
|
||||
],
|
||||
"metadata": {"source": "{url}", "topic": "{topic_slug}", "quality_score": 0.92}
|
||||
"metadata": {"source": "{url}", "quality_score": 0.92}
|
||||
}
|
||||
```
|
||||
|
||||
### Step 6: Log Export Job
|
||||
```bash
|
||||
python scripts/log_export.py --name "January 2025 Export" --type project_files --docs 45
|
||||
```
|
||||
|
||||
## Cross-Reference Generation
|
||||
```bash
|
||||
python scripts/add_crossrefs.py --input ~/reference-library/exports/
|
||||
```
|
||||
|
||||
Links related documents based on overlapping key concepts.
|
||||
|
||||
## Output Verification
|
||||
|
||||
After export, verify:
|
||||
- [ ] All files readable and valid markdown
|
||||
- [ ] INDEX.md links resolve correctly
|
||||
- [ ] No broken cross-references
|
||||
- [ ] Total token count matches expectation
|
||||
- [ ] No duplicate content
|
||||
|
||||
```bash
|
||||
python scripts/verify_export.py --path ~/reference-library/exports/
|
||||
uv run python scripts/exporter.py log --name "April 2026 Export" --type project_files --docs 45
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/query_approved.py` - Get approved content from DB
|
||||
- `scripts/export_project.py` - Main export for project files
|
||||
- `scripts/export_finetuning.py` - JSONL export for fine-tuning
|
||||
- `scripts/generate_index.py` - Generate INDEX.md
|
||||
- `scripts/add_crossrefs.py` - Add cross-references
|
||||
- `scripts/log_export.py` - Log export job to DB
|
||||
- `scripts/verify_export.py` - Verify export integrity
|
||||
|
||||
## Configuration
|
||||
|
||||
```yaml
|
||||
# ~/.config/reference-curator/export_config.yaml
|
||||
output:
|
||||
base_path: ~/reference-library/exports/
|
||||
project_files:
|
||||
structure: nested_by_topic
|
||||
index_file: INDEX.md
|
||||
include_metadata: true
|
||||
fine_tuning:
|
||||
format: jsonl
|
||||
max_tokens_per_sample: 4096
|
||||
|
||||
quality:
|
||||
min_score_for_export: 0.80
|
||||
```
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `exporter.py project` | Export as nested markdown files |
|
||||
| `exporter.py finetuning` | Export as JSONL training dataset |
|
||||
| `exporter.py index` | Generate INDEX.md table of contents |
|
||||
| `exporter.py crossrefs` | Add cross-reference links |
|
||||
| `exporter.py verify` | Verify export integrity |
|
||||
| `exporter.py log` | Log export job to DB |
|
||||
|
||||
## Integration
|
||||
|
||||
|
||||
@@ -0,0 +1,451 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Markdown Exporter CLI — export approved content as structured files.
|
||||
|
||||
Usage:
|
||||
python exporter.py project --output ~/reference-library/exports/ [--min-score 0.80]
|
||||
python exporter.py finetuning --output ~/reference-library/exports/training.jsonl
|
||||
python exporter.py index --output ~/reference-library/exports/INDEX.md
|
||||
python exporter.py crossrefs --input ~/reference-library/exports/
|
||||
python exporter.py verify --path ~/reference-library/exports/
|
||||
python exporter.py log --name "Jan 2025 Export" --type project_files --docs 40
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.utils import count_tokens, slugify
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Markdown Exporter — export approved references."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output directory path")
|
||||
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
|
||||
@click.option("--structure", default="nested_by_topic",
|
||||
type=click.Choice(["nested_by_topic", "flat"]))
|
||||
@click.option("--include-metadata", is_flag=True, default=True, help="Include source metadata")
|
||||
def project(output, min_score, structure, include_metadata):
|
||||
"""Export approved content as markdown project files."""
|
||||
out_dir = Path(output).expanduser()
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url,
|
||||
dc.structured_content, dc.summary, dc.key_concepts,
|
||||
dc.token_count_distilled,
|
||||
rl.quality_score,
|
||||
s.credibility_tier, s.vendor, s.source_name
|
||||
FROM documents d
|
||||
JOIN distilled_content dc ON d.doc_id = dc.doc_id
|
||||
JOIN review_logs rl ON dc.distill_id = rl.distill_id
|
||||
JOIN sources s ON d.source_id = s.source_id
|
||||
WHERE dc.review_status = %s
|
||||
AND rl.decision = %s
|
||||
AND rl.review_id = (
|
||||
SELECT MAX(rl2.review_id)
|
||||
FROM review_logs rl2
|
||||
WHERE rl2.distill_id = dc.distill_id
|
||||
)
|
||||
ORDER BY rl.quality_score DESC""",
|
||||
("approved", "approve"),
|
||||
)
|
||||
|
||||
# Filter by min score
|
||||
rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]
|
||||
|
||||
if not rows:
|
||||
console.print("[yellow]No approved documents found above minimum score.[/yellow]")
|
||||
return
|
||||
|
||||
# Get topic mappings
|
||||
with db_session() as db:
|
||||
topic_rows = db.fetch_all(
|
||||
"""SELECT dt.doc_id, t.topic_name, t.topic_slug
|
||||
FROM document_topics dt
|
||||
JOIN topics t ON dt.topic_id = t.topic_id"""
|
||||
)
|
||||
|
||||
doc_topics = defaultdict(list)
|
||||
for tr in topic_rows:
|
||||
doc_topics[tr["doc_id"]].append(tr)
|
||||
|
||||
exported = 0
|
||||
total_tokens = 0
|
||||
|
||||
if structure == "nested_by_topic":
|
||||
exported, total_tokens = _export_nested(rows, doc_topics, out_dir, include_metadata)
|
||||
else:
|
||||
exported, total_tokens = _export_flat(rows, doc_topics, out_dir, include_metadata)
|
||||
|
||||
console.print(
|
||||
f"[green]Exported {exported} documents ({total_tokens:,} tokens) to {out_dir}[/green]"
|
||||
)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output JSONL file path")
|
||||
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
|
||||
@click.option("--max-tokens", default=4096, type=int, help="Max tokens per sample")
|
||||
@click.option("--system-prompt", default="You are an expert on AI and prompt engineering.",
|
||||
help="System prompt for training samples")
|
||||
def finetuning(output, min_score, max_tokens, system_prompt):
|
||||
"""Export approved content as JSONL fine-tuning dataset."""
|
||||
out_path = Path(output).expanduser()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url,
|
||||
dc.structured_content, dc.summary,
|
||||
dc.token_count_distilled,
|
||||
rl.quality_score
|
||||
FROM documents d
|
||||
JOIN distilled_content dc ON d.doc_id = dc.doc_id
|
||||
JOIN review_logs rl ON dc.distill_id = rl.distill_id
|
||||
WHERE dc.review_status = %s
|
||||
AND rl.decision = %s
|
||||
AND rl.review_id = (
|
||||
SELECT MAX(rl2.review_id)
|
||||
FROM review_logs rl2
|
||||
WHERE rl2.distill_id = dc.distill_id
|
||||
)
|
||||
ORDER BY rl.quality_score DESC""",
|
||||
("approved", "approve"),
|
||||
)
|
||||
|
||||
rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]
|
||||
|
||||
count = 0
|
||||
with open(out_path, "w") as f:
|
||||
for row in rows:
|
||||
content = row.get("structured_content", "")
|
||||
if count_tokens(content) > max_tokens:
|
||||
content = content[:max_tokens * 4] # Approximate truncation
|
||||
|
||||
sample = {
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": f"Explain {row.get('title', 'this topic')}"},
|
||||
{"role": "assistant", "content": content},
|
||||
],
|
||||
"metadata": {
|
||||
"source": row.get("url"),
|
||||
"quality_score": float(row["quality_score"]) if row.get("quality_score") else None,
|
||||
"doc_id": row.get("doc_id"),
|
||||
},
|
||||
}
|
||||
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
|
||||
count += 1
|
||||
|
||||
console.print(f"[green]Exported {count} samples to {out_path}[/green]")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output INDEX.md path")
|
||||
@click.option("--exports-dir", type=click.Path(exists=True),
|
||||
help="Exports directory to scan (defaults to parent of output)")
|
||||
def index(output, exports_dir):
|
||||
"""Generate INDEX.md with table of contents."""
|
||||
out_path = Path(output).expanduser()
|
||||
scan_dir = Path(exports_dir).expanduser() if exports_dir else out_path.parent
|
||||
|
||||
lines = [
|
||||
"# Reference Library Index",
|
||||
"",
|
||||
f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}*",
|
||||
"",
|
||||
]
|
||||
|
||||
# Scan for topic directories
|
||||
topic_dirs = sorted([d for d in scan_dir.iterdir() if d.is_dir() and not d.name.startswith("_")])
|
||||
|
||||
if topic_dirs:
|
||||
lines.append("## Topics\n")
|
||||
for topic_dir in topic_dirs:
|
||||
md_files = sorted(topic_dir.glob("*.md"))
|
||||
if md_files:
|
||||
topic_name = topic_dir.name.replace("-", " ").title()
|
||||
lines.append(f"### {topic_name}\n")
|
||||
for md_file in md_files:
|
||||
if md_file.name.startswith("_"):
|
||||
continue
|
||||
title = _extract_md_title(md_file)
|
||||
rel_path = md_file.relative_to(scan_dir)
|
||||
lines.append(f"- [{title}]({rel_path})")
|
||||
lines.append("")
|
||||
|
||||
# Scan for flat files
|
||||
flat_files = sorted([f for f in scan_dir.glob("*.md")
|
||||
if f.name not in ("INDEX.md", "_index.md")])
|
||||
if flat_files:
|
||||
lines.append("## Documents\n")
|
||||
for md_file in flat_files:
|
||||
title = _extract_md_title(md_file)
|
||||
lines.append(f"- [{title}]({md_file.name})")
|
||||
lines.append("")
|
||||
|
||||
out_path.write_text("\n".join(lines))
|
||||
console.print(f"[green]Generated INDEX.md at {out_path}[/green]")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--input", "input_dir", required=True, type=click.Path(exists=True),
|
||||
help="Exports directory to add cross-references to")
|
||||
def crossrefs(input_dir):
|
||||
"""Add cross-reference links between related documents."""
|
||||
scan_dir = Path(input_dir).expanduser()
|
||||
md_files = list(scan_dir.rglob("*.md"))
|
||||
|
||||
if not md_files:
|
||||
console.print("[yellow]No markdown files found.[/yellow]")
|
||||
return
|
||||
|
||||
# Build concept index: concept → list of (file, title)
|
||||
concept_index: dict[str, list[tuple[Path, str]]] = defaultdict(list)
|
||||
file_concepts: dict[Path, set[str]] = {}
|
||||
|
||||
for md_file in md_files:
|
||||
if md_file.name in ("INDEX.md", "_index.md"):
|
||||
continue
|
||||
content = md_file.read_text(errors="replace")
|
||||
title = _extract_md_title(md_file)
|
||||
concepts = _extract_concepts(content)
|
||||
file_concepts[md_file] = concepts
|
||||
for concept in concepts:
|
||||
concept_index[concept].append((md_file, title))
|
||||
|
||||
# Add cross-references
|
||||
modified = 0
|
||||
for md_file in md_files:
|
||||
if md_file.name in ("INDEX.md", "_index.md"):
|
||||
continue
|
||||
my_concepts = file_concepts.get(md_file, set())
|
||||
related: dict[str, str] = {} # title → relative path
|
||||
|
||||
for concept in my_concepts:
|
||||
for other_file, other_title in concept_index.get(concept, []):
|
||||
if other_file != md_file and other_title not in related:
|
||||
try:
|
||||
rel = other_file.relative_to(scan_dir)
|
||||
except ValueError:
|
||||
rel = other_file
|
||||
related[other_title] = str(rel)
|
||||
|
||||
if related and len(related) <= 10:
|
||||
content = md_file.read_text(errors="replace")
|
||||
# Remove existing Related section if present
|
||||
content = re.sub(r"\n## Related\n.*$", "", content, flags=re.DOTALL)
|
||||
content = content.rstrip() + "\n\n## Related\n\n"
|
||||
for title, path in sorted(related.items())[:5]:
|
||||
content += f"- [{title}]({path})\n"
|
||||
md_file.write_text(content)
|
||||
modified += 1
|
||||
|
||||
console.print(f"[green]Added cross-references to {modified} files[/green]")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--path", required=True, type=click.Path(exists=True), help="Exports directory to verify")
|
||||
def verify(path):
|
||||
"""Verify export integrity."""
|
||||
scan_dir = Path(path).expanduser()
|
||||
md_files = list(scan_dir.rglob("*.md"))
|
||||
|
||||
issues = []
|
||||
total_tokens = 0
|
||||
total_files = len(md_files)
|
||||
|
||||
for md_file in md_files:
|
||||
content = md_file.read_text(errors="replace")
|
||||
tokens = count_tokens(content)
|
||||
total_tokens += tokens
|
||||
|
||||
# Check for empty files
|
||||
if len(content.strip()) < 10:
|
||||
issues.append(f"Empty or near-empty: {md_file.relative_to(scan_dir)}")
|
||||
|
||||
# Check for broken internal links
|
||||
for match in re.finditer(r"\[([^\]]+)\]\(([^)]+)\)", content):
|
||||
link_path = match.group(2)
|
||||
if link_path.startswith("http"):
|
||||
continue
|
||||
resolved = (md_file.parent / link_path).resolve()
|
||||
if not resolved.exists():
|
||||
issues.append(
|
||||
f"Broken link in {md_file.relative_to(scan_dir)}: {link_path}"
|
||||
)
|
||||
|
||||
# Check INDEX.md exists
|
||||
if not (scan_dir / "INDEX.md").is_file():
|
||||
issues.append("Missing INDEX.md")
|
||||
|
||||
# Report
|
||||
console.print(f"\n[bold]Export Verification: {scan_dir}[/bold]")
|
||||
console.print(f" Files: {total_files}")
|
||||
console.print(f" Total tokens: {total_tokens:,}")
|
||||
|
||||
if issues:
|
||||
console.print(f"\n[red]Issues ({len(issues)}):[/red]")
|
||||
for issue in issues[:20]:
|
||||
console.print(f" - {issue}")
|
||||
else:
|
||||
console.print(f"\n[green]All checks passed.[/green]")
|
||||
|
||||
click.echo(json.dumps({
|
||||
"files": total_files,
|
||||
"total_tokens": total_tokens,
|
||||
"issues": len(issues),
|
||||
"issue_details": issues[:20],
|
||||
}))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--name", required=True, help="Export job name")
|
||||
@click.option("--type", "export_type", required=True,
|
||||
type=click.Choice(["project_files", "fine_tuning", "knowledge_base"]))
|
||||
@click.option("--docs", required=True, type=int, help="Number of documents exported")
|
||||
@click.option("--path", type=click.Path(), help="Export output path")
|
||||
@click.option("--tokens", type=int, help="Total tokens exported")
|
||||
def log(name, export_type, docs, path, tokens):
|
||||
"""Log an export job to the database."""
|
||||
with db_session() as db:
|
||||
export_id = db.insert_returning_id(
|
||||
"""INSERT INTO export_jobs
|
||||
(export_name, export_type, output_path, total_documents, total_tokens,
|
||||
status, started_at, completed_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(name, export_type, path, docs, tokens,
|
||||
"completed", datetime.now().isoformat(), datetime.now().isoformat()),
|
||||
)
|
||||
|
||||
console.print(f"[green]Logged export:[/green] export_id={export_id} — {name}")
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
def _export_nested(rows, doc_topics, out_dir, include_metadata):
|
||||
"""Export documents in nested topic directory structure."""
|
||||
exported = 0
|
||||
total_tokens = 0
|
||||
topic_docs = defaultdict(list)
|
||||
|
||||
for row in rows:
|
||||
topics = doc_topics.get(row["doc_id"], [])
|
||||
if topics:
|
||||
for t in topics:
|
||||
topic_docs[t["topic_slug"]].append(row)
|
||||
else:
|
||||
topic_docs["uncategorized"].append(row)
|
||||
|
||||
for topic_slug, docs in sorted(topic_docs.items()):
|
||||
topic_dir = out_dir / topic_slug
|
||||
topic_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Topic index
|
||||
topic_name = topic_slug.replace("-", " ").title()
|
||||
index_lines = [f"# {topic_name}\n"]
|
||||
|
||||
for i, doc in enumerate(docs):
|
||||
filename = f"{i:02d}-{slugify(doc.get('title', 'untitled'))}.md"
|
||||
content = _format_document(doc, include_metadata)
|
||||
(topic_dir / filename).write_text(content)
|
||||
index_lines.append(f"- [{doc.get('title', 'Untitled')}]({filename})")
|
||||
exported += 1
|
||||
total_tokens += doc.get("token_count_distilled") or count_tokens(content)
|
||||
|
||||
(topic_dir / "_index.md").write_text("\n".join(index_lines) + "\n")
|
||||
|
||||
return exported, total_tokens
|
||||
|
||||
|
||||
def _export_flat(rows, doc_topics, out_dir, include_metadata):
|
||||
"""Export documents as flat files."""
|
||||
exported = 0
|
||||
total_tokens = 0
|
||||
|
||||
for row in rows:
|
||||
topics = doc_topics.get(row["doc_id"], [])
|
||||
topic_prefix = topics[0]["topic_slug"] + "-" if topics else ""
|
||||
filename = f"{topic_prefix}{slugify(row.get('title', 'untitled'))}.md"
|
||||
content = _format_document(row, include_metadata)
|
||||
(out_dir / filename).write_text(content)
|
||||
exported += 1
|
||||
total_tokens += row.get("token_count_distilled") or count_tokens(content)
|
||||
|
||||
return exported, total_tokens
|
||||
|
||||
|
||||
def _format_document(row: dict, include_metadata: bool) -> str:
|
||||
"""Format a document for export."""
|
||||
lines = []
|
||||
|
||||
if include_metadata:
|
||||
lines.extend([
|
||||
f"# {row.get('title', 'Untitled')}",
|
||||
"",
|
||||
f"**Source:** {row.get('url', 'N/A')}",
|
||||
f"**Tier:** {row.get('credibility_tier', 'N/A')} | "
|
||||
f"**Vendor:** {row.get('vendor', 'N/A')} | "
|
||||
f"**Score:** {float(row['quality_score']):.2f}" if row.get("quality_score") else "",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
])
|
||||
|
||||
content = row.get("structured_content", "")
|
||||
if content:
|
||||
lines.append(content)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _extract_md_title(md_file: Path) -> str:
|
||||
"""Extract title from a markdown file."""
|
||||
try:
|
||||
for line in md_file.read_text(errors="replace").split("\n")[:10]:
|
||||
if line.startswith("# ") and not line.startswith("##"):
|
||||
return line[2:].strip()
|
||||
except Exception:
|
||||
pass
|
||||
return md_file.stem.replace("-", " ").title()
|
||||
|
||||
|
||||
def _extract_concepts(content: str) -> set[str]:
|
||||
"""Extract key concepts from markdown content for cross-referencing."""
|
||||
concepts = set()
|
||||
|
||||
# Extract from ## Key Concepts section
|
||||
m = re.search(r"## Key Concepts?\n(.*?)(?=\n##|\Z)", content, re.DOTALL)
|
||||
if m:
|
||||
for line in m.group(1).split("\n"):
|
||||
line = line.strip().lstrip("- *")
|
||||
if line and ":" in line:
|
||||
concept = line.split(":")[0].strip("*").strip()
|
||||
if 2 < len(concept) < 50:
|
||||
concepts.add(concept.lower())
|
||||
|
||||
# Extract bold terms
|
||||
for m in re.finditer(r"\*\*([^*]{3,40})\*\*", content):
|
||||
concepts.add(m.group(1).lower())
|
||||
|
||||
return concepts
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -1,6 +1,6 @@
|
||||
# Pipeline Orchestrator
|
||||
|
||||
Coordinates the full 6-skill reference curation workflow with QA loop handling.
|
||||
Coordinates the full 6-skill reference curation workflow with QA loop handling. Manages pipeline state (init, advance, pause, resume, complete) while Claude orchestrates the actual stage execution.
|
||||
|
||||
## Trigger Keywords
|
||||
"curate references", "full pipeline", "run curation", "reference-curator-pipeline"
|
||||
@@ -22,257 +22,144 @@ Coordinates the full 6-skill reference curation workflow with QA loop handling.
|
||||
crawler ─┘
|
||||
```
|
||||
|
||||
## Input Detection
|
||||
## Pipeline State Management
|
||||
|
||||
Parse input to determine mode:
|
||||
|
||||
```python
|
||||
def detect_input_mode(input_value):
|
||||
if input_value.endswith('.json') and os.path.exists(input_value):
|
||||
return 'manifest'
|
||||
elif input_value.startswith('http://') or input_value.startswith('https://'):
|
||||
return 'urls'
|
||||
else:
|
||||
return 'topic'
|
||||
### Initialize a Run
|
||||
```bash
|
||||
uv run python scripts/pipeline.py init \
|
||||
--input "prompt engineering" --type topic \
|
||||
--options '{"max_sources": 10, "auto_approve": true}'
|
||||
```
|
||||
|
||||
## Pipeline Execution
|
||||
### Advance to Next Stage
|
||||
```bash
|
||||
uv run python scripts/pipeline.py advance \
|
||||
--run-id 1 --stage crawling \
|
||||
--stats '{"sources_discovered": 8}'
|
||||
```
|
||||
|
||||
### Stage 1: Reference Discovery (Topic Mode Only)
|
||||
### Pause on Error
|
||||
```bash
|
||||
uv run python scripts/pipeline.py pause \
|
||||
--run-id 1 --error "Crawl timeout on page 45" --stage crawling
|
||||
```
|
||||
|
||||
### Resume from Pause
|
||||
```bash
|
||||
uv run python scripts/pipeline.py resume --run-id 1
|
||||
```
|
||||
|
||||
### Complete Pipeline
|
||||
```bash
|
||||
uv run python scripts/pipeline.py complete \
|
||||
--run-id 1 --export-path ~/reference-library/exports/ --export-count 40
|
||||
```
|
||||
|
||||
### Check Status
|
||||
```bash
|
||||
uv run python scripts/pipeline.py status --run-id 1
|
||||
uv run python scripts/pipeline.py status --all
|
||||
```
|
||||
|
||||
## QA Loop Tracking
|
||||
|
||||
```bash
|
||||
# Skip if input mode is 'urls' or 'manifest'
|
||||
if mode == 'topic':
|
||||
/reference-discovery "$TOPIC" --max-sources $MAX_SOURCES
|
||||
# Output: manifest.json
|
||||
# Track a refactor iteration for a document
|
||||
uv run python scripts/pipeline.py track-iteration \
|
||||
--run-id 1 --doc-id 42 --action refactor
|
||||
|
||||
# Track a deep research iteration
|
||||
uv run python scripts/pipeline.py track-iteration \
|
||||
--run-id 1 --doc-id 42 --action deep_research
|
||||
```
|
||||
|
||||
Returns one of:
|
||||
- `re_distill` — proceed with refactor
|
||||
- `re_crawl_and_distill` — proceed with deep research
|
||||
- `needs_manual_review` — max iterations exceeded
|
||||
|
||||
| Decision | Max Iterations |
|
||||
|----------|----------------|
|
||||
| REFACTOR | 3 |
|
||||
| DEEP_RESEARCH | 2 |
|
||||
| Combined total | 5 |
|
||||
|
||||
## Pipeline Execution Flow
|
||||
|
||||
### Stage 1: Reference Discovery (Topic Mode Only)
|
||||
```
|
||||
If mode == 'topic':
|
||||
→ Claude runs WebSearch
|
||||
→ discover.py create-manifest
|
||||
→ discover.py dedup
|
||||
→ pipeline.py advance --stage crawling
|
||||
```
|
||||
|
||||
### Stage 2: Web Crawler
|
||||
|
||||
```bash
|
||||
# From manifest or URLs
|
||||
/web-crawler $INPUT --max-pages $MAX_PAGES
|
||||
# Output: crawled files in ~/reference-library/raw/
|
||||
```
|
||||
→ Claude uses Firecrawl MCP tools
|
||||
→ crawl_mgr.py store-result
|
||||
→ pipeline.py advance --stage storing
|
||||
```
|
||||
|
||||
### Stage 3: Content Repository
|
||||
|
||||
```bash
|
||||
/content-repository store
|
||||
# Output: documents stored in MySQL or file-based storage
|
||||
```
|
||||
→ repo.py store (for each crawled doc)
|
||||
→ pipeline.py advance --stage evaluating
|
||||
```
|
||||
|
||||
### Stage 4: Content Distiller
|
||||
|
||||
```bash
|
||||
/content-distiller all-pending
|
||||
# Output: distilled content records
|
||||
### Stage 4: Gemini Quality Gate (Pre-Distillation)
|
||||
```
|
||||
→ reviewer.py gemini-evaluate-pending --topic "$TOPIC" --auto-approve
|
||||
→ APPROVE: proceed to distillation
|
||||
→ DEEP_RESEARCH: pipeline.py track-iteration → crawler (re-crawl)
|
||||
→ REJECT: skip document entirely
|
||||
→ pipeline.py advance --stage distilling
|
||||
```
|
||||
|
||||
### Stage 5: Quality Reviewer
|
||||
|
||||
```bash
|
||||
if auto_approve:
|
||||
/quality-reviewer all-pending --auto-approve --threshold $THRESHOLD
|
||||
else:
|
||||
/quality-reviewer all-pending
|
||||
### Stage 5: Content Distiller (Approved Only)
|
||||
```
|
||||
→ distiller.py load-pending
|
||||
→ Claude distills each approved document
|
||||
→ distiller.py store
|
||||
→ pipeline.py advance --stage exporting
|
||||
```
|
||||
|
||||
Handle QA decisions:
|
||||
- **APPROVE**: Add to export queue
|
||||
- **REFACTOR**: Re-run distiller with feedback (track iteration count)
|
||||
- **DEEP_RESEARCH**: Run crawler for additional sources, then distill
|
||||
- **REJECT**: Archive with reason
|
||||
|
||||
### Stage 6: Markdown Exporter
|
||||
|
||||
```bash
|
||||
/markdown-exporter $EXPORT_FORMAT
|
||||
# Output: files in ~/reference-library/exports/
|
||||
```
|
||||
|
||||
## State Management
|
||||
|
||||
### Initialize Pipeline State
|
||||
|
||||
```python
|
||||
def init_pipeline_state(run_id, input_value, options):
|
||||
state = {
|
||||
"run_id": run_id,
|
||||
"run_type": detect_input_mode(input_value),
|
||||
"input_value": input_value,
|
||||
"status": "running",
|
||||
"current_stage": "discovery",
|
||||
"options": options,
|
||||
"stats": {
|
||||
"sources_discovered": 0,
|
||||
"pages_crawled": 0,
|
||||
"documents_stored": 0,
|
||||
"documents_distilled": 0,
|
||||
"approved": 0,
|
||||
"refactored": 0,
|
||||
"deep_researched": 0,
|
||||
"rejected": 0,
|
||||
"needs_manual_review": 0
|
||||
},
|
||||
"started_at": datetime.now().isoformat()
|
||||
}
|
||||
save_state(run_id, state)
|
||||
return state
|
||||
```
|
||||
|
||||
### MySQL State (Preferred)
|
||||
|
||||
```sql
|
||||
INSERT INTO pipeline_runs (run_type, input_value, options)
|
||||
VALUES ('topic', 'Claude system prompts', '{"max_sources": 10}');
|
||||
```
|
||||
|
||||
### File-Based Fallback
|
||||
|
||||
```
|
||||
~/reference-library/pipeline_state/run_XXX/
|
||||
├── state.json # Current stage and stats
|
||||
├── manifest.json # Discovered sources
|
||||
├── crawl_results.json # Crawled document paths
|
||||
├── review_log.json # QA decisions per document
|
||||
└── errors.log # Any errors encountered
|
||||
```
|
||||
|
||||
## QA Loop Logic
|
||||
|
||||
```python
|
||||
MAX_REFACTOR_ITERATIONS = 3
|
||||
MAX_DEEP_RESEARCH_ITERATIONS = 2
|
||||
MAX_TOTAL_ITERATIONS = 5
|
||||
|
||||
def handle_qa_decision(doc_id, decision, iteration_counts):
|
||||
refactor_count = iteration_counts.get('refactor', 0)
|
||||
research_count = iteration_counts.get('deep_research', 0)
|
||||
total = refactor_count + research_count
|
||||
|
||||
if total >= MAX_TOTAL_ITERATIONS:
|
||||
return 'needs_manual_review'
|
||||
|
||||
if decision == 'refactor':
|
||||
if refactor_count >= MAX_REFACTOR_ITERATIONS:
|
||||
return 'needs_manual_review'
|
||||
iteration_counts['refactor'] = refactor_count + 1
|
||||
return 're_distill'
|
||||
|
||||
if decision == 'deep_research':
|
||||
if research_count >= MAX_DEEP_RESEARCH_ITERATIONS:
|
||||
return 'needs_manual_review'
|
||||
iteration_counts['deep_research'] = research_count + 1
|
||||
return 're_crawl_and_distill'
|
||||
|
||||
return decision # approve or reject
|
||||
→ exporter.py project
|
||||
→ exporter.py index
|
||||
→ exporter.py crossrefs
|
||||
→ exporter.py verify
|
||||
→ pipeline.py complete
|
||||
```
|
||||
|
||||
## Checkpoint Strategy
|
||||
|
||||
Save checkpoint after each stage completes:
|
||||
|
||||
| Stage | Checkpoint | Resume Point |
|
||||
|-------|------------|--------------|
|
||||
| discovery | `manifest.json` created | → crawler |
|
||||
| crawl | `crawl_results.json` | → repository |
|
||||
| store | DB records or file list | → distiller |
|
||||
| discovery | manifest.json created | → crawler |
|
||||
| crawl | crawl_result.json | → repository |
|
||||
| store | DB records | → distiller |
|
||||
| distill | distilled_content records | → reviewer |
|
||||
| review | review_logs records | → exporter or loop |
|
||||
| export | final export complete | Done |
|
||||
|
||||
## Progress Reporting
|
||||
## Scripts
|
||||
|
||||
Report progress to user at key checkpoints:
|
||||
|
||||
```
|
||||
[Pipeline] Stage 1/6: Discovery - Found 8 sources
|
||||
[Pipeline] Stage 2/6: Crawling - 45/50 pages complete
|
||||
[Pipeline] Stage 3/6: Storing - 45 documents saved
|
||||
[Pipeline] Stage 4/6: Distilling - 45 documents processed
|
||||
[Pipeline] Stage 5/6: Reviewing - 40 approved, 3 refactored, 2 rejected
|
||||
[Pipeline] Stage 6/6: Exporting - 40 documents exported
|
||||
[Pipeline] Complete! See ~/reference-library/exports/
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```python
|
||||
def handle_stage_error(stage, error, state):
|
||||
state['status'] = 'paused'
|
||||
state['error_message'] = str(error)
|
||||
state['error_stage'] = stage
|
||||
save_state(state['run_id'], state)
|
||||
|
||||
# Log to errors.log
|
||||
log_error(state['run_id'], stage, error)
|
||||
|
||||
# Report to user
|
||||
return f"Pipeline paused at {stage}: {error}. Resume with run_id {state['run_id']}"
|
||||
```
|
||||
|
||||
## Resume Pipeline
|
||||
|
||||
```python
|
||||
def resume_pipeline(run_id):
|
||||
state = load_state(run_id)
|
||||
|
||||
if state['status'] != 'paused':
|
||||
return f"Pipeline {run_id} is {state['status']}, cannot resume"
|
||||
|
||||
stage = state['current_stage']
|
||||
state['status'] = 'running'
|
||||
state['error_message'] = None
|
||||
save_state(run_id, state)
|
||||
|
||||
# Resume from failed stage
|
||||
return execute_from_stage(stage, state)
|
||||
```
|
||||
|
||||
## Output Summary
|
||||
|
||||
On completion, generate summary:
|
||||
|
||||
```json
|
||||
{
|
||||
"run_id": 123,
|
||||
"status": "completed",
|
||||
"duration_minutes": 15,
|
||||
"stats": {
|
||||
"sources_discovered": 5,
|
||||
"pages_crawled": 45,
|
||||
"documents_stored": 45,
|
||||
"documents_distilled": 45,
|
||||
"approved": 40,
|
||||
"refactored": 8,
|
||||
"deep_researched": 2,
|
||||
"rejected": 3,
|
||||
"needs_manual_review": 2
|
||||
},
|
||||
"exports": {
|
||||
"format": "project_files",
|
||||
"path": "~/reference-library/exports/",
|
||||
"document_count": 40
|
||||
},
|
||||
"errors": []
|
||||
}
|
||||
```
|
||||
|
||||
## Integration Points
|
||||
|
||||
| Skill | Called By | Provides |
|
||||
|-------|-----------|----------|
|
||||
| reference-discovery | Orchestrator | manifest.json |
|
||||
| web-crawler | Orchestrator | Raw crawled files |
|
||||
| content-repository | Orchestrator | Stored documents |
|
||||
| content-distiller | Orchestrator, QA loop | Distilled content |
|
||||
| quality-reviewer | Orchestrator | QA decisions |
|
||||
| markdown-exporter | Orchestrator | Final exports |
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `pipeline.py init` | Initialize a new pipeline run |
|
||||
| `pipeline.py advance` | Advance to next stage |
|
||||
| `pipeline.py pause` | Pause on error |
|
||||
| `pipeline.py resume` | Resume from pause |
|
||||
| `pipeline.py complete` | Mark pipeline complete |
|
||||
| `pipeline.py status` | Show run status |
|
||||
| `pipeline.py track-iteration` | Track QA loop iterations |
|
||||
|
||||
## Configuration
|
||||
|
||||
Read from `~/.config/reference-curator/pipeline_config.yaml`:
|
||||
Reads from `~/.config/reference-curator/pipeline_config.yaml`:
|
||||
|
||||
```yaml
|
||||
pipeline:
|
||||
@@ -288,9 +175,4 @@ qa_loop:
|
||||
|
||||
export:
|
||||
default_format: project_files
|
||||
include_rejected: false
|
||||
|
||||
state:
|
||||
backend: mysql # or 'file'
|
||||
state_directory: ~/reference-library/pipeline_state/
|
||||
```
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pipeline Orchestrator CLI — manage pipeline runs and state.
|
||||
|
||||
This script provides state management for the 6-stage pipeline.
|
||||
Claude orchestrates the actual stages via slash commands.
|
||||
|
||||
Usage:
|
||||
python pipeline.py init --input "prompt engineering" --type topic [--options '{"max_sources": 10}']
|
||||
python pipeline.py advance --run-id 1 --stage crawling [--stats '{"pages_crawled": 45}']
|
||||
python pipeline.py pause --run-id 1 --error "Crawl timeout" --stage crawling
|
||||
python pipeline.py resume --run-id 1
|
||||
python pipeline.py complete --run-id 1 [--export-path ~/reference-library/exports/]
|
||||
python pipeline.py status [--run-id 1]
|
||||
python pipeline.py track-iteration --run-id 1 --doc-id 42 --action refactor
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.config import get_pipeline_config
|
||||
|
||||
console = Console()
|
||||
|
||||
# QA loop limits
|
||||
MAX_REFACTOR = 3
|
||||
MAX_DEEP_RESEARCH = 2
|
||||
MAX_TOTAL = 5
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Pipeline Orchestrator — manage pipeline run state."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--input", "input_value", required=True, help="Topic, URL(s), or manifest path")
|
||||
@click.option("--type", "run_type", required=True,
|
||||
type=click.Choice(["topic", "urls", "manifest"]))
|
||||
@click.option("--options", type=str, help="Pipeline options as JSON string")
|
||||
def init(input_value, run_type, options):
|
||||
"""Initialize a new pipeline run."""
|
||||
opts = json.loads(options) if options else {}
|
||||
|
||||
# Merge with config defaults
|
||||
try:
|
||||
cfg = get_pipeline_config()
|
||||
defaults = cfg.get("pipeline", {})
|
||||
for key in ("max_sources", "max_pages", "auto_approve", "approval_threshold"):
|
||||
if key not in opts and key in defaults:
|
||||
opts[key] = defaults[key]
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
stats = {
|
||||
"sources_discovered": 0,
|
||||
"pages_crawled": 0,
|
||||
"documents_stored": 0,
|
||||
"documents_distilled": 0,
|
||||
"approved": 0,
|
||||
"refactored": 0,
|
||||
"deep_researched": 0,
|
||||
"rejected": 0,
|
||||
"needs_manual_review": 0,
|
||||
}
|
||||
|
||||
# Determine starting stage
|
||||
start_stage = "discovery" if run_type == "topic" else "crawling"
|
||||
|
||||
with db_session() as db:
|
||||
run_id = db.insert_returning_id(
|
||||
"""INSERT INTO pipeline_runs
|
||||
(run_type, input_value, status, current_stage, options, stats, started_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)""",
|
||||
(run_type, input_value, "running", start_stage,
|
||||
json.dumps(opts), json.dumps(stats), datetime.now().isoformat()),
|
||||
)
|
||||
|
||||
console.print(f"[green]Pipeline initialized:[/green] run_id={run_id}")
|
||||
console.print(f" Type: {run_type}")
|
||||
console.print(f" Input: {input_value}")
|
||||
console.print(f" Starting stage: {start_stage}")
|
||||
click.echo(json.dumps({"run_id": run_id, "status": "running", "stage": start_stage}))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--run-id", required=True, type=int, help="Pipeline run ID")
|
||||
@click.option("--stage", required=True,
|
||||
type=click.Choice(["discovery", "crawling", "storing", "evaluating",
|
||||
"distilling", "exporting"]))
|
||||
@click.option("--stats", type=str, help="Stats update as JSON string (merged with existing)")
|
||||
def advance(run_id, stage, stats):
|
||||
"""Advance pipeline to the next stage."""
|
||||
stats_update = json.loads(stats) if stats else {}
|
||||
|
||||
with db_session() as db:
|
||||
run = db.fetch_one(
|
||||
"SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,)
|
||||
)
|
||||
if not run:
|
||||
console.print(f"[red]Error:[/red] run_id={run_id} not found")
|
||||
sys.exit(1)
|
||||
|
||||
if run["status"] != "running":
|
||||
console.print(f"[red]Error:[/red] Pipeline is {run['status']}, not running")
|
||||
sys.exit(1)
|
||||
|
||||
# Merge stats
|
||||
current_stats = json.loads(run["stats"]) if isinstance(run["stats"], str) else (run["stats"] or {})
|
||||
current_stats.update(stats_update)
|
||||
|
||||
db.execute(
|
||||
"""UPDATE pipeline_runs
|
||||
SET current_stage = %s, stats = %s
|
||||
WHERE run_id = %s""",
|
||||
(stage, json.dumps(current_stats), run_id),
|
||||
)
|
||||
|
||||
stage_num = ["discovery", "crawling", "storing", "evaluating", "distilling", "exporting"].index(stage) + 1
|
||||
console.print(f"[green]Pipeline advanced:[/green] Stage {stage_num}/6 — {stage}")
|
||||
click.echo(json.dumps({"run_id": run_id, "stage": stage, "stats": current_stats}))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--run-id", required=True, type=int, help="Pipeline run ID")
|
||||
@click.option("--error", required=True, help="Error message")
|
||||
@click.option("--stage", required=True, help="Stage where error occurred")
|
||||
def pause(run_id, error, stage):
|
||||
"""Pause pipeline due to error."""
|
||||
with db_session() as db:
|
||||
db.execute(
|
||||
"""UPDATE pipeline_runs
|
||||
SET status = %s, error_message = %s, error_stage = %s
|
||||
WHERE run_id = %s""",
|
||||
("paused", error, stage, run_id),
|
||||
)
|
||||
|
||||
console.print(f"[yellow]Pipeline paused:[/yellow] run_id={run_id} at {stage}")
|
||||
console.print(f" Error: {error}")
|
||||
console.print(f" Resume with: python pipeline.py resume --run-id {run_id}")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--run-id", required=True, type=int, help="Pipeline run ID to resume")
|
||||
def resume(run_id):
|
||||
"""Resume a paused pipeline."""
|
||||
with db_session() as db:
|
||||
run = db.fetch_one(
|
||||
"SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,)
|
||||
)
|
||||
if not run:
|
||||
console.print(f"[red]Error:[/red] run_id={run_id} not found")
|
||||
sys.exit(1)
|
||||
|
||||
if run["status"] != "paused":
|
||||
console.print(f"[red]Error:[/red] Pipeline is {run['status']}, not paused")
|
||||
sys.exit(1)
|
||||
|
||||
db.execute(
|
||||
"""UPDATE pipeline_runs
|
||||
SET status = %s, error_message = %s
|
||||
WHERE run_id = %s""",
|
||||
("running", None, run_id),
|
||||
)
|
||||
|
||||
stage = run["current_stage"]
|
||||
console.print(f"[green]Pipeline resumed:[/green] run_id={run_id}, continuing from {stage}")
|
||||
click.echo(json.dumps({"run_id": run_id, "status": "running", "resume_stage": stage}))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--run-id", required=True, type=int, help="Pipeline run ID")
|
||||
@click.option("--export-path", type=click.Path(), help="Export output path")
|
||||
@click.option("--export-count", type=int, help="Number of documents exported")
|
||||
def complete(run_id, export_path, export_count):
|
||||
"""Mark pipeline as completed."""
|
||||
with db_session() as db:
|
||||
db.execute(
|
||||
"""UPDATE pipeline_runs
|
||||
SET status = %s, completed_at = %s, export_path = %s, export_document_count = %s
|
||||
WHERE run_id = %s""",
|
||||
("completed", datetime.now().isoformat(), export_path, export_count, run_id),
|
||||
)
|
||||
|
||||
run = db.fetch_one("SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,))
|
||||
|
||||
stats = json.loads(run["stats"]) if isinstance(run["stats"], str) else (run["stats"] or {})
|
||||
|
||||
console.print(f"\n[bold green]Pipeline Complete![/bold green] run_id={run_id}")
|
||||
console.print(f" Sources: {stats.get('sources_discovered', 0)}")
|
||||
console.print(f" Crawled: {stats.get('pages_crawled', 0)}")
|
||||
console.print(f" Stored: {stats.get('documents_stored', 0)}")
|
||||
console.print(f" Approved: {stats.get('approved', 0)}")
|
||||
console.print(f" Rejected: {stats.get('rejected', 0)}")
|
||||
if export_path:
|
||||
console.print(f" Exports: {export_path}")
|
||||
|
||||
click.echo(json.dumps({"run_id": run_id, "status": "completed", "stats": stats}, default=str))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--run-id", type=int, help="Show specific run (or latest if omitted)")
|
||||
@click.option("--all", "show_all", is_flag=True, help="Show all runs")
|
||||
def status(run_id, show_all):
|
||||
"""Show pipeline run status."""
|
||||
with db_session() as db:
|
||||
if run_id:
|
||||
rows = db.fetch_all(
|
||||
"SELECT * FROM pipeline_runs WHERE run_id = %s", (run_id,)
|
||||
)
|
||||
elif show_all:
|
||||
rows = db.fetch_all(
|
||||
"SELECT * FROM pipeline_runs ORDER BY started_at DESC LIMIT %s", (20,)
|
||||
)
|
||||
else:
|
||||
rows = db.fetch_all(
|
||||
"SELECT * FROM pipeline_runs ORDER BY started_at DESC LIMIT %s", (1,)
|
||||
)
|
||||
|
||||
if not rows:
|
||||
console.print("[dim]No pipeline runs found.[/dim]")
|
||||
return
|
||||
|
||||
for run in rows:
|
||||
stats = json.loads(run["stats"]) if isinstance(run["stats"], str) else (run["stats"] or {})
|
||||
status_color = {
|
||||
"running": "green", "completed": "blue",
|
||||
"failed": "red", "paused": "yellow",
|
||||
}.get(run["status"], "white")
|
||||
|
||||
console.print(f"\n[bold]Pipeline Run #{run['run_id']}[/bold]")
|
||||
console.print(f" Status: [{status_color}]{run['status']}[/{status_color}]")
|
||||
console.print(f" Type: {run['run_type']}")
|
||||
console.print(f" Input: {run['input_value']}")
|
||||
console.print(f" Stage: {run['current_stage']}")
|
||||
console.print(f" Started: {run['started_at']}")
|
||||
|
||||
if run.get("error_message"):
|
||||
console.print(f" [red]Error:[/red] {run['error_message']} (at {run.get('error_stage')})")
|
||||
|
||||
if stats:
|
||||
table = Table(title="Pipeline Stats", show_header=False)
|
||||
table.add_column("Metric")
|
||||
table.add_column("Value", justify="right")
|
||||
for k, v in stats.items():
|
||||
if v:
|
||||
table.add_row(k.replace("_", " ").title(), str(v))
|
||||
console.print(table)
|
||||
|
||||
|
||||
@cli.command("track-iteration")
|
||||
@click.option("--run-id", required=True, type=int, help="Pipeline run ID")
|
||||
@click.option("--doc-id", required=True, type=int, help="Document ID")
|
||||
@click.option("--action", required=True, type=click.Choice(["refactor", "deep_research"]),
|
||||
help="QA loop action")
|
||||
def track_iteration(run_id, doc_id, action):
|
||||
"""Track QA loop iteration for a document.
|
||||
|
||||
Returns the routing decision:
|
||||
- 're_distill': proceed with refactor
|
||||
- 're_crawl_and_distill': proceed with deep research
|
||||
- 'needs_manual_review': max iterations exceeded
|
||||
"""
|
||||
with db_session() as db:
|
||||
tracker = db.fetch_one(
|
||||
"""SELECT * FROM pipeline_iteration_tracker
|
||||
WHERE run_id = %s AND doc_id = %s""",
|
||||
(run_id, doc_id),
|
||||
)
|
||||
|
||||
if tracker:
|
||||
refactor_count = tracker.get("refactor_count", 0)
|
||||
deep_research_count = tracker.get("deep_research_count", 0)
|
||||
else:
|
||||
refactor_count = 0
|
||||
deep_research_count = 0
|
||||
|
||||
total = refactor_count + deep_research_count
|
||||
|
||||
# Check limits
|
||||
if total >= MAX_TOTAL:
|
||||
result = "needs_manual_review"
|
||||
elif action == "refactor" and refactor_count >= MAX_REFACTOR:
|
||||
result = "needs_manual_review"
|
||||
elif action == "deep_research" and deep_research_count >= MAX_DEEP_RESEARCH:
|
||||
result = "needs_manual_review"
|
||||
elif action == "refactor":
|
||||
refactor_count += 1
|
||||
result = "re_distill"
|
||||
else:
|
||||
deep_research_count += 1
|
||||
result = "re_crawl_and_distill"
|
||||
|
||||
# Upsert tracker
|
||||
if tracker:
|
||||
db.execute(
|
||||
"""UPDATE pipeline_iteration_tracker
|
||||
SET refactor_count = %s, deep_research_count = %s,
|
||||
final_decision = %s
|
||||
WHERE run_id = %s AND doc_id = %s""",
|
||||
(refactor_count, deep_research_count,
|
||||
"needs_manual_review" if result == "needs_manual_review" else None,
|
||||
run_id, doc_id),
|
||||
)
|
||||
else:
|
||||
db.insert_returning_id(
|
||||
"""INSERT INTO pipeline_iteration_tracker
|
||||
(run_id, doc_id, refactor_count, deep_research_count, final_decision)
|
||||
VALUES (%s, %s, %s, %s, %s)""",
|
||||
(run_id, doc_id, refactor_count, deep_research_count,
|
||||
"needs_manual_review" if result == "needs_manual_review" else None),
|
||||
)
|
||||
|
||||
console.print(
|
||||
f"[{'yellow' if result == 'needs_manual_review' else 'green'}]"
|
||||
f"Iteration tracked:[/] doc_id={doc_id}, {action} "
|
||||
f"(refactor: {refactor_count}/{MAX_REFACTOR}, "
|
||||
f"research: {deep_research_count}/{MAX_DEEP_RESEARCH}) → {result}"
|
||||
)
|
||||
click.echo(json.dumps({
|
||||
"run_id": run_id,
|
||||
"doc_id": doc_id,
|
||||
"action": action,
|
||||
"result": result,
|
||||
"refactor_count": refactor_count,
|
||||
"deep_research_count": deep_research_count,
|
||||
"total_iterations": refactor_count + deep_research_count,
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -51,6 +51,7 @@ COMMANDS=(
|
||||
"content-distiller"
|
||||
"quality-reviewer"
|
||||
"markdown-exporter"
|
||||
"reference-curator-pipeline"
|
||||
)
|
||||
|
||||
# ============================================================================
|
||||
@@ -541,6 +542,16 @@ check_status() {
|
||||
print_info "MySQL not configured or client not installed"
|
||||
fi
|
||||
|
||||
# Check Python package
|
||||
print_substep "Python Package (refcurator)"
|
||||
if uv run python3 -c "import refcurator; print(refcurator.__version__)" &>/dev/null; then
|
||||
local pkg_version=$(uv run python3 -c "import refcurator; print(refcurator.__version__)" 2>/dev/null)
|
||||
print_success "refcurator $pkg_version installed"
|
||||
else
|
||||
print_warning "refcurator package not installed — scripts will not work"
|
||||
all_ok=false
|
||||
fi
|
||||
|
||||
# Check Claude Code commands
|
||||
print_substep "Claude Code Commands ($CLAUDE_COMMANDS_DIR)"
|
||||
for cmd in "${COMMANDS[@]}"; do
|
||||
@@ -670,11 +681,43 @@ install() {
|
||||
setup_database
|
||||
fi
|
||||
|
||||
install_python_package
|
||||
register_commands
|
||||
register_skills
|
||||
post_install
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Install Python Package
|
||||
# ============================================================================
|
||||
|
||||
install_python_package() {
|
||||
print_step "Step 5b: Installing Python Package (refcurator)"
|
||||
|
||||
print_substep "Checking for uv package manager"
|
||||
if command -v uv &> /dev/null; then
|
||||
print_success "uv found: $(uv --version 2>/dev/null)"
|
||||
else
|
||||
print_warning "uv not found — install with: curl -LsSf https://astral.sh/uv/install.sh | sh"
|
||||
print_info "Skipping Python package installation"
|
||||
return 0
|
||||
fi
|
||||
|
||||
print_substep "Installing refcurator package"
|
||||
local pkg_dir="$SCRIPT_DIR/shared/lib"
|
||||
|
||||
if [[ -f "$pkg_dir/pyproject.toml" ]]; then
|
||||
if uv pip install -e "$pkg_dir" 2>/dev/null; then
|
||||
print_success "refcurator package installed (editable mode)"
|
||||
else
|
||||
print_warning "Failed to install refcurator — scripts may not work"
|
||||
print_info "Try manually: uv pip install -e $pkg_dir"
|
||||
fi
|
||||
else
|
||||
print_error "pyproject.toml not found at $pkg_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Minimal Installation (Firecrawl only)
|
||||
# ============================================================================
|
||||
@@ -712,6 +755,7 @@ EOF
|
||||
|
||||
install_configs
|
||||
create_directories
|
||||
install_python_package
|
||||
register_commands
|
||||
register_skills
|
||||
post_install
|
||||
|
||||
34
custom-skills/90-reference-curator/shared/lib/pyproject.toml
Normal file
34
custom-skills/90-reference-curator/shared/lib/pyproject.toml
Normal file
@@ -0,0 +1,34 @@
|
||||
[project]
|
||||
name = "refcurator"
|
||||
version = "1.0.0"
|
||||
description = "Reference Curator — data management for the reference curation pipeline"
|
||||
requires-python = ">=3.12"
|
||||
license = "MIT"
|
||||
authors = [
|
||||
{ name = "Andrew Yim", email = "andrew@ourdigital.org" },
|
||||
]
|
||||
dependencies = [
|
||||
"pymysql>=1.1.0",
|
||||
"click>=8.0",
|
||||
"pydantic>=2.0",
|
||||
"pyyaml>=6.0",
|
||||
"rich>=13.0",
|
||||
"python-dotenv>=1.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"ruff>=0.4",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Reference Curator — data management for the reference curation pipeline."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Configuration loading for the reference curator pipeline.
|
||||
|
||||
Loads YAML configs from ~/.config/reference-curator/ with env var substitution.
|
||||
Falls back to bundled defaults in shared/config/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load user env if present
|
||||
_env_file = Path.home() / ".reference-curator.env"
|
||||
if _env_file.is_file():
|
||||
load_dotenv(_env_file)
|
||||
|
||||
# Config search paths (user override → bundled defaults)
|
||||
USER_CONFIG_DIR = Path.home() / ".config" / "reference-curator"
|
||||
BUNDLED_CONFIG_DIR = Path(__file__).resolve().parents[4] / "config" # shared/config/
|
||||
|
||||
# Default storage paths
|
||||
DEFAULT_LIBRARY_PATH = Path(
|
||||
os.environ.get("REFERENCE_LIBRARY_PATH", "~/Documents/reference-library")
|
||||
).expanduser()
|
||||
DEFAULT_STATE_DIR = DEFAULT_LIBRARY_PATH / "pipeline_state"
|
||||
|
||||
|
||||
def _expand_env_vars(value: str) -> str:
|
||||
"""Expand ${VAR:-default} patterns in a string."""
|
||||
def _replace(match: re.Match) -> str:
|
||||
var_expr = match.group(1)
|
||||
if ":-" in var_expr:
|
||||
var_name, default = var_expr.split(":-", 1)
|
||||
return os.environ.get(var_name, default)
|
||||
return os.environ.get(var_expr, match.group(0))
|
||||
|
||||
return re.sub(r"\$\{([^}]+)}", _replace, value)
|
||||
|
||||
|
||||
def _expand_recursive(obj: Any) -> Any:
|
||||
"""Recursively expand env vars in a parsed YAML structure."""
|
||||
if isinstance(obj, str):
|
||||
expanded = _expand_env_vars(obj)
|
||||
# Expand ~ in path-like strings
|
||||
if expanded.startswith("~"):
|
||||
expanded = str(Path(expanded).expanduser())
|
||||
return expanded
|
||||
if isinstance(obj, dict):
|
||||
return {k: _expand_recursive(v) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_expand_recursive(item) for item in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def load_config(name: str) -> dict:
|
||||
"""Load a YAML config file by name (without extension).
|
||||
|
||||
Searches user config dir first, then bundled defaults.
|
||||
Expands ${VAR:-default} env var patterns in all string values.
|
||||
|
||||
Args:
|
||||
name: Config file name without .yaml extension
|
||||
(e.g., "db_config", "pipeline_config", "crawl_config", "export_config")
|
||||
|
||||
Returns:
|
||||
Parsed and expanded config dict.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If config file not found in any search path.
|
||||
"""
|
||||
filename = f"{name}.yaml"
|
||||
|
||||
for config_dir in [USER_CONFIG_DIR, BUNDLED_CONFIG_DIR]:
|
||||
config_path = config_dir / filename
|
||||
if config_path.is_file():
|
||||
with open(config_path) as f:
|
||||
raw = yaml.safe_load(f) or {}
|
||||
return _expand_recursive(raw)
|
||||
|
||||
raise FileNotFoundError(
|
||||
f"Config '{filename}' not found in {USER_CONFIG_DIR} or {BUNDLED_CONFIG_DIR}"
|
||||
)
|
||||
|
||||
|
||||
def get_db_config() -> dict:
|
||||
"""Load database configuration."""
|
||||
return load_config("db_config")
|
||||
|
||||
|
||||
def get_pipeline_config() -> dict:
|
||||
"""Load pipeline orchestrator configuration."""
|
||||
return load_config("pipeline_config")
|
||||
|
||||
|
||||
def get_crawl_config() -> dict:
|
||||
"""Load crawler configuration."""
|
||||
return load_config("crawl_config")
|
||||
|
||||
|
||||
def get_export_config() -> dict:
|
||||
"""Load export configuration."""
|
||||
return load_config("export_config")
|
||||
|
||||
|
||||
def get_library_path() -> Path:
|
||||
"""Get the reference library base path."""
|
||||
return DEFAULT_LIBRARY_PATH
|
||||
|
||||
|
||||
def get_state_dir() -> Path:
|
||||
"""Get the pipeline state directory."""
|
||||
try:
|
||||
cfg = get_pipeline_config()
|
||||
state_dir = cfg.get("state", {}).get("state_directory")
|
||||
if state_dir:
|
||||
return Path(state_dir).expanduser()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return DEFAULT_STATE_DIR
|
||||
|
||||
|
||||
def get_state_backend() -> str:
|
||||
"""Get the state backend type: 'mysql' or 'file'."""
|
||||
try:
|
||||
cfg = get_pipeline_config()
|
||||
return cfg.get("state", {}).get("backend", "file")
|
||||
except FileNotFoundError:
|
||||
return "file"
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Database abstraction layer with MySQL and file-based backends.
|
||||
|
||||
Usage:
|
||||
from refcurator.db import get_backend
|
||||
|
||||
db = get_backend()
|
||||
rows = db.fetch_all("SELECT * FROM documents WHERE crawl_status = %s", ("pending",))
|
||||
doc_id = db.insert_returning_id("INSERT INTO documents (...) VALUES (...)", params)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Protocol, Sequence
|
||||
|
||||
from refcurator.config import get_db_config, get_state_backend, get_state_dir
|
||||
|
||||
logger = logging.getLogger("refcurator.db")
|
||||
|
||||
|
||||
class DatabaseBackend(Protocol):
|
||||
"""Protocol for database backends."""
|
||||
|
||||
def execute(self, sql: str, params: Sequence = ()) -> int:
|
||||
"""Execute a statement, return affected row count."""
|
||||
...
|
||||
|
||||
def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]:
|
||||
"""Fetch a single row as dict."""
|
||||
...
|
||||
|
||||
def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]:
|
||||
"""Fetch all rows as list of dicts."""
|
||||
...
|
||||
|
||||
def insert_returning_id(self, sql: str, params: Sequence = ()) -> int:
|
||||
"""Insert a row and return the auto-generated ID."""
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
...
|
||||
|
||||
|
||||
class MySQLBackend:
|
||||
"""MySQL backend using PyMySQL."""
|
||||
|
||||
def __init__(self, config: dict | None = None):
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
|
||||
if config is None:
|
||||
config = get_db_config().get("mysql", {})
|
||||
|
||||
self._conn = pymysql.connect(
|
||||
host=config.get("host", "localhost"),
|
||||
port=int(config.get("port", 3306)),
|
||||
user=config.get("user", "root"),
|
||||
password=config.get("password", ""),
|
||||
database=config.get("database", "reference_library"),
|
||||
charset="utf8mb4",
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
autocommit=True,
|
||||
)
|
||||
|
||||
def execute(self, sql: str, params: Sequence = ()) -> int:
|
||||
with self._conn.cursor() as cur:
|
||||
return cur.execute(sql, params)
|
||||
|
||||
def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.fetchone()
|
||||
|
||||
def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.fetchall()
|
||||
|
||||
def insert_returning_id(self, sql: str, params: Sequence = ()) -> int:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.lastrowid
|
||||
|
||||
def close(self) -> None:
|
||||
self._conn.close()
|
||||
|
||||
|
||||
class FileBackend:
|
||||
"""JSON file-based backend for use without MySQL.
|
||||
|
||||
Stores data as JSON arrays in the state directory.
|
||||
Supports basic CRUD but not complex queries or JOINs.
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: Path | None = None):
|
||||
self._dir = state_dir or get_state_dir()
|
||||
self._dir.mkdir(parents=True, exist_ok=True)
|
||||
self._cache: dict[str, list[dict]] = {}
|
||||
self._counters: dict[str, int] = {}
|
||||
self._load_counters()
|
||||
|
||||
def _table_path(self, table: str) -> Path:
|
||||
return self._dir / f"{table}.json"
|
||||
|
||||
def _load_table(self, table: str) -> list[dict]:
|
||||
if table not in self._cache:
|
||||
path = self._table_path(table)
|
||||
if path.is_file():
|
||||
self._cache[table] = json.loads(path.read_text())
|
||||
else:
|
||||
self._cache[table] = []
|
||||
return self._cache[table]
|
||||
|
||||
def _save_table(self, table: str) -> None:
|
||||
path = self._table_path(table)
|
||||
path.write_text(json.dumps(self._cache.get(table, []), indent=2, default=str))
|
||||
|
||||
def _load_counters(self) -> None:
|
||||
counter_path = self._dir / "_counters.json"
|
||||
if counter_path.is_file():
|
||||
self._counters = json.loads(counter_path.read_text())
|
||||
|
||||
def _save_counters(self) -> None:
|
||||
counter_path = self._dir / "_counters.json"
|
||||
counter_path.write_text(json.dumps(self._counters, indent=2))
|
||||
|
||||
def _next_id(self, table: str) -> int:
|
||||
current = self._counters.get(table, 0)
|
||||
self._counters[table] = current + 1
|
||||
self._save_counters()
|
||||
return current + 1
|
||||
|
||||
# --- Protocol methods ---
|
||||
# These provide basic support for the most common operations.
|
||||
# Complex SQL is not supported; use MySQL for full functionality.
|
||||
|
||||
def execute(self, sql: str, params: Sequence = ()) -> int:
|
||||
"""Basic INSERT/UPDATE/DELETE support via SQL pattern matching."""
|
||||
sql_lower = sql.strip().lower()
|
||||
table = _extract_table_name(sql)
|
||||
|
||||
if sql_lower.startswith("insert"):
|
||||
return self._handle_insert(table, sql, params)
|
||||
elif sql_lower.startswith("update"):
|
||||
return self._handle_update(table, sql, params)
|
||||
elif sql_lower.startswith("delete"):
|
||||
return self._handle_delete(table, sql, params)
|
||||
|
||||
logger.warning("FileBackend: unsupported SQL operation: %s", sql[:60])
|
||||
return 0
|
||||
|
||||
def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]:
|
||||
rows = self.fetch_all(sql, params)
|
||||
return rows[0] if rows else None
|
||||
|
||||
def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]:
|
||||
table = _extract_table_name(sql)
|
||||
if not table:
|
||||
logger.warning("FileBackend: cannot extract table from: %s", sql[:60])
|
||||
return []
|
||||
|
||||
# Hard-fail on SQL patterns that FileBackend cannot handle correctly
|
||||
_reject_unsupported_sql(sql)
|
||||
|
||||
rows = self._load_table(table)
|
||||
|
||||
# Basic WHERE clause filtering
|
||||
conditions = _extract_where_conditions(sql, params)
|
||||
if conditions:
|
||||
rows = [r for r in rows if _matches_conditions(r, conditions)]
|
||||
|
||||
# Basic ORDER BY
|
||||
order_col = _extract_order_by(sql)
|
||||
if order_col:
|
||||
desc = "desc" in sql.lower().split("order by")[-1].lower()
|
||||
rows = sorted(rows, key=lambda r: r.get(order_col, ""), reverse=desc)
|
||||
|
||||
# Basic LIMIT
|
||||
limit = _extract_limit(sql)
|
||||
if limit is not None:
|
||||
rows = rows[:limit]
|
||||
|
||||
return rows
|
||||
|
||||
def insert_returning_id(self, sql: str, params: Sequence = ()) -> int:
|
||||
table = _extract_table_name(sql)
|
||||
self._handle_insert(table, sql, params)
|
||||
return self._counters.get(table, 0)
|
||||
|
||||
def close(self) -> None:
|
||||
pass # No connection to close
|
||||
|
||||
# --- Internal handlers ---
|
||||
|
||||
def _handle_insert(self, table: str, sql: str, params: Sequence) -> int:
|
||||
columns = _extract_insert_columns(sql)
|
||||
if not columns or len(columns) != len(params):
|
||||
logger.warning("FileBackend: column/param mismatch for INSERT into %s", table)
|
||||
return 0
|
||||
|
||||
row = dict(zip(columns, params))
|
||||
pk = _primary_key_for(table)
|
||||
if pk and pk not in row:
|
||||
row[pk] = self._next_id(table)
|
||||
|
||||
rows = self._load_table(table)
|
||||
rows.append(row)
|
||||
self._cache[table] = rows
|
||||
self._save_table(table)
|
||||
return 1
|
||||
|
||||
def _handle_update(self, table: str, sql: str, params: Sequence) -> int:
|
||||
rows = self._load_table(table)
|
||||
set_cols = _extract_set_columns(sql)
|
||||
conditions = _extract_where_conditions(sql, params[len(set_cols):])
|
||||
set_values = list(params[:len(set_cols)])
|
||||
|
||||
count = 0
|
||||
for row in rows:
|
||||
if _matches_conditions(row, conditions):
|
||||
for col, val in zip(set_cols, set_values):
|
||||
row[col] = val
|
||||
count += 1
|
||||
|
||||
if count > 0:
|
||||
self._save_table(table)
|
||||
return count
|
||||
|
||||
def _handle_delete(self, table: str, sql: str, params: Sequence) -> int:
|
||||
rows = self._load_table(table)
|
||||
conditions = _extract_where_conditions(sql, params)
|
||||
before = len(rows)
|
||||
self._cache[table] = [r for r in rows if not _matches_conditions(r, conditions)]
|
||||
self._save_table(table)
|
||||
return before - len(self._cache[table])
|
||||
|
||||
|
||||
class UnsupportedQueryError(Exception):
|
||||
"""Raised when FileBackend encounters SQL it cannot handle correctly."""
|
||||
pass
|
||||
|
||||
|
||||
def _reject_unsupported_sql(sql: str) -> None:
|
||||
"""Raise UnsupportedQueryError if the SQL uses patterns FileBackend cannot handle.
|
||||
|
||||
FileBackend only supports single-table SELECT with simple WHERE col = %s.
|
||||
JOINs, subqueries, aggregates, and GROUP BY would return wrong results silently.
|
||||
"""
|
||||
import re
|
||||
sql_upper = sql.upper()
|
||||
|
||||
unsupported = []
|
||||
if re.search(r"\bJOIN\b", sql_upper):
|
||||
unsupported.append("JOIN")
|
||||
if re.search(r"\bGROUP\s+BY\b", sql_upper):
|
||||
unsupported.append("GROUP BY")
|
||||
if re.search(r"\b(MAX|MIN|SUM|AVG|COUNT)\s*\(", sql_upper):
|
||||
unsupported.append("aggregate functions")
|
||||
if re.search(r"\bLEFT\s+JOIN\b", sql_upper):
|
||||
unsupported.append("LEFT JOIN")
|
||||
if re.search(r"\(\s*SELECT\b", sql_upper):
|
||||
unsupported.append("subquery")
|
||||
|
||||
if unsupported:
|
||||
raise UnsupportedQueryError(
|
||||
f"FileBackend cannot execute queries with {', '.join(unsupported)}. "
|
||||
f"Configure MySQL or use 'backend: mysql' in pipeline_config.yaml. "
|
||||
f"Query: {sql[:80]}..."
|
||||
)
|
||||
|
||||
|
||||
# --- SQL parsing helpers (minimal, covers common patterns) ---
|
||||
|
||||
def _extract_table_name(sql: str) -> str:
|
||||
"""Extract the primary table name from a SQL statement."""
|
||||
import re
|
||||
sql_clean = sql.strip()
|
||||
|
||||
# INSERT INTO table
|
||||
m = re.search(r"(?:insert\s+into|update|delete\s+from|from)\s+(\w+)", sql_clean, re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_insert_columns(sql: str) -> list[str]:
|
||||
"""Extract column names from INSERT INTO table (col1, col2, ...)."""
|
||||
import re
|
||||
m = re.search(r"\(([^)]+)\)\s*VALUES", sql, re.I)
|
||||
if m:
|
||||
return [c.strip() for c in m.group(1).split(",")]
|
||||
return []
|
||||
|
||||
|
||||
def _extract_set_columns(sql: str) -> list[str]:
|
||||
"""Extract column names from UPDATE ... SET col1 = %s, col2 = %s."""
|
||||
import re
|
||||
m = re.search(r"SET\s+(.+?)(?:\s+WHERE|$)", sql, re.I | re.S)
|
||||
if m:
|
||||
return [c.strip().split("=")[0].strip() for c in m.group(1).split(",")]
|
||||
return []
|
||||
|
||||
|
||||
def _extract_where_conditions(sql: str, params: Sequence) -> list[tuple[str, Any]]:
|
||||
"""Extract simple col = %s conditions from WHERE clause."""
|
||||
import re
|
||||
m = re.search(r"WHERE\s+(.+?)(?:\s+ORDER|\s+LIMIT|$)", sql, re.I | re.S)
|
||||
if not m:
|
||||
return []
|
||||
|
||||
where_clause = m.group(1)
|
||||
cols = re.findall(r"(\w+)\s*=\s*%s", where_clause)
|
||||
# Map to params (taking from the end of params for UPDATE, all for SELECT)
|
||||
return list(zip(cols, params[-len(cols):] if cols else []))
|
||||
|
||||
|
||||
def _extract_order_by(sql: str) -> str:
|
||||
"""Extract the first ORDER BY column."""
|
||||
import re
|
||||
m = re.search(r"ORDER\s+BY\s+(\w+)", sql, re.I)
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
def _extract_limit(sql: str) -> int | None:
|
||||
"""Extract LIMIT value."""
|
||||
import re
|
||||
m = re.search(r"LIMIT\s+(\d+)", sql, re.I)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _matches_conditions(row: dict, conditions: list[tuple[str, Any]]) -> bool:
|
||||
"""Check if a row matches all WHERE conditions."""
|
||||
return all(str(row.get(col)) == str(val) for col, val in conditions)
|
||||
|
||||
|
||||
def _primary_key_for(table: str) -> str:
|
||||
"""Return the auto-increment primary key name for a table."""
|
||||
pk_map = {
|
||||
"sources": "source_id",
|
||||
"documents": "doc_id",
|
||||
"distilled_content": "distill_id",
|
||||
"review_logs": "review_id",
|
||||
"topics": "topic_id",
|
||||
"export_jobs": "export_id",
|
||||
"pipeline_runs": "run_id",
|
||||
"pipeline_iteration_tracker": "tracker_id",
|
||||
"crawl_schedule": "schedule_id",
|
||||
"change_detection": "change_id",
|
||||
}
|
||||
return pk_map.get(table, "")
|
||||
|
||||
|
||||
# --- Factory ---
|
||||
|
||||
def get_backend(backend_type: str | None = None) -> DatabaseBackend:
|
||||
"""Create and return the appropriate database backend.
|
||||
|
||||
Args:
|
||||
backend_type: 'mysql' or 'file'. If None, reads from pipeline config.
|
||||
|
||||
Returns:
|
||||
A DatabaseBackend instance.
|
||||
"""
|
||||
if backend_type is None:
|
||||
backend_type = get_state_backend()
|
||||
|
||||
if backend_type == "mysql":
|
||||
return MySQLBackend()
|
||||
|
||||
return FileBackend()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db_session(backend_type: str | None = None):
|
||||
"""Context manager for database sessions.
|
||||
|
||||
Usage:
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all("SELECT * FROM documents")
|
||||
"""
|
||||
db = get_backend(backend_type)
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Gemini CLI wrapper for independent content quality evaluation.
|
||||
|
||||
Uses the Gemini CLI (google/gemini-cli) to evaluate raw crawled content
|
||||
before distillation, providing third-party quality assessment.
|
||||
|
||||
Requires: `npm install -g @google/gemini-cli` and Google auth configured.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("refcurator.gemini")
|
||||
|
||||
GEMINI_CMD = "gemini"
|
||||
TIMEOUT_SECONDS = 60
|
||||
|
||||
EVALUATION_PROMPT = """You are evaluating a raw reference document for inclusion in a curated knowledge base.
|
||||
|
||||
Topic: {topic}
|
||||
Source URL: {source_url}
|
||||
|
||||
Score each criterion from 0.0 to 1.0:
|
||||
- relevance: Does this content actually relate to the topic "{topic}"?
|
||||
- authority: Is this from an authoritative, official source (official docs, research paper) or low-quality (scraped blog, forum post, SEO spam)?
|
||||
- completeness: Is this a complete article with substance, or a navigation fragment, error page, stub, or boilerplate?
|
||||
- freshness: Does the information appear current and not outdated? Look for version numbers, dates, deprecated APIs.
|
||||
- distill_value: Does this contain unique, valuable information worth summarizing, or is it redundant with what official docs already cover?
|
||||
|
||||
Return ONLY a valid JSON object with no markdown formatting, no code fences, no explanation:
|
||||
{{"relevance": 0.0, "authority": 0.0, "completeness": 0.0, "freshness": 0.0, "distill_value": 0.0, "verdict": "approve", "reason": "brief explanation"}}
|
||||
|
||||
The verdict must be one of: "approve", "reject", "deep_research"
|
||||
- approve: source is worth distilling (score >= 0.75 typical)
|
||||
- reject: not worth distilling (low quality, irrelevant, or fragment)
|
||||
- deep_research: partially relevant but needs supplementary sources"""
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Check if the Gemini CLI is installed and authenticated."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[GEMINI_CMD, "--help"],
|
||||
capture_output=True, timeout=10,
|
||||
)
|
||||
return result.returncode == 0
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_content(
|
||||
content: str,
|
||||
topic: str,
|
||||
source_url: str = "",
|
||||
timeout: int = TIMEOUT_SECONDS,
|
||||
) -> Optional[dict]:
|
||||
"""Evaluate raw content using Gemini CLI.
|
||||
|
||||
Args:
|
||||
content: Raw document content (markdown/text)
|
||||
topic: The curation topic for relevance scoring
|
||||
source_url: Original URL of the content
|
||||
timeout: Subprocess timeout in seconds
|
||||
|
||||
Returns:
|
||||
Parsed evaluation dict with scores, verdict, and reason.
|
||||
Returns None if Gemini is unavailable or evaluation fails.
|
||||
"""
|
||||
# Truncate very long content to avoid overwhelming the model
|
||||
max_chars = 50_000
|
||||
if len(content) > max_chars:
|
||||
content = content[:max_chars] + "\n\n[... content truncated for evaluation ...]"
|
||||
|
||||
prompt = EVALUATION_PROMPT.format(topic=topic, source_url=source_url)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[GEMINI_CMD, prompt],
|
||||
input=content,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.warning("Gemini CLI failed (exit %d): %s", result.returncode, result.stderr[:200])
|
||||
return None
|
||||
|
||||
return _parse_response(result.stdout)
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.warning("Gemini CLI not found. Install with: npm install -g @google/gemini-cli")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Gemini CLI timed out after %ds", timeout)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("Gemini evaluation failed: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_response(output: str) -> Optional[dict]:
|
||||
"""Parse Gemini's response, handling markdown-wrapped JSON."""
|
||||
text = output.strip()
|
||||
|
||||
# Try direct JSON parse first
|
||||
try:
|
||||
return _validate_evaluation(json.loads(text))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Try extracting JSON from markdown code fences
|
||||
m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
|
||||
if m:
|
||||
try:
|
||||
return _validate_evaluation(json.loads(m.group(1).strip()))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Try finding a JSON object anywhere in the output
|
||||
m = re.search(r"\{[^{}]*\"relevance\"[^{}]*\}", text, re.DOTALL)
|
||||
if m:
|
||||
try:
|
||||
return _validate_evaluation(json.loads(m.group(0)))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
logger.warning("Could not parse Gemini response as JSON: %s", text[:200])
|
||||
return None
|
||||
|
||||
|
||||
def _validate_evaluation(data: dict) -> Optional[dict]:
|
||||
"""Validate that the evaluation has required fields and reasonable values."""
|
||||
required_scores = ["relevance", "authority", "completeness", "freshness", "distill_value"]
|
||||
|
||||
for field in required_scores:
|
||||
if field not in data:
|
||||
logger.warning("Missing required field: %s", field)
|
||||
return None
|
||||
score = data[field]
|
||||
if not isinstance(score, (int, float)) or score < 0 or score > 1:
|
||||
logger.warning("Invalid score for %s: %s", field, score)
|
||||
return None
|
||||
|
||||
if "verdict" not in data or data["verdict"] not in ("approve", "reject", "deep_research"):
|
||||
data["verdict"] = _derive_verdict(data)
|
||||
|
||||
if "reason" not in data:
|
||||
data["reason"] = ""
|
||||
|
||||
# Add weighted score
|
||||
data["weighted_score"] = round(
|
||||
data["relevance"] * 0.25
|
||||
+ data["authority"] * 0.25
|
||||
+ data["completeness"] * 0.20
|
||||
+ data["freshness"] * 0.15
|
||||
+ data["distill_value"] * 0.15,
|
||||
4,
|
||||
)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _derive_verdict(data: dict) -> str:
|
||||
"""Derive verdict from scores if Gemini didn't provide one."""
|
||||
score = (
|
||||
data.get("relevance", 0) * 0.25
|
||||
+ data.get("authority", 0) * 0.25
|
||||
+ data.get("completeness", 0) * 0.20
|
||||
+ data.get("freshness", 0) * 0.15
|
||||
+ data.get("distill_value", 0) * 0.15
|
||||
)
|
||||
if score >= 0.75:
|
||||
return "approve"
|
||||
elif score >= 0.50:
|
||||
return "deep_research"
|
||||
else:
|
||||
return "reject"
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Manifest I/O for reference discovery and crawl results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from refcurator.models import CrawlResult, CrawlResultEntry, Manifest, ManifestURL
|
||||
from refcurator.utils import normalize_url
|
||||
|
||||
|
||||
def read_manifest(path: Path) -> Manifest:
|
||||
"""Read a manifest JSON file."""
|
||||
data = json.loads(path.read_text())
|
||||
return Manifest(**data)
|
||||
|
||||
|
||||
def write_manifest(manifest: Manifest, path: Path) -> None:
|
||||
"""Write a manifest to a JSON file."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(manifest.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def merge_manifests(manifests: list[Manifest]) -> Manifest:
|
||||
"""Merge multiple manifests, deduplicating URLs."""
|
||||
seen: dict[str, ManifestURL] = {}
|
||||
topic_parts = []
|
||||
|
||||
for m in manifests:
|
||||
if m.topic:
|
||||
topic_parts.append(m.topic)
|
||||
for url_entry in m.urls:
|
||||
normalized = normalize_url(url_entry.url)
|
||||
existing = seen.get(normalized)
|
||||
if existing is None or (
|
||||
url_entry.credibility_score
|
||||
and (existing.credibility_score or 0) < url_entry.credibility_score
|
||||
):
|
||||
seen[normalized] = url_entry
|
||||
|
||||
urls = list(seen.values())
|
||||
return Manifest(
|
||||
discovery_date=datetime.now().isoformat(),
|
||||
topic=" + ".join(topic_parts) if topic_parts else None,
|
||||
total_urls=len(urls),
|
||||
urls=urls,
|
||||
)
|
||||
|
||||
|
||||
def dedup_manifest_urls(manifest: Manifest, existing_urls: set[str]) -> Manifest:
|
||||
"""Remove URLs already in the existing set (normalized comparison)."""
|
||||
existing_normalized = {normalize_url(u) for u in existing_urls}
|
||||
filtered = [u for u in manifest.urls if normalize_url(u.url) not in existing_normalized]
|
||||
return Manifest(
|
||||
discovery_date=manifest.discovery_date,
|
||||
topic=manifest.topic,
|
||||
total_urls=len(filtered),
|
||||
urls=filtered,
|
||||
)
|
||||
|
||||
|
||||
def read_crawl_result(path: Path) -> CrawlResult:
|
||||
"""Read a crawl result JSON file."""
|
||||
data = json.loads(path.read_text())
|
||||
return CrawlResult(**data)
|
||||
|
||||
|
||||
def write_crawl_result(result: CrawlResult, path: Path) -> None:
|
||||
"""Write a crawl result to a JSON file."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(result.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def create_crawl_result(
|
||||
entries: list[dict],
|
||||
crawler: str = "firecrawl",
|
||||
) -> CrawlResult:
|
||||
"""Create a CrawlResult from a list of crawl entry dicts."""
|
||||
docs = [CrawlResultEntry(**e) for e in entries]
|
||||
completed = [d for d in docs if d.status == "completed"]
|
||||
failed = [d for d in docs if d.status != "completed"]
|
||||
|
||||
return CrawlResult(
|
||||
crawl_date=datetime.now().isoformat(),
|
||||
crawler_used=crawler,
|
||||
total_crawled=len(completed),
|
||||
total_failed=len(failed),
|
||||
documents=docs,
|
||||
)
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Pydantic v2 models matching the reference_library MySQL schema."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, computed_field
|
||||
|
||||
|
||||
# --- Enums matching MySQL ENUMs ---
|
||||
|
||||
class SourceType(str, Enum):
|
||||
official_docs = "official_docs"
|
||||
engineering_blog = "engineering_blog"
|
||||
research_paper = "research_paper"
|
||||
github_repo = "github_repo"
|
||||
community_guide = "community_guide"
|
||||
pdf_document = "pdf_document"
|
||||
api_reference = "api_reference"
|
||||
|
||||
|
||||
class CredibilityTier(str, Enum):
|
||||
tier1_official = "tier1_official"
|
||||
tier2_verified = "tier2_verified"
|
||||
tier3_community = "tier3_community"
|
||||
|
||||
|
||||
class DocType(str, Enum):
|
||||
webpage = "webpage"
|
||||
pdf = "pdf"
|
||||
markdown = "markdown"
|
||||
api_spec = "api_spec"
|
||||
code_sample = "code_sample"
|
||||
|
||||
|
||||
class Language(str, Enum):
|
||||
en = "en"
|
||||
ko = "ko"
|
||||
mixed = "mixed"
|
||||
|
||||
|
||||
class CrawlMethod(str, Enum):
|
||||
firecrawl = "firecrawl"
|
||||
scrapy = "scrapy"
|
||||
aiohttp = "aiohttp"
|
||||
nodejs = "nodejs"
|
||||
manual = "manual"
|
||||
api = "api"
|
||||
|
||||
|
||||
class CrawlStatus(str, Enum):
|
||||
pending = "pending"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
stale = "stale"
|
||||
|
||||
|
||||
class ReviewStatus(str, Enum):
|
||||
pending = "pending"
|
||||
in_review = "in_review"
|
||||
approved = "approved"
|
||||
needs_refactor = "needs_refactor"
|
||||
rejected = "rejected"
|
||||
|
||||
|
||||
class ReviewerType(str, Enum):
|
||||
auto_qa = "auto_qa"
|
||||
human = "human"
|
||||
claude_review = "claude_review"
|
||||
gemini_review = "gemini_review"
|
||||
|
||||
|
||||
class Decision(str, Enum):
|
||||
approve = "approve"
|
||||
refactor = "refactor"
|
||||
deep_research = "deep_research"
|
||||
reject = "reject"
|
||||
|
||||
|
||||
class PipelineStatus(str, Enum):
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
paused = "paused"
|
||||
|
||||
|
||||
class PipelineStage(str, Enum):
|
||||
discovery = "discovery"
|
||||
crawling = "crawling"
|
||||
storing = "storing"
|
||||
evaluating = "evaluating"
|
||||
distilling = "distilling"
|
||||
exporting = "exporting"
|
||||
|
||||
|
||||
class RunType(str, Enum):
|
||||
topic = "topic"
|
||||
urls = "urls"
|
||||
manifest = "manifest"
|
||||
|
||||
|
||||
class ExportType(str, Enum):
|
||||
project_files = "project_files"
|
||||
fine_tuning = "fine_tuning"
|
||||
training_dataset = "training_dataset"
|
||||
knowledge_base = "knowledge_base"
|
||||
|
||||
|
||||
class OutputFormat(str, Enum):
|
||||
markdown = "markdown"
|
||||
jsonl = "jsonl"
|
||||
parquet = "parquet"
|
||||
sqlite = "sqlite"
|
||||
|
||||
|
||||
class Frequency(str, Enum):
|
||||
daily = "daily"
|
||||
weekly = "weekly"
|
||||
biweekly = "biweekly"
|
||||
monthly = "monthly"
|
||||
on_demand = "on_demand"
|
||||
|
||||
|
||||
class ChangeType(str, Enum):
|
||||
content_updated = "content_updated"
|
||||
url_moved = "url_moved"
|
||||
deleted = "deleted"
|
||||
new_version = "new_version"
|
||||
|
||||
|
||||
class FinalDecision(str, Enum):
|
||||
approved = "approved"
|
||||
rejected = "rejected"
|
||||
needs_manual_review = "needs_manual_review"
|
||||
|
||||
|
||||
# --- Core Table Models ---
|
||||
|
||||
class Source(BaseModel):
|
||||
source_id: Optional[int] = None
|
||||
source_name: str
|
||||
source_type: SourceType
|
||||
base_url: Optional[str] = None
|
||||
credibility_tier: CredibilityTier = CredibilityTier.tier3_community
|
||||
vendor: Optional[str] = None
|
||||
is_active: bool = True
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class Document(BaseModel):
|
||||
doc_id: Optional[int] = None
|
||||
source_id: int
|
||||
title: str
|
||||
url: Optional[str] = None
|
||||
url_hash: Optional[str] = None # Generated column in MySQL
|
||||
doc_type: DocType
|
||||
language: Language = Language.en
|
||||
original_publish_date: Optional[date] = None
|
||||
last_modified_date: Optional[date] = None
|
||||
crawl_date: Optional[datetime] = None
|
||||
crawl_method: CrawlMethod = CrawlMethod.firecrawl
|
||||
crawl_status: CrawlStatus = CrawlStatus.pending
|
||||
raw_content_path: Optional[str] = None
|
||||
raw_content_size: Optional[int] = None
|
||||
version: int = 1
|
||||
previous_version_id: Optional[int] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class DistilledContent(BaseModel):
|
||||
distill_id: Optional[int] = None
|
||||
doc_id: int
|
||||
summary: Optional[str] = None
|
||||
key_concepts: Optional[list[dict[str, Any]]] = None
|
||||
code_snippets: Optional[list[dict[str, Any]]] = None
|
||||
structured_content: Optional[str] = None
|
||||
token_count_original: Optional[int] = None
|
||||
token_count_distilled: Optional[int] = None
|
||||
distill_model: Optional[str] = None
|
||||
distill_date: Optional[datetime] = None
|
||||
review_status: ReviewStatus = ReviewStatus.pending
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def compression_ratio(self) -> Optional[float]:
|
||||
if self.token_count_original and self.token_count_distilled:
|
||||
return round(self.token_count_distilled / self.token_count_original * 100, 2)
|
||||
return None
|
||||
|
||||
|
||||
class ReviewLog(BaseModel):
|
||||
review_id: Optional[int] = None
|
||||
distill_id: int
|
||||
review_round: int = 1
|
||||
reviewer_type: ReviewerType
|
||||
quality_score: Optional[float] = None
|
||||
assessment: Optional[dict[str, float]] = None
|
||||
decision: Decision
|
||||
feedback: Optional[str] = None
|
||||
refactor_instructions: Optional[str] = None
|
||||
research_queries: Optional[list[str]] = None
|
||||
reviewed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class Topic(BaseModel):
|
||||
topic_id: Optional[int] = None
|
||||
topic_name: str
|
||||
topic_slug: str
|
||||
parent_topic_id: Optional[int] = None
|
||||
description: Optional[str] = None
|
||||
|
||||
|
||||
class DocumentTopic(BaseModel):
|
||||
doc_id: int
|
||||
topic_id: int
|
||||
relevance_score: float = 1.0
|
||||
|
||||
|
||||
class ExportJob(BaseModel):
|
||||
export_id: Optional[int] = None
|
||||
export_name: str
|
||||
export_type: ExportType
|
||||
output_format: OutputFormat = OutputFormat.markdown
|
||||
topic_filter: Optional[list[int]] = None
|
||||
date_range_start: Optional[date] = None
|
||||
date_range_end: Optional[date] = None
|
||||
min_quality_score: float = 0.80
|
||||
output_path: Optional[str] = None
|
||||
total_documents: Optional[int] = None
|
||||
total_tokens: Optional[int] = None
|
||||
status: str = "pending"
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class PipelineRun(BaseModel):
|
||||
run_id: Optional[int] = None
|
||||
run_type: RunType
|
||||
input_value: str
|
||||
status: PipelineStatus = PipelineStatus.running
|
||||
current_stage: PipelineStage = PipelineStage.discovery
|
||||
options: Optional[dict[str, Any]] = None
|
||||
stats: Optional[dict[str, int]] = Field(default_factory=lambda: {
|
||||
"sources_discovered": 0,
|
||||
"pages_crawled": 0,
|
||||
"documents_stored": 0,
|
||||
"documents_distilled": 0,
|
||||
"approved": 0,
|
||||
"refactored": 0,
|
||||
"deep_researched": 0,
|
||||
"rejected": 0,
|
||||
"needs_manual_review": 0,
|
||||
})
|
||||
export_path: Optional[str] = None
|
||||
export_document_count: Optional[int] = None
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
error_stage: Optional[str] = None
|
||||
|
||||
|
||||
class PipelineIterationTracker(BaseModel):
|
||||
tracker_id: Optional[int] = None
|
||||
run_id: int
|
||||
doc_id: int
|
||||
refactor_count: int = 0
|
||||
deep_research_count: int = 0
|
||||
final_decision: Optional[FinalDecision] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
# --- Non-DB Models (manifest/crawl/assessment) ---
|
||||
|
||||
class ManifestURL(BaseModel):
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
credibility_tier: Optional[str] = None
|
||||
credibility_score: Optional[float] = None
|
||||
source_type: Optional[str] = None
|
||||
vendor: Optional[str] = None
|
||||
|
||||
|
||||
class Manifest(BaseModel):
|
||||
discovery_date: Optional[str] = None
|
||||
topic: Optional[str] = None
|
||||
total_urls: int = 0
|
||||
urls: list[ManifestURL] = Field(default_factory=list)
|
||||
|
||||
|
||||
class CrawlResultEntry(BaseModel):
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
raw_path: str
|
||||
content_size: int = 0
|
||||
status: str = "completed"
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class CrawlResult(BaseModel):
|
||||
crawl_date: Optional[str] = None
|
||||
crawler_used: str = "firecrawl"
|
||||
total_crawled: int = 0
|
||||
total_failed: int = 0
|
||||
documents: list[CrawlResultEntry] = Field(default_factory=list)
|
||||
|
||||
|
||||
class QAAssessment(BaseModel):
|
||||
"""Legacy model for post-distillation Claude self-review (deprecated)."""
|
||||
accuracy: float = 0.0
|
||||
completeness: float = 0.0
|
||||
clarity: float = 0.0
|
||||
prompt_engineering_quality: float = 0.0
|
||||
usability: float = 0.0
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def weighted_score(self) -> float:
|
||||
return round(
|
||||
self.accuracy * 0.25
|
||||
+ self.completeness * 0.20
|
||||
+ self.clarity * 0.20
|
||||
+ self.prompt_engineering_quality * 0.25
|
||||
+ self.usability * 0.10,
|
||||
4,
|
||||
)
|
||||
|
||||
|
||||
class SourceQAAssessment(BaseModel):
|
||||
"""Pre-distillation source quality assessment via Gemini."""
|
||||
relevance: float = 0.0
|
||||
authority: float = 0.0
|
||||
completeness: float = 0.0
|
||||
freshness: float = 0.0
|
||||
distill_value: float = 0.0
|
||||
verdict: str = ""
|
||||
reason: str = ""
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def weighted_score(self) -> float:
|
||||
return round(
|
||||
self.relevance * 0.25
|
||||
+ self.authority * 0.25
|
||||
+ self.completeness * 0.20
|
||||
+ self.freshness * 0.15
|
||||
+ self.distill_value * 0.15,
|
||||
4,
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Common utilities for the reference curator pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
"""Normalize a URL for deduplication.
|
||||
|
||||
- Lowercase scheme and host
|
||||
- Remove trailing slashes
|
||||
- Sort query parameters
|
||||
- Remove common tracking params (utm_*, ref, fbclid)
|
||||
- Remove fragment
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
scheme = parsed.scheme.lower()
|
||||
netloc = parsed.netloc.lower()
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
# Sort query params, removing tracking params
|
||||
tracking_params = {"utm_source", "utm_medium", "utm_campaign", "utm_content",
|
||||
"utm_term", "ref", "fbclid", "gclid", "mc_cid", "mc_eid"}
|
||||
params = parse_qs(parsed.query, keep_blank_values=True)
|
||||
filtered = {k: v for k, v in sorted(params.items()) if k not in tracking_params}
|
||||
query = urlencode(filtered, doseq=True)
|
||||
|
||||
return urlunparse((scheme, netloc, path, "", query, ""))
|
||||
|
||||
|
||||
def url_hash(url: str) -> str:
|
||||
"""SHA-256 hash of normalized URL. Matches the url_hash column in schema.sql."""
|
||||
return hashlib.sha256(normalize_url(url).encode()).hexdigest()
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
"""Convert text to a URL/folder-friendly slug.
|
||||
|
||||
>>> slugify("Prompt Engineering Best Practices")
|
||||
'prompt-engineering-best-practices'
|
||||
"""
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = text.encode("ascii", "ignore").decode()
|
||||
text = text.lower()
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
text = text.strip("-")
|
||||
return text or "untitled"
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""Approximate token count using chars/4 heuristic.
|
||||
|
||||
Good enough for compression ratio calculations without requiring tiktoken.
|
||||
"""
|
||||
return max(1, len(text) // 4)
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO", run_id: int | None = None) -> logging.Logger:
|
||||
"""Configure and return a logger for the reference curator pipeline."""
|
||||
logger = logging.getLogger("refcurator")
|
||||
if not logger.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
fmt = "[refcurator]"
|
||||
if run_id:
|
||||
fmt += f" [run:{run_id}]"
|
||||
fmt += " %(levelname)s: %(message)s"
|
||||
handler.setFormatter(logging.Formatter(fmt))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
return logger
|
||||
Reference in New Issue
Block a user