feat(reference-curator): implement Python scripts + Gemini quality gate
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,46 +1,42 @@
|
||||
# Content Repository
|
||||
|
||||
MySQL storage management for the reference library. Handles document storage, version control, deduplication, and retrieval.
|
||||
MySQL storage management for the reference library. Handles document storage, version control, deduplication, and retrieval. Supports MySQL primary backend with JSON file fallback.
|
||||
|
||||
## Trigger Keywords
|
||||
"store content", "save to database", "check duplicates", "version tracking", "document retrieval", "reference library DB"
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- MySQL 8.0+ with utf8mb4 charset
|
||||
- Config file at `~/.config/reference-curator/db_config.yaml`
|
||||
- Database `reference_library` initialized
|
||||
- `refcurator` package installed (`uv pip install -e shared/lib/`)
|
||||
- MySQL 8.0+ (optional — falls back to JSON file storage)
|
||||
- Config at `~/.config/reference-curator/db_config.yaml` (optional)
|
||||
|
||||
## Database Setup
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Initialize database
|
||||
mysql -u root -p < references/schema.sql
|
||||
|
||||
# Verify tables
|
||||
mysql -u root -p reference_library -e "SHOW TABLES;"
|
||||
```
|
||||
|
||||
## Core Scripts
|
||||
|
||||
### Store Document
|
||||
```bash
|
||||
python scripts/store_document.py \
|
||||
--source-id 1 \
|
||||
--title "Prompt Engineering Guide" \
|
||||
# Store a document
|
||||
uv run python scripts/repo.py store \
|
||||
--source-id 1 --title "Prompt Engineering Guide" \
|
||||
--url "https://docs.anthropic.com/..." \
|
||||
--doc-type webpage \
|
||||
--raw-path ~/reference-library/raw/2025/01/abc123.md
|
||||
```
|
||||
--doc-type webpage --raw-path ~/reference-library/raw/abc123.md
|
||||
|
||||
### Check Duplicate
|
||||
```bash
|
||||
python scripts/check_duplicate.py --url "https://docs.anthropic.com/..."
|
||||
```
|
||||
# Check for duplicates
|
||||
uv run python scripts/repo.py check-dup --url "https://docs.anthropic.com/..."
|
||||
|
||||
### Query by Topic
|
||||
```bash
|
||||
python scripts/query_topic.py --topic-slug prompt-engineering --min-quality 0.80
|
||||
# Query by topic
|
||||
uv run python scripts/repo.py query-topic --topic-slug prompt-engineering --min-quality 0.80
|
||||
|
||||
# Get repository stats
|
||||
uv run python scripts/repo.py stats
|
||||
|
||||
# Find stale documents (older than 30 days)
|
||||
uv run python scripts/repo.py find-stale --days 30
|
||||
|
||||
# Get pending reviews
|
||||
uv run python scripts/repo.py pending-reviews --output pending.json
|
||||
|
||||
# Get export-ready content
|
||||
uv run python scripts/repo.py export-ready --min-score 0.85
|
||||
```
|
||||
|
||||
## Table Quick Reference
|
||||
@@ -61,31 +57,17 @@ python scripts/query_topic.py --topic-slug prompt-engineering --min-quality 0.80
|
||||
|
||||
**review_status:** `pending` → `in_review` → `approved` | `needs_refactor` | `rejected`
|
||||
|
||||
## Common Queries
|
||||
|
||||
### Find Stale Documents
|
||||
```bash
|
||||
python scripts/find_stale.py --output stale_docs.json
|
||||
```
|
||||
|
||||
### Get Pending Reviews
|
||||
```bash
|
||||
python scripts/pending_reviews.py --output pending.json
|
||||
```
|
||||
|
||||
### Export-Ready Content
|
||||
```bash
|
||||
python scripts/export_ready.py --min-score 0.85 --output ready.json
|
||||
```
|
||||
|
||||
## Scripts
|
||||
|
||||
- `scripts/store_document.py` - Store new document
|
||||
- `scripts/check_duplicate.py` - URL deduplication
|
||||
- `scripts/query_topic.py` - Query by topic
|
||||
- `scripts/find_stale.py` - Find stale documents
|
||||
- `scripts/pending_reviews.py` - Get pending reviews
|
||||
- `scripts/db_utils.py` - Database connection utilities
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `repo.py store` | Store a new document |
|
||||
| `repo.py check-dup` | URL deduplication check |
|
||||
| `repo.py query-topic` | Query documents by topic |
|
||||
| `repo.py find-stale` | Find stale documents |
|
||||
| `repo.py pending-reviews` | Get pending reviews |
|
||||
| `repo.py export-ready` | Get approved content ready for export |
|
||||
| `repo.py stats` | Show repository statistics |
|
||||
|
||||
## Integration
|
||||
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Content Repository CLI — CRUD operations for the reference library.
|
||||
|
||||
Usage:
|
||||
python repo.py store --source-id 1 --title "Doc" --url "https://..." --doc-type webpage --raw-path /path/to/file
|
||||
python repo.py check-dup --url "https://..."
|
||||
python repo.py query-topic --topic-slug prompt-engineering [--min-quality 0.80]
|
||||
python repo.py find-stale [--output stale.json]
|
||||
python repo.py pending-reviews [--output pending.json]
|
||||
python repo.py export-ready [--min-score 0.85] [--output ready.json]
|
||||
python repo.py stats
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from refcurator.db import db_session
|
||||
from refcurator.utils import url_hash, normalize_url
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""Content Repository — manage documents in the reference library."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--source-id", required=True, type=int, help="Source ID from sources table")
|
||||
@click.option("--title", required=True, help="Document title")
|
||||
@click.option("--url", required=True, help="Document URL")
|
||||
@click.option("--doc-type", required=True,
|
||||
type=click.Choice(["webpage", "pdf", "markdown", "api_spec", "code_sample"]))
|
||||
@click.option("--raw-path", required=True, type=click.Path(), help="Path to raw content file")
|
||||
@click.option("--crawl-method", default="firecrawl",
|
||||
type=click.Choice(["firecrawl", "scrapy", "aiohttp", "nodejs", "manual", "api"]))
|
||||
@click.option("--language", default="en", type=click.Choice(["en", "ko", "mixed"]))
|
||||
def store(source_id, title, url, doc_type, raw_path, crawl_method, language):
|
||||
"""Store a new document in the repository."""
|
||||
raw = Path(raw_path)
|
||||
if not raw.is_file():
|
||||
console.print(f"[red]Error:[/red] Raw file not found: {raw_path}")
|
||||
sys.exit(1)
|
||||
|
||||
content_size = raw.stat().st_size
|
||||
|
||||
with db_session() as db:
|
||||
# Check for duplicate
|
||||
existing = db.fetch_one(
|
||||
"SELECT doc_id, title FROM documents WHERE url_hash = %s",
|
||||
(url_hash(url),),
|
||||
)
|
||||
if existing:
|
||||
console.print(
|
||||
f"[yellow]Duplicate:[/yellow] URL already stored as doc_id={existing['doc_id']} "
|
||||
f"({existing['title']})"
|
||||
)
|
||||
sys.exit(0)
|
||||
|
||||
doc_id = db.insert_returning_id(
|
||||
"""INSERT INTO documents
|
||||
(source_id, title, url, doc_type, language, crawl_date,
|
||||
crawl_method, crawl_status, raw_content_path, raw_content_size)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||
(source_id, title, url, doc_type, language, datetime.now().isoformat(),
|
||||
crawl_method, "completed", str(raw.resolve()), content_size),
|
||||
)
|
||||
|
||||
console.print(f"[green]Stored:[/green] doc_id={doc_id} — {title}")
|
||||
click.echo(json.dumps({"doc_id": doc_id, "title": title, "url": url}))
|
||||
|
||||
|
||||
@cli.command("check-dup")
|
||||
@click.option("--url", required=True, help="URL to check for duplicates")
|
||||
def check_dup(url):
|
||||
"""Check if a URL already exists in the repository."""
|
||||
h = url_hash(url)
|
||||
|
||||
with db_session() as db:
|
||||
existing = db.fetch_one(
|
||||
"SELECT doc_id, title, url, crawl_status FROM documents WHERE url_hash = %s",
|
||||
(h,),
|
||||
)
|
||||
|
||||
if existing:
|
||||
console.print(f"[yellow]Duplicate found:[/yellow] doc_id={existing['doc_id']}")
|
||||
click.echo(json.dumps(existing, default=str))
|
||||
else:
|
||||
console.print("[green]No duplicate found.[/green]")
|
||||
click.echo(json.dumps({"duplicate": False, "url": url}))
|
||||
|
||||
|
||||
@cli.command("query-topic")
|
||||
@click.option("--topic-slug", required=True, help="Topic slug to query")
|
||||
@click.option("--min-quality", default=0.0, type=float, help="Minimum quality score")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def query_topic(topic_slug, min_quality, output):
|
||||
"""Query documents by topic."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url, d.crawl_status,
|
||||
dt.relevance_score, t.topic_name
|
||||
FROM documents d
|
||||
JOIN document_topics dt ON d.doc_id = dt.doc_id
|
||||
JOIN topics t ON dt.topic_id = t.topic_id
|
||||
WHERE t.topic_slug = %s
|
||||
ORDER BY dt.relevance_score DESC""",
|
||||
(topic_slug,),
|
||||
)
|
||||
|
||||
if min_quality > 0:
|
||||
# Filter by review score if available
|
||||
rows = [r for r in rows if r.get("relevance_score", 0) >= min_quality]
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]Wrote {len(rows)} results to {output}[/green]")
|
||||
else:
|
||||
_print_doc_table(rows, f"Topic: {topic_slug}")
|
||||
|
||||
|
||||
@cli.command("find-stale")
|
||||
@click.option("--days", default=30, type=int, help="Documents older than N days")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def find_stale(days, output):
|
||||
"""Find documents that may be outdated."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT doc_id, title, url, crawl_date, crawl_status
|
||||
FROM documents
|
||||
WHERE crawl_status = %s
|
||||
ORDER BY crawl_date ASC""",
|
||||
("completed",),
|
||||
)
|
||||
|
||||
# Filter by age
|
||||
cutoff = datetime.now()
|
||||
stale = []
|
||||
for r in rows:
|
||||
crawl_date = r.get("crawl_date")
|
||||
if crawl_date:
|
||||
if isinstance(crawl_date, str):
|
||||
crawl_date = datetime.fromisoformat(crawl_date)
|
||||
age_days = (cutoff - crawl_date).days
|
||||
if age_days >= days:
|
||||
r["age_days"] = age_days
|
||||
stale.append(r)
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(stale, indent=2, default=str))
|
||||
console.print(f"[green]Found {len(stale)} stale documents, wrote to {output}[/green]")
|
||||
else:
|
||||
_print_doc_table(stale, f"Stale documents (>{days} days)")
|
||||
|
||||
|
||||
@cli.command("pending-reviews")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def pending_reviews(output):
|
||||
"""Get documents pending quality review."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT dc.distill_id, d.doc_id, d.title, d.url,
|
||||
dc.token_count_distilled, dc.distill_date
|
||||
FROM distilled_content dc
|
||||
JOIN documents d ON dc.doc_id = d.doc_id
|
||||
WHERE dc.review_status = %s
|
||||
ORDER BY dc.distill_date ASC""",
|
||||
("pending",),
|
||||
)
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]{len(rows)} pending reviews, wrote to {output}[/green]")
|
||||
else:
|
||||
table = Table(title="Pending Reviews")
|
||||
table.add_column("distill_id", style="cyan")
|
||||
table.add_column("doc_id")
|
||||
table.add_column("Title")
|
||||
table.add_column("Tokens", justify="right")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("distill_id", "")),
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:50],
|
||||
str(r.get("token_count_distilled", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
@cli.command("export-ready")
|
||||
@click.option("--min-score", default=0.80, type=float, help="Minimum quality score")
|
||||
@click.option("--output", type=click.Path(), help="Output JSON file path")
|
||||
def export_ready(min_score, output):
|
||||
"""Get documents approved and ready for export."""
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all(
|
||||
"""SELECT d.doc_id, d.title, d.url,
|
||||
dc.structured_content, dc.token_count_distilled,
|
||||
rl.quality_score, rl.decision
|
||||
FROM documents d
|
||||
JOIN distilled_content dc ON d.doc_id = dc.doc_id
|
||||
JOIN review_logs rl ON dc.distill_id = rl.distill_id
|
||||
WHERE dc.review_status = %s
|
||||
AND rl.decision = %s
|
||||
AND rl.review_id = (
|
||||
SELECT MAX(rl2.review_id)
|
||||
FROM review_logs rl2
|
||||
WHERE rl2.distill_id = dc.distill_id
|
||||
)
|
||||
ORDER BY rl.quality_score DESC""",
|
||||
("approved", "approve"),
|
||||
)
|
||||
|
||||
# Filter by min score
|
||||
rows = [r for r in rows if (r.get("quality_score") or 0) >= min_score]
|
||||
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(rows, indent=2, default=str))
|
||||
console.print(f"[green]{len(rows)} export-ready documents, wrote to {output}[/green]")
|
||||
else:
|
||||
table = Table(title=f"Export-Ready (score >= {min_score})")
|
||||
table.add_column("doc_id", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("Score", justify="right")
|
||||
table.add_column("Tokens", justify="right")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:50],
|
||||
f"{r.get('quality_score', 0):.2f}",
|
||||
str(r.get("token_count_distilled", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
@cli.command()
|
||||
def stats():
|
||||
"""Show repository statistics."""
|
||||
with db_session() as db:
|
||||
doc_count = db.fetch_one("SELECT COUNT(*) as cnt FROM documents") or {"cnt": 0}
|
||||
source_count = db.fetch_one("SELECT COUNT(*) as cnt FROM sources") or {"cnt": 0}
|
||||
|
||||
status_rows = db.fetch_all(
|
||||
"""SELECT crawl_status, COUNT(*) as cnt
|
||||
FROM documents
|
||||
GROUP BY crawl_status"""
|
||||
)
|
||||
|
||||
review_rows = db.fetch_all(
|
||||
"""SELECT review_status, COUNT(*) as cnt
|
||||
FROM distilled_content
|
||||
GROUP BY review_status"""
|
||||
)
|
||||
|
||||
topic_rows = db.fetch_all(
|
||||
"""SELECT t.topic_name, COUNT(dt.doc_id) as cnt
|
||||
FROM topics t
|
||||
LEFT JOIN document_topics dt ON t.topic_id = dt.topic_id
|
||||
GROUP BY t.topic_id
|
||||
ORDER BY cnt DESC"""
|
||||
)
|
||||
|
||||
console.print()
|
||||
console.print(f"[bold]Reference Library Statistics[/bold]")
|
||||
console.print(f" Sources: {source_count['cnt']}")
|
||||
console.print(f" Documents: {doc_count['cnt']}")
|
||||
console.print()
|
||||
|
||||
if status_rows:
|
||||
table = Table(title="Documents by Crawl Status")
|
||||
table.add_column("Status")
|
||||
table.add_column("Count", justify="right")
|
||||
for r in status_rows:
|
||||
table.add_row(str(r["crawl_status"]), str(r["cnt"]))
|
||||
console.print(table)
|
||||
|
||||
if review_rows:
|
||||
table = Table(title="Distilled Content by Review Status")
|
||||
table.add_column("Status")
|
||||
table.add_column("Count", justify="right")
|
||||
for r in review_rows:
|
||||
table.add_row(str(r["review_status"]), str(r["cnt"]))
|
||||
console.print(table)
|
||||
|
||||
if topic_rows:
|
||||
table = Table(title="Documents by Topic")
|
||||
table.add_column("Topic")
|
||||
table.add_column("Documents", justify="right")
|
||||
for r in topic_rows:
|
||||
table.add_row(str(r["topic_name"]), str(r["cnt"]))
|
||||
console.print(table)
|
||||
|
||||
result = {
|
||||
"sources": source_count["cnt"],
|
||||
"documents": doc_count["cnt"],
|
||||
"by_status": {r["crawl_status"]: r["cnt"] for r in status_rows},
|
||||
"by_review": {r["review_status"]: r["cnt"] for r in review_rows},
|
||||
"by_topic": {r["topic_name"]: r["cnt"] for r in topic_rows},
|
||||
}
|
||||
click.echo(json.dumps(result, default=str))
|
||||
|
||||
|
||||
def _print_doc_table(rows: list[dict], title: str):
|
||||
"""Print a table of documents."""
|
||||
table = Table(title=title)
|
||||
table.add_column("doc_id", style="cyan")
|
||||
table.add_column("Title")
|
||||
table.add_column("URL")
|
||||
table.add_column("Status")
|
||||
for r in rows:
|
||||
table.add_row(
|
||||
str(r.get("doc_id", "")),
|
||||
str(r.get("title", ""))[:40],
|
||||
str(r.get("url", ""))[:50],
|
||||
str(r.get("crawl_status", "")),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
Reference in New Issue
Block a user