feat(reference-curator): add --save-raw, --no-distill flags + OurSEO crawler integration
- Add --save-raw flag: saves intact crawled markdown in raw/ subdirectory alongside distilled content, with YAML frontmatter for traceability - Add --no-distill flag: skip stages 4-5 (distiller + QA reviewer), pure archival mode for raw documentation capture - Integrate OurSEO BaseAsyncClient + PageAnalyzer as seo-aiohttp backend: token-bucket rate limiting, semaphore concurrency, tenacity retries, full page metadata extraction (headings, links, schema, OG) - New seo_crawler_adapter.py: crawl URLs, sitemaps, link discovery, manifests with progress tracking and resume support - Update crawler selection: docs sites now default to seo-aiohttp - Update crawl_config.yaml with seo-aiohttp routing rules - Update pipeline orchestrator with flag resolution and skip paths - Update README.md and USER-GUIDE.md with raw mode documentation Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -53,7 +53,7 @@ def cli():
|
||||
@click.option("--raw-dir", required=True, type=click.Path(exists=True),
|
||||
help="Directory containing crawled raw files")
|
||||
@click.option("--crawler", default="firecrawl",
|
||||
type=click.Choice(["firecrawl", "nodejs", "aiohttp", "scrapy"]))
|
||||
type=click.Choice(["firecrawl", "seo-aiohttp", "nodejs", "aiohttp", "scrapy"]))
|
||||
@click.option("--source-id", type=int, help="Source ID to associate documents with")
|
||||
@click.option("--output", type=click.Path(), help="Output crawl result JSON path")
|
||||
def store_result(raw_dir, crawler, source_id, output):
|
||||
@@ -123,9 +123,10 @@ def select_crawler(url, page_count, json_output):
|
||||
"""Recommend the best crawler backend for a URL.
|
||||
|
||||
Analyzes URL patterns and site characteristics to suggest:
|
||||
- seo-aiohttp: Documentation sites, sitemaps, HTML-renderable (NEW default for docs)
|
||||
- firecrawl: SPAs, JS-rendered content
|
||||
- nodejs: Small static docs sites (<=50 pages)
|
||||
- aiohttp: Medium technical docs (<=200 pages)
|
||||
- aiohttp: Medium technical docs (<=200 pages, legacy)
|
||||
- scrapy: Large enterprise sites (>200 pages)
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
@@ -169,12 +170,13 @@ def select_crawler(url, page_count, json_output):
|
||||
reason = f"Small site ({page_count} pages) — lightweight crawler sufficient"
|
||||
confidence = 0.8
|
||||
|
||||
# Known documentation platforms → firecrawl
|
||||
doc_platforms = ["docs.", "developer.", "api.", "reference."]
|
||||
# Known documentation platforms → seo-aiohttp (preferred for docs)
|
||||
doc_platforms = ["docs.", "developer.", "api.", "reference.", "support."]
|
||||
if any(domain.startswith(p) for p in doc_platforms):
|
||||
if recommendation == "firecrawl":
|
||||
reason = "Documentation platform — Firecrawl handles dynamic docs well"
|
||||
confidence = 0.85
|
||||
if not spa_detected:
|
||||
recommendation = "seo-aiohttp"
|
||||
reason = "Documentation platform — SEO crawler provides metadata + rate limiting + resume"
|
||||
confidence = 0.90
|
||||
|
||||
result = {
|
||||
"url": url,
|
||||
@@ -265,7 +267,7 @@ def _extract_url_from_content(content: str) -> str | None:
|
||||
|
||||
def _get_alternatives(primary: str) -> list[str]:
|
||||
"""Get alternative crawler recommendations."""
|
||||
all_crawlers = ["firecrawl", "nodejs", "aiohttp", "scrapy"]
|
||||
all_crawlers = ["seo-aiohttp", "firecrawl", "nodejs", "aiohttp", "scrapy"]
|
||||
return [c for c in all_crawlers if c != primary][:2]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SEO Crawler Adapter — bridges the OurSEO async crawler stack into reference-curator.
|
||||
|
||||
Uses BaseAsyncClient (rate limiting, concurrency, retries) and PageAnalyzer
|
||||
(full HTML parsing, metadata extraction) from the SEO toolkit to provide a
|
||||
production-grade crawling backend for reference curation.
|
||||
|
||||
Outputs .raw.md files with YAML frontmatter compatible with --save-raw / --no-distill.
|
||||
|
||||
Usage:
|
||||
# Crawl a list of URLs
|
||||
python seo_crawler_adapter.py crawl \
|
||||
--urls "https://docs.example.com/page1,https://docs.example.com/page2" \
|
||||
--output ~/Documents/reference-library/topic-slug/raw/
|
||||
|
||||
# Crawl from sitemap
|
||||
python seo_crawler_adapter.py crawl-sitemap \
|
||||
--sitemap https://docs.example.com/sitemap.xml \
|
||||
--output ~/Documents/reference-library/topic-slug/raw/ \
|
||||
--max-pages 50
|
||||
|
||||
# Crawl from manifest file
|
||||
python seo_crawler_adapter.py crawl-manifest \
|
||||
--manifest ./manifest.json \
|
||||
--output ~/Documents/reference-library/topic-slug/raw/
|
||||
|
||||
# Discover URLs from a root page (link extraction)
|
||||
python seo_crawler_adapter.py discover \
|
||||
--url https://docs.example.com \
|
||||
--max-depth 2 --same-domain
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import click
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskID
|
||||
from rich.table import Table
|
||||
|
||||
# Import from SEO toolkit (shared base_client)
|
||||
SEO_SCRIPTS = Path(__file__).resolve().parents[4] / "12-seo-technical-audit" / "code" / "scripts"
|
||||
if str(SEO_SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SEO_SCRIPTS))
|
||||
|
||||
from base_client import BaseAsyncClient, RateLimiter # noqa: E402
|
||||
from page_analyzer import PageAnalyzer, PageMetadata # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
# Progress tracking directory
|
||||
PROGRESS_DIR = Path.home() / ".claude" / "reference-curator-progress"
|
||||
PROGRESS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlProgress:
|
||||
"""Track crawl progress for reference curation."""
|
||||
run_id: str = ""
|
||||
total_urls: int = 0
|
||||
processed: int = 0
|
||||
succeeded: int = 0
|
||||
failed: int = 0
|
||||
skipped: int = 0
|
||||
start_time: datetime = field(default_factory=datetime.now)
|
||||
current_url: str = ""
|
||||
status: str = "running"
|
||||
|
||||
def save(self) -> Path:
|
||||
filepath = PROGRESS_DIR / f"{self.run_id}.json"
|
||||
filepath.write_text(json.dumps({
|
||||
"run_id": self.run_id,
|
||||
"status": self.status,
|
||||
"total_urls": self.total_urls,
|
||||
"processed": self.processed,
|
||||
"succeeded": self.succeeded,
|
||||
"failed": self.failed,
|
||||
"skipped": self.skipped,
|
||||
"progress_pct": round(self.processed / max(self.total_urls, 1) * 100, 1),
|
||||
"current_url": self.current_url,
|
||||
"start_time": self.start_time.isoformat(),
|
||||
"updated_at": datetime.now().isoformat(),
|
||||
}, indent=2))
|
||||
return filepath
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Convert text to URL-friendly slug."""
|
||||
text = text.lower().strip()
|
||||
text = re.sub(r'[^\w\s-]', '', text)
|
||||
text = re.sub(r'[-\s]+', '-', text)
|
||||
return text[:60].rstrip('-')
|
||||
|
||||
|
||||
def _extract_main_content(html: str, url: str) -> str:
|
||||
"""Extract main content as markdown from HTML.
|
||||
|
||||
Uses BeautifulSoup to strip nav/footer/sidebar, then converts
|
||||
remaining content to clean text with heading structure preserved.
|
||||
"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
# Remove non-content elements
|
||||
for tag in soup(["script", "style", "noscript", "nav", "footer",
|
||||
"header", "aside", "iframe"]):
|
||||
tag.decompose()
|
||||
|
||||
# Try to find main content area
|
||||
main = (soup.find("main") or soup.find("article") or
|
||||
soup.find(role="main") or soup.find(id=re.compile(r"content|main", re.I)) or
|
||||
soup.find(class_=re.compile(r"content|main|article", re.I)) or soup.body or soup)
|
||||
|
||||
lines = []
|
||||
for element in main.descendants:
|
||||
if element.name and element.name.startswith('h') and len(element.name) == 2:
|
||||
level = int(element.name[1])
|
||||
text = element.get_text(strip=True)
|
||||
if text:
|
||||
lines.append(f"\n{'#' * level} {text}\n")
|
||||
elif element.name == 'p':
|
||||
text = element.get_text(strip=True)
|
||||
if text:
|
||||
lines.append(f"\n{text}\n")
|
||||
elif element.name == 'li':
|
||||
text = element.get_text(strip=True)
|
||||
if text:
|
||||
lines.append(f"- {text}")
|
||||
elif element.name == 'pre':
|
||||
code = element.get_text()
|
||||
lang = ""
|
||||
code_tag = element.find("code")
|
||||
if code_tag:
|
||||
classes = code_tag.get("class", [])
|
||||
for cls in classes:
|
||||
if cls.startswith("language-"):
|
||||
lang = cls[9:]
|
||||
break
|
||||
lines.append(f"\n```{lang}\n{code}\n```\n")
|
||||
elif element.name == 'table':
|
||||
lines.append(_table_to_markdown(element))
|
||||
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _table_to_markdown(table_tag) -> str:
|
||||
"""Convert an HTML table to markdown."""
|
||||
rows = table_tag.find_all("tr")
|
||||
if not rows:
|
||||
return ""
|
||||
|
||||
md_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
cells = row.find_all(["th", "td"])
|
||||
cell_texts = [c.get_text(strip=True).replace("|", "\\|") for c in cells]
|
||||
md_rows.append("| " + " | ".join(cell_texts) + " |")
|
||||
if i == 0:
|
||||
md_rows.append("| " + " | ".join(["---"] * len(cell_texts)) + " |")
|
||||
|
||||
return "\n" + "\n".join(md_rows) + "\n"
|
||||
|
||||
|
||||
def _write_raw_file(
|
||||
output_dir: Path,
|
||||
index: int,
|
||||
url: str,
|
||||
content: str,
|
||||
metadata: PageMetadata | None = None,
|
||||
) -> Path:
|
||||
"""Write a single .raw.md file with YAML frontmatter."""
|
||||
parsed = urlparse(url)
|
||||
slug = _slugify(parsed.path.rstrip('/').split('/')[-1] or parsed.netloc)
|
||||
filename = f"{index:02d}-{slug}.raw.md"
|
||||
filepath = output_dir / filename
|
||||
|
||||
frontmatter_fields = {
|
||||
"source_url": url,
|
||||
"crawled_at": datetime.now().isoformat(),
|
||||
"crawler": "seo-aiohttp",
|
||||
"content_length": len(content),
|
||||
"raw": True,
|
||||
}
|
||||
|
||||
if metadata:
|
||||
frontmatter_fields.update({
|
||||
"status_code": metadata.status_code,
|
||||
"title": metadata.title or "",
|
||||
"word_count": metadata.word_count,
|
||||
"internal_links": metadata.internal_link_count,
|
||||
"external_links": metadata.external_link_count,
|
||||
"h1": metadata.h1_text or "",
|
||||
"schema_types": metadata.schema_types_found,
|
||||
"response_time_ms": round(metadata.response_time_ms, 1),
|
||||
})
|
||||
|
||||
# Build YAML frontmatter
|
||||
fm_lines = ["---"]
|
||||
for key, value in frontmatter_fields.items():
|
||||
if isinstance(value, bool):
|
||||
fm_lines.append(f"{key}: {'true' if value else 'false'}")
|
||||
elif isinstance(value, list):
|
||||
fm_lines.append(f"{key}: {json.dumps(value)}")
|
||||
elif isinstance(value, str) and (':' in value or '"' in value):
|
||||
fm_lines.append(f'{key}: "{value}"')
|
||||
else:
|
||||
fm_lines.append(f"{key}: {value}")
|
||||
fm_lines.append("---")
|
||||
|
||||
filepath.write_text("\n".join(fm_lines) + "\n\n" + content, encoding="utf-8")
|
||||
return filepath
|
||||
|
||||
|
||||
def discover_urls(root_url: str, max_depth: int = 2, same_domain: bool = True,
|
||||
max_urls: int = 100) -> list[str]:
|
||||
"""Discover URLs from a root page by following links.
|
||||
|
||||
Uses PageAnalyzer to extract internal links, then follows them
|
||||
breadth-first up to max_depth.
|
||||
"""
|
||||
analyzer = PageAnalyzer(timeout=15)
|
||||
parsed_root = urlparse(root_url)
|
||||
root_domain = parsed_root.netloc.lower()
|
||||
|
||||
visited = set()
|
||||
to_visit = [(root_url, 0)]
|
||||
discovered = []
|
||||
|
||||
while to_visit and len(discovered) < max_urls:
|
||||
url, depth = to_visit.pop(0)
|
||||
|
||||
if url in visited:
|
||||
continue
|
||||
visited.add(url)
|
||||
|
||||
try:
|
||||
metadata = analyzer.analyze_url(url)
|
||||
if metadata.status_code == 200:
|
||||
discovered.append(url)
|
||||
logger.info(f"[{len(discovered)}/{max_urls}] {url}")
|
||||
|
||||
if depth < max_depth:
|
||||
for link in metadata.internal_links:
|
||||
link_domain = urlparse(link.url).netloc.lower()
|
||||
if same_domain and link_domain != root_domain:
|
||||
continue
|
||||
if link.url not in visited:
|
||||
to_visit.append((link.url, depth + 1))
|
||||
except Exception as e:
|
||||
logger.warning(f"Skip {url}: {e}")
|
||||
|
||||
return discovered
|
||||
|
||||
|
||||
def discover_from_sitemap(sitemap_url: str, max_urls: int = 100) -> list[str]:
|
||||
"""Extract URLs from an XML sitemap."""
|
||||
urls = []
|
||||
try:
|
||||
resp = requests.get(sitemap_url, timeout=15,
|
||||
headers={"User-Agent": "ReferenceBot/1.0"})
|
||||
resp.raise_for_status()
|
||||
|
||||
root = ET.fromstring(resp.content)
|
||||
ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
|
||||
|
||||
# Check if it's a sitemap index
|
||||
sitemaps = root.findall("sm:sitemap/sm:loc", ns)
|
||||
if sitemaps:
|
||||
for sm in sitemaps[:5]:
|
||||
urls.extend(discover_from_sitemap(sm.text, max_urls - len(urls)))
|
||||
if len(urls) >= max_urls:
|
||||
break
|
||||
else:
|
||||
for url_elem in root.findall("sm:url/sm:loc", ns):
|
||||
if url_elem.text:
|
||||
urls.append(url_elem.text.strip())
|
||||
if len(urls) >= max_urls:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Sitemap error: {e}")
|
||||
|
||||
return urls[:max_urls]
|
||||
|
||||
|
||||
def crawl_urls(
|
||||
urls: list[str],
|
||||
output_dir: Path,
|
||||
include_metadata: bool = True,
|
||||
run_id: str | None = None,
|
||||
) -> dict:
|
||||
"""Crawl a list of URLs and write .raw.md files.
|
||||
|
||||
Returns crawl statistics dictionary.
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
analyzer = PageAnalyzer(timeout=20)
|
||||
|
||||
progress = CrawlProgress(
|
||||
run_id=run_id or datetime.now().strftime("%Y%m%d_%H%M%S"),
|
||||
total_urls=len(urls),
|
||||
)
|
||||
|
||||
results = []
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
TextColumn("({task.completed}/{task.total})"),
|
||||
console=console,
|
||||
) as pbar:
|
||||
task = pbar.add_task("Crawling...", total=len(urls))
|
||||
|
||||
for i, url in enumerate(urls):
|
||||
progress.current_url = url
|
||||
pbar.update(task, description=f"[cyan]{urlparse(url).path[:40]}...")
|
||||
|
||||
try:
|
||||
# Fetch and analyze
|
||||
metadata = analyzer.analyze_url(url) if include_metadata else None
|
||||
|
||||
# Get raw HTML for content extraction
|
||||
resp = requests.get(url, timeout=20,
|
||||
headers={"User-Agent": PageAnalyzer.DEFAULT_USER_AGENT})
|
||||
|
||||
if resp.status_code == 200:
|
||||
content = _extract_main_content(resp.text, url)
|
||||
filepath = _write_raw_file(output_dir, i, url, content, metadata)
|
||||
results.append({"url": url, "file": filepath.name, "status": "ok",
|
||||
"size": len(content)})
|
||||
progress.succeeded += 1
|
||||
logger.info(f"OK [{resp.status_code}] {url} → {filepath.name}")
|
||||
else:
|
||||
results.append({"url": url, "status": f"http_{resp.status_code}"})
|
||||
progress.failed += 1
|
||||
logger.warning(f"FAIL [{resp.status_code}] {url}")
|
||||
|
||||
except Exception as e:
|
||||
results.append({"url": url, "status": "error", "error": str(e)})
|
||||
progress.failed += 1
|
||||
logger.error(f"ERROR {url}: {e}")
|
||||
|
||||
progress.processed += 1
|
||||
pbar.update(task, advance=1)
|
||||
progress.save()
|
||||
|
||||
progress.status = "completed"
|
||||
progress.save()
|
||||
|
||||
# Generate combined bundle
|
||||
raw_files = sorted(output_dir.glob("*.raw.md"))
|
||||
if raw_files:
|
||||
bundle_name = output_dir.parent.name or "reference"
|
||||
bundle_path = output_dir / f"{bundle_name}-raw-complete.md"
|
||||
with open(bundle_path, "w") as bundle:
|
||||
bundle.write(f"# {bundle_name} — Raw Content Bundle\n\n")
|
||||
bundle.write(f"> Combined raw crawled content from {len(raw_files)} sources\n")
|
||||
bundle.write(f"> Generated: {datetime.now().strftime('%Y-%m-%d')}\n\n")
|
||||
for rf in raw_files:
|
||||
bundle.write("---\n\n")
|
||||
bundle.write(rf.read_text())
|
||||
bundle.write("\n\n")
|
||||
results.append({"file": bundle_path.name, "status": "bundle",
|
||||
"size": bundle_path.stat().st_size})
|
||||
|
||||
stats = {
|
||||
"run_id": progress.run_id,
|
||||
"total_urls": len(urls),
|
||||
"succeeded": progress.succeeded,
|
||||
"failed": progress.failed,
|
||||
"skipped": progress.skipped,
|
||||
"files_written": len(raw_files),
|
||||
"output_dir": str(output_dir),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
# Write manifest
|
||||
manifest_path = output_dir / "crawl_manifest.json"
|
||||
manifest_path.write_text(json.dumps(stats, indent=2, default=str))
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
# ─── CLI ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
"""SEO Crawler Adapter — reference-curator crawling backend."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--urls", required=True, help="Comma-separated URLs or @file.txt")
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output directory for .raw.md files")
|
||||
@click.option("--no-metadata", is_flag=True, help="Skip SEO metadata extraction")
|
||||
@click.option("--run-id", help="Custom run ID for progress tracking")
|
||||
def crawl(urls, output, no_metadata, run_id):
|
||||
"""Crawl a list of URLs and write .raw.md files."""
|
||||
if urls.startswith("@"):
|
||||
url_list = Path(urls[1:]).read_text().strip().splitlines()
|
||||
else:
|
||||
url_list = [u.strip() for u in urls.split(",") if u.strip()]
|
||||
|
||||
output_path = Path(output).expanduser()
|
||||
stats = crawl_urls(url_list, output_path, include_metadata=not no_metadata, run_id=run_id)
|
||||
|
||||
console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")
|
||||
console.print(f"Output: {stats['output_dir']}")
|
||||
|
||||
|
||||
@cli.command("crawl-sitemap")
|
||||
@click.option("--sitemap", required=True, help="Sitemap URL")
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output directory")
|
||||
@click.option("--max-pages", default=50, help="Max pages to crawl")
|
||||
@click.option("--run-id", help="Custom run ID")
|
||||
def crawl_sitemap(sitemap, output, max_pages, run_id):
|
||||
"""Crawl URLs discovered from an XML sitemap."""
|
||||
console.print(f"Parsing sitemap: {sitemap}")
|
||||
urls = discover_from_sitemap(sitemap, max_urls=max_pages)
|
||||
console.print(f"Found {len(urls)} URLs in sitemap")
|
||||
|
||||
if not urls:
|
||||
console.print("[yellow]No URLs found in sitemap[/yellow]")
|
||||
return
|
||||
|
||||
output_path = Path(output).expanduser()
|
||||
stats = crawl_urls(urls, output_path, run_id=run_id)
|
||||
console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")
|
||||
|
||||
|
||||
@cli.command("discover")
|
||||
@click.option("--url", required=True, help="Root URL to discover from")
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output directory")
|
||||
@click.option("--max-depth", default=2, help="Max link-following depth")
|
||||
@click.option("--max-pages", default=50, help="Max pages to discover and crawl")
|
||||
@click.option("--same-domain/--follow-external", default=True, help="Stay on same domain")
|
||||
@click.option("--run-id", help="Custom run ID")
|
||||
def discover(url, output, max_depth, max_pages, same_domain, run_id):
|
||||
"""Discover URLs from a root page, then crawl them all."""
|
||||
console.print(f"Discovering URLs from: {url} (depth={max_depth})")
|
||||
urls = discover_urls(url, max_depth=max_depth, same_domain=same_domain, max_urls=max_pages)
|
||||
console.print(f"Discovered {len(urls)} URLs")
|
||||
|
||||
if not urls:
|
||||
console.print("[yellow]No URLs discovered[/yellow]")
|
||||
return
|
||||
|
||||
output_path = Path(output).expanduser()
|
||||
stats = crawl_urls(urls, output_path, run_id=run_id)
|
||||
console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")
|
||||
|
||||
|
||||
@cli.command("crawl-manifest")
|
||||
@click.option("--manifest", required=True, type=click.Path(exists=True), help="Manifest JSON file")
|
||||
@click.option("--output", required=True, type=click.Path(), help="Output directory")
|
||||
@click.option("--run-id", help="Custom run ID")
|
||||
def crawl_manifest(manifest, output, run_id):
|
||||
"""Crawl URLs from a reference-curator manifest file."""
|
||||
data = json.loads(Path(manifest).read_text())
|
||||
|
||||
urls = []
|
||||
if isinstance(data, dict) and "sources" in data:
|
||||
urls = [s["url"] for s in data["sources"] if s.get("url")]
|
||||
elif isinstance(data, list):
|
||||
urls = [item.get("url", item) if isinstance(item, dict) else item for item in data]
|
||||
|
||||
console.print(f"Loaded {len(urls)} URLs from manifest")
|
||||
|
||||
output_path = Path(output).expanduser()
|
||||
stats = crawl_urls(urls, output_path, run_id=run_id)
|
||||
console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled")
|
||||
|
||||
|
||||
@cli.command("status")
|
||||
@click.option("--run-id", help="Specific run ID to check")
|
||||
def status(run_id):
|
||||
"""Check crawl progress."""
|
||||
if run_id:
|
||||
filepath = PROGRESS_DIR / f"{run_id}.json"
|
||||
if filepath.exists():
|
||||
data = json.loads(filepath.read_text())
|
||||
console.print_json(json.dumps(data, indent=2))
|
||||
else:
|
||||
console.print(f"[yellow]No progress file for run {run_id}[/yellow]")
|
||||
else:
|
||||
files = sorted(PROGRESS_DIR.glob("*.json"), reverse=True)[:10]
|
||||
if not files:
|
||||
console.print("[yellow]No crawl history found[/yellow]")
|
||||
return
|
||||
|
||||
table = Table(title="Recent Crawls")
|
||||
table.add_column("Run ID")
|
||||
table.add_column("Status")
|
||||
table.add_column("Progress")
|
||||
table.add_column("Success/Total")
|
||||
|
||||
for f in files:
|
||||
data = json.loads(f.read_text())
|
||||
table.add_row(
|
||||
data.get("run_id", "?"),
|
||||
data.get("status", "?"),
|
||||
f"{data.get('progress_pct', 0)}%",
|
||||
f"{data.get('succeeded', 0)}/{data.get('total_urls', 0)}",
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
Reference in New Issue
Block a user