Andrew Yim
b1c2dca080
- Add --save-raw flag: saves intact crawled markdown in raw/ subdirectory alongside distilled content, with YAML frontmatter for traceability - Add --no-distill flag: skip stages 4-5 (distiller + QA reviewer), pure archival mode for raw documentation capture - Integrate OurSEO BaseAsyncClient + PageAnalyzer as seo-aiohttp backend: token-bucket rate limiting, semaphore concurrency, tenacity retries, full page metadata extraction (headings, links, schema, OG) - New seo_crawler_adapter.py: crawl URLs, sitemaps, link discovery, manifests with progress tracking and resume support - Update crawler selection: docs sites now default to seo-aiohttp - Update crawl_config.yaml with seo-aiohttp routing rules - Update pipeline orchestrator with flag resolution and skip paths - Update README.md and USER-GUIDE.md with raw mode documentation Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
163 lines
4.1 KiB
YAML
163 lines
4.1 KiB
YAML
# Reference Curator - Crawl Configuration
|
|
# Location: ~/.config/reference-curator/crawl_config.yaml
|
|
#
|
|
# Environment variables (set in ~/.reference-curator.env):
|
|
# CRAWLER_PROJECT_PATH - Path to crawler project (optional)
|
|
# REFERENCE_LIBRARY_PATH - Path to reference library storage (optional)
|
|
|
|
# Default crawler backend
|
|
# Options: seo-aiohttp, firecrawl, nodejs, aiohttp, scrapy
|
|
# seo-aiohttp: Uses OurSEO BaseAsyncClient + PageAnalyzer (recommended for docs)
|
|
# firecrawl: MCP-based, best for SPAs/JS-rendered sites
|
|
default_crawler: ${DEFAULT_CRAWLER:-seo-aiohttp}
|
|
|
|
# Intelligent routing rules
|
|
# Claude will select the appropriate crawler based on these criteria
|
|
routing:
|
|
seo-aiohttp:
|
|
conditions:
|
|
- is_documentation_site == true
|
|
- has_sitemap == true
|
|
- needs_metadata == true
|
|
- max_pages <= 200
|
|
description: "OurSEO async crawler — rate-limited, resumable, full metadata extraction. Best for documentation sites."
|
|
|
|
nodejs:
|
|
conditions:
|
|
- max_pages <= 50
|
|
- single_domain == true
|
|
- no_javascript_rendering == true
|
|
description: "Fast, lightweight - best for small documentation sites"
|
|
|
|
aiohttp:
|
|
conditions:
|
|
- max_pages <= 200
|
|
- needs_async == true
|
|
- seo_extraction == true
|
|
description: "Async with SEO extraction - best for technical docs"
|
|
|
|
scrapy:
|
|
conditions:
|
|
- max_pages > 200
|
|
- multi_domain == true
|
|
- needs_pipeline == true
|
|
description: "Enterprise-grade - best for large sites with complex structure"
|
|
|
|
firecrawl:
|
|
conditions:
|
|
- needs_javascript_rendering == true
|
|
- spa_site == true
|
|
- dynamic_content == true
|
|
description: "JS rendering - best for SPAs and dynamic content (MCP-based, always available)"
|
|
|
|
# Crawler locations (configurable via environment)
|
|
crawlers:
|
|
seo-aiohttp:
|
|
enabled: true # Always available (uses OurSEO scripts from 12-seo-technical-audit)
|
|
type: local
|
|
path: ${SKILL_PATH:-../../../90-reference-curator}/02-web-crawler-orchestrator/code/scripts/
|
|
command: python seo_crawler_adapter.py
|
|
capabilities:
|
|
- rate_limiting
|
|
- progress_tracking
|
|
- resume_support
|
|
- metadata_extraction
|
|
- sitemap_discovery
|
|
- link_discovery
|
|
- raw_file_output
|
|
install: "pip install requests beautifulsoup4 click rich tenacity"
|
|
|
|
nodejs:
|
|
enabled: ${NODEJS_CRAWLER_ENABLED:-false}
|
|
path: ${CRAWLER_PROJECT_PATH}/util/js-crawler/
|
|
command: node src/crawler.js
|
|
install: npm install
|
|
|
|
aiohttp:
|
|
enabled: ${AIOHTTP_CRAWLER_ENABLED:-false}
|
|
path: ${CRAWLER_PROJECT_PATH}/
|
|
command: python -m seo_agent.crawler
|
|
install: uv sync
|
|
|
|
scrapy:
|
|
enabled: ${SCRAPY_CRAWLER_ENABLED:-false}
|
|
path: ${CRAWLER_PROJECT_PATH}/
|
|
command: scrapy crawl seo_spider
|
|
install: uv sync
|
|
|
|
firecrawl:
|
|
enabled: true # Always available via MCP
|
|
type: mcp
|
|
server: firecrawl
|
|
tools:
|
|
- firecrawl_scrape
|
|
- firecrawl_crawl
|
|
- firecrawl_map
|
|
install: "Configure firecrawl MCP server in Claude Code settings"
|
|
|
|
# Rate limiting
|
|
rate_limit:
|
|
requests_per_minute: 20
|
|
concurrent_requests: 3
|
|
|
|
# Retry settings
|
|
retry:
|
|
max_retries: 3
|
|
backoff_multiplier: 2
|
|
initial_delay_seconds: 10
|
|
|
|
# Default crawl options
|
|
default_options:
|
|
timeout: 30000 # milliseconds
|
|
max_depth: 3
|
|
max_pages: 100
|
|
respect_robots_txt: true
|
|
user_agent: "ReferenceBot/1.0"
|
|
|
|
# Content processing
|
|
processing:
|
|
max_content_size_mb: 50
|
|
supported_formats:
|
|
- html
|
|
- pdf
|
|
- markdown
|
|
|
|
# Storage paths (configurable)
|
|
raw_content_dir: ${REFERENCE_LIBRARY_PATH:-~/Documents/05_AI Agent/10_Reference Library}/raw/
|
|
processed_dir: ${REFERENCE_LIBRARY_PATH:-~/Documents/05_AI Agent/10_Reference Library}/processed/
|
|
|
|
# URL filtering
|
|
url_filter:
|
|
skip_extensions:
|
|
- .jpg
|
|
- .jpeg
|
|
- .png
|
|
- .gif
|
|
- .svg
|
|
- .webp
|
|
- .css
|
|
- .js
|
|
- .woff
|
|
- .woff2
|
|
- .ico
|
|
skip_patterns:
|
|
- /wp-admin/
|
|
- /wp-includes/
|
|
- /login
|
|
- /logout
|
|
|
|
# Site type detection hints
|
|
site_detection:
|
|
spa_indicators:
|
|
- "react"
|
|
- "angular"
|
|
- "vue"
|
|
- "next.js"
|
|
- "nuxt"
|
|
static_indicators:
|
|
- "hugo"
|
|
- "jekyll"
|
|
- "mkdocs"
|
|
- "docusaurus"
|
|
- "gitbook"
|