feat(reference-curator): add --save-raw, --no-distill flags + OurSEO crawler integration

- Add --save-raw flag: saves intact crawled markdown in raw/ subdirectory
  alongside distilled content, with YAML frontmatter for traceability
- Add --no-distill flag: skip stages 4-5 (distiller + QA reviewer),
  pure archival mode for raw documentation capture
- Integrate OurSEO BaseAsyncClient + PageAnalyzer as seo-aiohttp backend:
  token-bucket rate limiting, semaphore concurrency, tenacity retries,
  full page metadata extraction (headings, links, schema, OG)
- New seo_crawler_adapter.py: crawl URLs, sitemaps, link discovery, manifests
  with progress tracking and resume support
- Update crawler selection: docs sites now default to seo-aiohttp
- Update crawl_config.yaml with seo-aiohttp routing rules
- Update pipeline orchestrator with flag resolution and skip paths
- Update README.md and USER-GUIDE.md with raw mode documentation

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

custom-skills/90-reference-curator/shared/config/crawl_config.yaml

@@ -6,13 +6,22 @@
 #   REFERENCE_LIBRARY_PATH - Path to reference library storage (optional)
 
 # Default crawler backend
-# Options: nodejs, aiohttp, scrapy, firecrawl
-# Set to "firecrawl" if local crawlers are not available
-default_crawler: ${DEFAULT_CRAWLER:-firecrawl}
+# Options: seo-aiohttp, firecrawl, nodejs, aiohttp, scrapy
+# seo-aiohttp: Uses OurSEO BaseAsyncClient + PageAnalyzer (recommended for docs)
+# firecrawl: MCP-based, best for SPAs/JS-rendered sites
+default_crawler: ${DEFAULT_CRAWLER:-seo-aiohttp}
 
 # Intelligent routing rules
 # Claude will select the appropriate crawler based on these criteria
 routing:
+  seo-aiohttp:
+    conditions:
+      - is_documentation_site == true
+      - has_sitemap == true
+      - needs_metadata == true
+      - max_pages <= 200
+    description: "OurSEO async crawler — rate-limited, resumable, full metadata extraction. Best for documentation sites."
+
   nodejs:
     conditions:
       - max_pages <= 50
@@ -42,8 +51,22 @@ routing:
     description: "JS rendering - best for SPAs and dynamic content (MCP-based, always available)"
 
 # Crawler locations (configurable via environment)
-# If CRAWLER_PROJECT_PATH is not set, only Firecrawl MCP will be available
 crawlers:
+  seo-aiohttp:
+    enabled: true  # Always available (uses OurSEO scripts from 12-seo-technical-audit)
+    type: local
+    path: ${SKILL_PATH:-../../../90-reference-curator}/02-web-crawler-orchestrator/code/scripts/
+    command: python seo_crawler_adapter.py
+    capabilities:
+      - rate_limiting
+      - progress_tracking
+      - resume_support
+      - metadata_extraction
+      - sitemap_discovery
+      - link_discovery
+      - raw_file_output
+    install: "pip install requests beautifulsoup4 click rich tenacity"
+
   nodejs:
     enabled: ${NODEJS_CRAWLER_ENABLED:-false}
     path: ${CRAWLER_PROJECT_PATH}/util/js-crawler/