diff --git a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md index c6879cb..4fbd6bb 100644 --- a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/CLAUDE.md @@ -134,6 +134,7 @@ All crawlers respect these limits: - 20 requests/minute (SEO crawler: configurable via `BaseAsyncClient`) - 3-5 concurrent requests (SEO crawler: semaphore-controlled) - Exponential backoff on 429/5xx (SEO crawler: tenacity) +- Use `--delay` flag for polite crawling (recommended: 2-3s for Google properties) ## Error Handling @@ -142,6 +143,56 @@ All crawlers respect these limits: | Timeout | Retry with exponential backoff (max 3 attempts) | | Rate limit (429) | Token-bucket + exponential backoff | | Not found (404) | Log and skip | + +## ⚠️ GOTCHA: IP Block / CAPTCHA Detection & Wayback Fallback + +**Problem:** Large-volume crawls on sites like `support.google.com` trigger IP-level bot detection. Google redirects to `google.com/sorry/` with a CAPTCHA. Once triggered: +- Direct `requests`/`curl` returns 429 indefinitely (not a temporary rate limit) +- Playwright headless/headful also gets blocked (Google detects automation) +- The block persists even after the user solves the CAPTCHA in their browser (cookie-based, not IP-based lift) +- `WebFetch` and `read_webpage` MCP tools also fail (same IP or server-side block) + +**Detection:** Check for these signals in crawl responses: +1. HTTP 302 redirect to `google.com/sorry/index?continue=...` +2. Response body contains `solveSimpleChallenge` or `unusual traffic` +3. Persistent 429 after retries with exponential backoff (>3 consecutive 429s) + +**Solution: Wayback Machine Detour** + +When IP block / CAPTCHA is detected, automatically fall back to Wayback Machine: + +```bash +# Instead of: https://support.google.com/analytics/answer/9304153 +# Fetch from: https://web.archive.org/web/2024/https://support.google.com/analytics/answer/9304153 +``` + +The SEO crawler adapter supports this with the `--wayback-fallback` flag: + +```bash +uv run python scripts/seo_crawler_adapter.py crawl \ + --urls "@urls.txt" \ + --output ~/Documents/reference-library/topic/raw/ \ + --delay 1.5 \ + --wayback-fallback +``` + +**How it works:** +1. First attempts direct fetch for each URL +2. On 429 or CAPTCHA redirect, flags `captcha_detected` +3. After 3 consecutive CAPTCHA blocks, switches ALL remaining URLs to Wayback prefix +4. Wayback URLs use `https://web.archive.org/web/2024/{original_url}` +5. Frontmatter records `source: wayback` for traceability + +**Limitations:** +- Wayback snapshots may be weeks/months old (acceptable for documentation/help pages) +- Not all pages are archived (rare for major sites like Google, MDN, etc.) +- Wayback has its own rate limits (gentler — 1.5s delay is sufficient) +- Links in content point to Wayback URLs — the extractor strips these automatically + +**When to use `--delay` vs `--wayback-fallback`:** +- Start with `--delay 3.0` for large Google crawls (preventive) +- If 429s still occur, add `--wayback-fallback` (reactive) +- For non-Google sites, `--delay 2.0` is usually sufficient | Access denied (403) | Log, mark as `failed` | | JS rendering needed | Switch to Firecrawl MCP | | Connection error | Retry with backoff, then skip | diff --git a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/seo_crawler_adapter.py b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/seo_crawler_adapter.py index bd571d0..d01bfb4 100644 --- a/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/seo_crawler_adapter.py +++ b/custom-skills/90-reference-curator/02-web-crawler-orchestrator/code/scripts/seo_crawler_adapter.py @@ -293,16 +293,46 @@ def discover_from_sitemap(sitemap_url: str, max_urls: int = 100) -> list[str]: return urls[:max_urls] +WAYBACK_PREFIX = "https://web.archive.org/web/2024/" +CAPTCHA_THRESHOLD = 3 # Consecutive 429/CAPTCHA before switching to Wayback + + +def _is_captcha_response(resp: requests.Response) -> bool: + """Detect Google CAPTCHA / IP block responses.""" + if resp.status_code == 429: + return True + # Google sorry page redirect (302 → google.com/sorry/) + if resp.status_code == 302 and "google.com/sorry" in resp.headers.get("location", ""): + return True + # Check response body for CAPTCHA markers + if resp.status_code == 200: + body = resp.text[:2000].lower() + if "unusual traffic" in body or "solvesimplechallenge" in body or "google.com/sorry" in body: + return True + return False + + def crawl_urls( urls: list[str], output_dir: Path, include_metadata: bool = True, run_id: str | None = None, + delay: float = 0.0, + max_retries: int = 3, + wayback_fallback: bool = False, ) -> dict: """Crawl a list of URLs and write .raw.md files. + Args: + delay: Seconds to wait between requests (polite crawling). + max_retries: Number of retries on 429/5xx with exponential backoff. + wayback_fallback: Auto-switch to Wayback Machine on persistent IP block/CAPTCHA. + Returns crawl statistics dictionary. """ + import time + import random + output_dir.mkdir(parents=True, exist_ok=True) analyzer = PageAnalyzer(timeout=20) @@ -312,6 +342,31 @@ def crawl_urls( ) results = [] + consecutive_captcha = 0 + using_wayback = False + + def _fetch_with_retry(url: str) -> requests.Response | None: + """Fetch URL with retry on 429/5xx and exponential backoff.""" + last_resp = None + for attempt in range(max_retries + 1): + try: + last_resp = requests.get(url, timeout=20, + headers={"User-Agent": PageAnalyzer.DEFAULT_USER_AGENT}) + if last_resp.status_code == 429 or last_resp.status_code >= 500: + if attempt < max_retries: + wait = (2 ** attempt) * 2 + random.uniform(1.0, 3.0) + logger.info(f" ↳ {last_resp.status_code} — retry {attempt+1}/{max_retries} in {wait:.1f}s") + time.sleep(wait) + continue + return last_resp + except requests.RequestException as e: + if attempt < max_retries: + wait = (2 ** attempt) * 2 + random.uniform(1.0, 3.0) + logger.info(f" ↳ Error — retry {attempt+1}/{max_retries} in {wait:.1f}s") + time.sleep(wait) + else: + raise + return last_resp with Progress( SpinnerColumn(), @@ -325,27 +380,72 @@ def crawl_urls( for i, url in enumerate(urls): progress.current_url = url - pbar.update(task, description=f"[cyan]{urlparse(url).path[:40]}...") + + # Determine fetch URL (direct or Wayback) + fetch_url = url + source = "direct" + if using_wayback: + fetch_url = WAYBACK_PREFIX + url + source = "wayback" + pbar.update(task, description=f"[yellow]WB[/yellow] [cyan]{urlparse(url).path[:35]}...") + else: + pbar.update(task, description=f"[cyan]{urlparse(url).path[:40]}...") try: + # Polite delay between requests + if delay > 0 and i > 0: + time.sleep(delay) + # Fetch and analyze - metadata = analyzer.analyze_url(url) if include_metadata else None + metadata = analyzer.analyze_url(url) if (include_metadata and not using_wayback) else None - # Get raw HTML for content extraction - resp = requests.get(url, timeout=20, - headers={"User-Agent": PageAnalyzer.DEFAULT_USER_AGENT}) + # Get raw HTML for content extraction (with retry) + resp = _fetch_with_retry(fetch_url) - if resp.status_code == 200: + # CAPTCHA / IP block detection + if resp and not using_wayback and _is_captcha_response(resp): + consecutive_captcha += 1 + logger.warning(f" ⚠ CAPTCHA/IP block detected ({consecutive_captcha}/{CAPTCHA_THRESHOLD})") + + if wayback_fallback and consecutive_captcha >= CAPTCHA_THRESHOLD: + using_wayback = True + console.print( + f"\n[bold yellow]⚠ CAPTCHA DETOUR:[/bold yellow] " + f"IP blocked after {consecutive_captcha} consecutive 429s. " + f"Switching to Wayback Machine for remaining {len(urls) - i} URLs.\n" + ) + # Retry current URL via Wayback + fetch_url = WAYBACK_PREFIX + url + source = "wayback" + resp = _fetch_with_retry(fetch_url) + elif not wayback_fallback and consecutive_captcha >= CAPTCHA_THRESHOLD: + console.print( + f"\n[bold red]⚠ IP BLOCKED:[/bold red] " + f"{consecutive_captcha} consecutive CAPTCHA/429 responses. " + f"Consider re-running with --wayback-fallback flag.\n" + ) + + if resp and resp.status_code == 200 and not _is_captcha_response(resp): + if not using_wayback: + consecutive_captcha = 0 # Reset on success content = _extract_main_content(resp.text, url) filepath = _write_raw_file(output_dir, i, url, content, metadata) + # Record source in frontmatter by re-reading and patching + if source == "wayback": + existing = filepath.read_text() + existing = existing.replace("crawler: seo-aiohttp", "crawler: seo-aiohttp\nsource: wayback", 1) + if "source: wayback" not in existing: + existing = existing.replace("---\n\n", f"source: {source}\n---\n\n", 1) + filepath.write_text(existing) results.append({"url": url, "file": filepath.name, "status": "ok", - "size": len(content)}) + "size": len(content), "source": source}) progress.succeeded += 1 - logger.info(f"OK [{resp.status_code}] {url} → {filepath.name}") + logger.info(f"OK [{resp.status_code}] {url} → {filepath.name} ({source})") else: - results.append({"url": url, "status": f"http_{resp.status_code}"}) + status = f"http_{resp.status_code}" if resp else "no_response" + results.append({"url": url, "status": status}) progress.failed += 1 - logger.warning(f"FAIL [{resp.status_code}] {url}") + logger.warning(f"FAIL [{status}] {url}") except Exception as e: results.append({"url": url, "status": "error", "error": str(e)}) @@ -406,7 +506,9 @@ def cli(): @click.option("--output", required=True, type=click.Path(), help="Output directory for .raw.md files") @click.option("--no-metadata", is_flag=True, help="Skip SEO metadata extraction") @click.option("--run-id", help="Custom run ID for progress tracking") -def crawl(urls, output, no_metadata, run_id): +@click.option("--delay", default=0.0, type=float, help="Seconds between requests (polite crawling)") +@click.option("--wayback-fallback", is_flag=True, help="Auto-switch to Wayback Machine on CAPTCHA/IP block") +def crawl(urls, output, no_metadata, run_id, delay, wayback_fallback): """Crawl a list of URLs and write .raw.md files.""" if urls.startswith("@"): url_list = Path(urls[1:]).read_text().strip().splitlines() @@ -414,9 +516,13 @@ def crawl(urls, output, no_metadata, run_id): url_list = [u.strip() for u in urls.split(",") if u.strip()] output_path = Path(output).expanduser() - stats = crawl_urls(url_list, output_path, include_metadata=not no_metadata, run_id=run_id) + stats = crawl_urls(url_list, output_path, include_metadata=not no_metadata, run_id=run_id, + delay=delay, wayback_fallback=wayback_fallback) + wayback_count = sum(1 for r in stats.get("results", []) if r.get("source") == "wayback") console.print(f"\n[green]Done![/green] {stats['succeeded']}/{stats['total_urls']} pages crawled") + if wayback_count: + console.print(f"[yellow]Wayback fallback used for {wayback_count} pages[/yellow]") console.print(f"Output: {stats['output_dir']}") diff --git a/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md b/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md index d22ea5d..1438138 100644 --- a/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md +++ b/custom-skills/90-reference-curator/07-pipeline-orchestrator/code/CLAUDE.md @@ -100,12 +100,22 @@ If mode == 'topic': ### Stage 2: Web Crawler ``` -→ Claude uses Firecrawl MCP tools +→ Claude uses Firecrawl MCP tools or seo_crawler_adapter.py → crawl_mgr.py store-result → If --save-raw or --no-distill: write raw .raw.md files to output → pipeline.py advance --stage storing ``` +**⚠️ GOTCHA — IP Block / CAPTCHA Detour:** +If the crawl stage encounters persistent 429 or CAPTCHA responses (e.g., Google's `google.com/sorry/` redirect), the pipeline should **NOT** retry indefinitely. Instead: +1. Detect: 3+ consecutive 429/CAPTCHA responses = IP blocked +2. Detour: Re-invoke `seo_crawler_adapter.py` with `--wayback-fallback` flag +3. The crawler auto-switches remaining URLs to `web.archive.org/web/2024/{url}` +4. Frontmatter records `source: wayback` for traceability +5. Resume normal pipeline from Stage 3 + +**Common triggers:** Large-volume crawls (>50 pages) on `support.google.com`, `developer.mozilla.org`, or any site with aggressive bot detection. **Prevention:** Always use `--delay 2.0` or higher for these domains. + ### Stage 3: Content Repository ``` → repo.py store (for each crawled doc)