Andrew Yim
f215c11c32
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
"""Common utilities for the reference curator pipeline."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
import unicodedata
|
|
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
|
|
|
|
|
def normalize_url(url: str) -> str:
|
|
"""Normalize a URL for deduplication.
|
|
|
|
- Lowercase scheme and host
|
|
- Remove trailing slashes
|
|
- Sort query parameters
|
|
- Remove common tracking params (utm_*, ref, fbclid)
|
|
- Remove fragment
|
|
"""
|
|
parsed = urlparse(url)
|
|
scheme = parsed.scheme.lower()
|
|
netloc = parsed.netloc.lower()
|
|
path = parsed.path.rstrip("/") or "/"
|
|
|
|
# Sort query params, removing tracking params
|
|
tracking_params = {"utm_source", "utm_medium", "utm_campaign", "utm_content",
|
|
"utm_term", "ref", "fbclid", "gclid", "mc_cid", "mc_eid"}
|
|
params = parse_qs(parsed.query, keep_blank_values=True)
|
|
filtered = {k: v for k, v in sorted(params.items()) if k not in tracking_params}
|
|
query = urlencode(filtered, doseq=True)
|
|
|
|
return urlunparse((scheme, netloc, path, "", query, ""))
|
|
|
|
|
|
def url_hash(url: str) -> str:
|
|
"""SHA-256 hash of normalized URL. Matches the url_hash column in schema.sql."""
|
|
return hashlib.sha256(normalize_url(url).encode()).hexdigest()
|
|
|
|
|
|
def slugify(text: str) -> str:
|
|
"""Convert text to a URL/folder-friendly slug.
|
|
|
|
>>> slugify("Prompt Engineering Best Practices")
|
|
'prompt-engineering-best-practices'
|
|
"""
|
|
text = unicodedata.normalize("NFKD", text)
|
|
text = text.encode("ascii", "ignore").decode()
|
|
text = text.lower()
|
|
text = re.sub(r"[^a-z0-9]+", "-", text)
|
|
text = text.strip("-")
|
|
return text or "untitled"
|
|
|
|
|
|
def count_tokens(text: str) -> int:
|
|
"""Approximate token count using chars/4 heuristic.
|
|
|
|
Good enough for compression ratio calculations without requiring tiktoken.
|
|
"""
|
|
return max(1, len(text) // 4)
|
|
|
|
|
|
def setup_logging(level: str = "INFO", run_id: int | None = None) -> logging.Logger:
|
|
"""Configure and return a logger for the reference curator pipeline."""
|
|
logger = logging.getLogger("refcurator")
|
|
if not logger.handlers:
|
|
handler = logging.StreamHandler()
|
|
fmt = "[refcurator]"
|
|
if run_id:
|
|
fmt += f" [run:{run_id}]"
|
|
fmt += " %(levelname)s: %(message)s"
|
|
handler.setFormatter(logging.Formatter(fmt))
|
|
logger.addHandler(handler)
|
|
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
|
return logger
|