feat(reference-curator): implement Python scripts + Gemini quality gate
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
34
custom-skills/90-reference-curator/shared/lib/pyproject.toml
Normal file
34
custom-skills/90-reference-curator/shared/lib/pyproject.toml
Normal file
@@ -0,0 +1,34 @@
|
||||
[project]
|
||||
name = "refcurator"
|
||||
version = "1.0.0"
|
||||
description = "Reference Curator — data management for the reference curation pipeline"
|
||||
requires-python = ">=3.12"
|
||||
license = "MIT"
|
||||
authors = [
|
||||
{ name = "Andrew Yim", email = "andrew@ourdigital.org" },
|
||||
]
|
||||
dependencies = [
|
||||
"pymysql>=1.1.0",
|
||||
"click>=8.0",
|
||||
"pydantic>=2.0",
|
||||
"pyyaml>=6.0",
|
||||
"rich>=13.0",
|
||||
"python-dotenv>=1.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"ruff>=0.4",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Reference Curator — data management for the reference curation pipeline."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Configuration loading for the reference curator pipeline.
|
||||
|
||||
Loads YAML configs from ~/.config/reference-curator/ with env var substitution.
|
||||
Falls back to bundled defaults in shared/config/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load user env if present
|
||||
_env_file = Path.home() / ".reference-curator.env"
|
||||
if _env_file.is_file():
|
||||
load_dotenv(_env_file)
|
||||
|
||||
# Config search paths (user override → bundled defaults)
|
||||
USER_CONFIG_DIR = Path.home() / ".config" / "reference-curator"
|
||||
BUNDLED_CONFIG_DIR = Path(__file__).resolve().parents[4] / "config" # shared/config/
|
||||
|
||||
# Default storage paths
|
||||
DEFAULT_LIBRARY_PATH = Path(
|
||||
os.environ.get("REFERENCE_LIBRARY_PATH", "~/Documents/reference-library")
|
||||
).expanduser()
|
||||
DEFAULT_STATE_DIR = DEFAULT_LIBRARY_PATH / "pipeline_state"
|
||||
|
||||
|
||||
def _expand_env_vars(value: str) -> str:
|
||||
"""Expand ${VAR:-default} patterns in a string."""
|
||||
def _replace(match: re.Match) -> str:
|
||||
var_expr = match.group(1)
|
||||
if ":-" in var_expr:
|
||||
var_name, default = var_expr.split(":-", 1)
|
||||
return os.environ.get(var_name, default)
|
||||
return os.environ.get(var_expr, match.group(0))
|
||||
|
||||
return re.sub(r"\$\{([^}]+)}", _replace, value)
|
||||
|
||||
|
||||
def _expand_recursive(obj: Any) -> Any:
|
||||
"""Recursively expand env vars in a parsed YAML structure."""
|
||||
if isinstance(obj, str):
|
||||
expanded = _expand_env_vars(obj)
|
||||
# Expand ~ in path-like strings
|
||||
if expanded.startswith("~"):
|
||||
expanded = str(Path(expanded).expanduser())
|
||||
return expanded
|
||||
if isinstance(obj, dict):
|
||||
return {k: _expand_recursive(v) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_expand_recursive(item) for item in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def load_config(name: str) -> dict:
|
||||
"""Load a YAML config file by name (without extension).
|
||||
|
||||
Searches user config dir first, then bundled defaults.
|
||||
Expands ${VAR:-default} env var patterns in all string values.
|
||||
|
||||
Args:
|
||||
name: Config file name without .yaml extension
|
||||
(e.g., "db_config", "pipeline_config", "crawl_config", "export_config")
|
||||
|
||||
Returns:
|
||||
Parsed and expanded config dict.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If config file not found in any search path.
|
||||
"""
|
||||
filename = f"{name}.yaml"
|
||||
|
||||
for config_dir in [USER_CONFIG_DIR, BUNDLED_CONFIG_DIR]:
|
||||
config_path = config_dir / filename
|
||||
if config_path.is_file():
|
||||
with open(config_path) as f:
|
||||
raw = yaml.safe_load(f) or {}
|
||||
return _expand_recursive(raw)
|
||||
|
||||
raise FileNotFoundError(
|
||||
f"Config '{filename}' not found in {USER_CONFIG_DIR} or {BUNDLED_CONFIG_DIR}"
|
||||
)
|
||||
|
||||
|
||||
def get_db_config() -> dict:
|
||||
"""Load database configuration."""
|
||||
return load_config("db_config")
|
||||
|
||||
|
||||
def get_pipeline_config() -> dict:
|
||||
"""Load pipeline orchestrator configuration."""
|
||||
return load_config("pipeline_config")
|
||||
|
||||
|
||||
def get_crawl_config() -> dict:
|
||||
"""Load crawler configuration."""
|
||||
return load_config("crawl_config")
|
||||
|
||||
|
||||
def get_export_config() -> dict:
|
||||
"""Load export configuration."""
|
||||
return load_config("export_config")
|
||||
|
||||
|
||||
def get_library_path() -> Path:
|
||||
"""Get the reference library base path."""
|
||||
return DEFAULT_LIBRARY_PATH
|
||||
|
||||
|
||||
def get_state_dir() -> Path:
|
||||
"""Get the pipeline state directory."""
|
||||
try:
|
||||
cfg = get_pipeline_config()
|
||||
state_dir = cfg.get("state", {}).get("state_directory")
|
||||
if state_dir:
|
||||
return Path(state_dir).expanduser()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return DEFAULT_STATE_DIR
|
||||
|
||||
|
||||
def get_state_backend() -> str:
|
||||
"""Get the state backend type: 'mysql' or 'file'."""
|
||||
try:
|
||||
cfg = get_pipeline_config()
|
||||
return cfg.get("state", {}).get("backend", "file")
|
||||
except FileNotFoundError:
|
||||
return "file"
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Database abstraction layer with MySQL and file-based backends.
|
||||
|
||||
Usage:
|
||||
from refcurator.db import get_backend
|
||||
|
||||
db = get_backend()
|
||||
rows = db.fetch_all("SELECT * FROM documents WHERE crawl_status = %s", ("pending",))
|
||||
doc_id = db.insert_returning_id("INSERT INTO documents (...) VALUES (...)", params)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Protocol, Sequence
|
||||
|
||||
from refcurator.config import get_db_config, get_state_backend, get_state_dir
|
||||
|
||||
logger = logging.getLogger("refcurator.db")
|
||||
|
||||
|
||||
class DatabaseBackend(Protocol):
|
||||
"""Protocol for database backends."""
|
||||
|
||||
def execute(self, sql: str, params: Sequence = ()) -> int:
|
||||
"""Execute a statement, return affected row count."""
|
||||
...
|
||||
|
||||
def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]:
|
||||
"""Fetch a single row as dict."""
|
||||
...
|
||||
|
||||
def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]:
|
||||
"""Fetch all rows as list of dicts."""
|
||||
...
|
||||
|
||||
def insert_returning_id(self, sql: str, params: Sequence = ()) -> int:
|
||||
"""Insert a row and return the auto-generated ID."""
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
...
|
||||
|
||||
|
||||
class MySQLBackend:
|
||||
"""MySQL backend using PyMySQL."""
|
||||
|
||||
def __init__(self, config: dict | None = None):
|
||||
import pymysql
|
||||
import pymysql.cursors
|
||||
|
||||
if config is None:
|
||||
config = get_db_config().get("mysql", {})
|
||||
|
||||
self._conn = pymysql.connect(
|
||||
host=config.get("host", "localhost"),
|
||||
port=int(config.get("port", 3306)),
|
||||
user=config.get("user", "root"),
|
||||
password=config.get("password", ""),
|
||||
database=config.get("database", "reference_library"),
|
||||
charset="utf8mb4",
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
autocommit=True,
|
||||
)
|
||||
|
||||
def execute(self, sql: str, params: Sequence = ()) -> int:
|
||||
with self._conn.cursor() as cur:
|
||||
return cur.execute(sql, params)
|
||||
|
||||
def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.fetchone()
|
||||
|
||||
def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.fetchall()
|
||||
|
||||
def insert_returning_id(self, sql: str, params: Sequence = ()) -> int:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.lastrowid
|
||||
|
||||
def close(self) -> None:
|
||||
self._conn.close()
|
||||
|
||||
|
||||
class FileBackend:
|
||||
"""JSON file-based backend for use without MySQL.
|
||||
|
||||
Stores data as JSON arrays in the state directory.
|
||||
Supports basic CRUD but not complex queries or JOINs.
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: Path | None = None):
|
||||
self._dir = state_dir or get_state_dir()
|
||||
self._dir.mkdir(parents=True, exist_ok=True)
|
||||
self._cache: dict[str, list[dict]] = {}
|
||||
self._counters: dict[str, int] = {}
|
||||
self._load_counters()
|
||||
|
||||
def _table_path(self, table: str) -> Path:
|
||||
return self._dir / f"{table}.json"
|
||||
|
||||
def _load_table(self, table: str) -> list[dict]:
|
||||
if table not in self._cache:
|
||||
path = self._table_path(table)
|
||||
if path.is_file():
|
||||
self._cache[table] = json.loads(path.read_text())
|
||||
else:
|
||||
self._cache[table] = []
|
||||
return self._cache[table]
|
||||
|
||||
def _save_table(self, table: str) -> None:
|
||||
path = self._table_path(table)
|
||||
path.write_text(json.dumps(self._cache.get(table, []), indent=2, default=str))
|
||||
|
||||
def _load_counters(self) -> None:
|
||||
counter_path = self._dir / "_counters.json"
|
||||
if counter_path.is_file():
|
||||
self._counters = json.loads(counter_path.read_text())
|
||||
|
||||
def _save_counters(self) -> None:
|
||||
counter_path = self._dir / "_counters.json"
|
||||
counter_path.write_text(json.dumps(self._counters, indent=2))
|
||||
|
||||
def _next_id(self, table: str) -> int:
|
||||
current = self._counters.get(table, 0)
|
||||
self._counters[table] = current + 1
|
||||
self._save_counters()
|
||||
return current + 1
|
||||
|
||||
# --- Protocol methods ---
|
||||
# These provide basic support for the most common operations.
|
||||
# Complex SQL is not supported; use MySQL for full functionality.
|
||||
|
||||
def execute(self, sql: str, params: Sequence = ()) -> int:
|
||||
"""Basic INSERT/UPDATE/DELETE support via SQL pattern matching."""
|
||||
sql_lower = sql.strip().lower()
|
||||
table = _extract_table_name(sql)
|
||||
|
||||
if sql_lower.startswith("insert"):
|
||||
return self._handle_insert(table, sql, params)
|
||||
elif sql_lower.startswith("update"):
|
||||
return self._handle_update(table, sql, params)
|
||||
elif sql_lower.startswith("delete"):
|
||||
return self._handle_delete(table, sql, params)
|
||||
|
||||
logger.warning("FileBackend: unsupported SQL operation: %s", sql[:60])
|
||||
return 0
|
||||
|
||||
def fetch_one(self, sql: str, params: Sequence = ()) -> Optional[dict]:
|
||||
rows = self.fetch_all(sql, params)
|
||||
return rows[0] if rows else None
|
||||
|
||||
def fetch_all(self, sql: str, params: Sequence = ()) -> list[dict]:
|
||||
table = _extract_table_name(sql)
|
||||
if not table:
|
||||
logger.warning("FileBackend: cannot extract table from: %s", sql[:60])
|
||||
return []
|
||||
|
||||
# Hard-fail on SQL patterns that FileBackend cannot handle correctly
|
||||
_reject_unsupported_sql(sql)
|
||||
|
||||
rows = self._load_table(table)
|
||||
|
||||
# Basic WHERE clause filtering
|
||||
conditions = _extract_where_conditions(sql, params)
|
||||
if conditions:
|
||||
rows = [r for r in rows if _matches_conditions(r, conditions)]
|
||||
|
||||
# Basic ORDER BY
|
||||
order_col = _extract_order_by(sql)
|
||||
if order_col:
|
||||
desc = "desc" in sql.lower().split("order by")[-1].lower()
|
||||
rows = sorted(rows, key=lambda r: r.get(order_col, ""), reverse=desc)
|
||||
|
||||
# Basic LIMIT
|
||||
limit = _extract_limit(sql)
|
||||
if limit is not None:
|
||||
rows = rows[:limit]
|
||||
|
||||
return rows
|
||||
|
||||
def insert_returning_id(self, sql: str, params: Sequence = ()) -> int:
|
||||
table = _extract_table_name(sql)
|
||||
self._handle_insert(table, sql, params)
|
||||
return self._counters.get(table, 0)
|
||||
|
||||
def close(self) -> None:
|
||||
pass # No connection to close
|
||||
|
||||
# --- Internal handlers ---
|
||||
|
||||
def _handle_insert(self, table: str, sql: str, params: Sequence) -> int:
|
||||
columns = _extract_insert_columns(sql)
|
||||
if not columns or len(columns) != len(params):
|
||||
logger.warning("FileBackend: column/param mismatch for INSERT into %s", table)
|
||||
return 0
|
||||
|
||||
row = dict(zip(columns, params))
|
||||
pk = _primary_key_for(table)
|
||||
if pk and pk not in row:
|
||||
row[pk] = self._next_id(table)
|
||||
|
||||
rows = self._load_table(table)
|
||||
rows.append(row)
|
||||
self._cache[table] = rows
|
||||
self._save_table(table)
|
||||
return 1
|
||||
|
||||
def _handle_update(self, table: str, sql: str, params: Sequence) -> int:
|
||||
rows = self._load_table(table)
|
||||
set_cols = _extract_set_columns(sql)
|
||||
conditions = _extract_where_conditions(sql, params[len(set_cols):])
|
||||
set_values = list(params[:len(set_cols)])
|
||||
|
||||
count = 0
|
||||
for row in rows:
|
||||
if _matches_conditions(row, conditions):
|
||||
for col, val in zip(set_cols, set_values):
|
||||
row[col] = val
|
||||
count += 1
|
||||
|
||||
if count > 0:
|
||||
self._save_table(table)
|
||||
return count
|
||||
|
||||
def _handle_delete(self, table: str, sql: str, params: Sequence) -> int:
|
||||
rows = self._load_table(table)
|
||||
conditions = _extract_where_conditions(sql, params)
|
||||
before = len(rows)
|
||||
self._cache[table] = [r for r in rows if not _matches_conditions(r, conditions)]
|
||||
self._save_table(table)
|
||||
return before - len(self._cache[table])
|
||||
|
||||
|
||||
class UnsupportedQueryError(Exception):
|
||||
"""Raised when FileBackend encounters SQL it cannot handle correctly."""
|
||||
pass
|
||||
|
||||
|
||||
def _reject_unsupported_sql(sql: str) -> None:
|
||||
"""Raise UnsupportedQueryError if the SQL uses patterns FileBackend cannot handle.
|
||||
|
||||
FileBackend only supports single-table SELECT with simple WHERE col = %s.
|
||||
JOINs, subqueries, aggregates, and GROUP BY would return wrong results silently.
|
||||
"""
|
||||
import re
|
||||
sql_upper = sql.upper()
|
||||
|
||||
unsupported = []
|
||||
if re.search(r"\bJOIN\b", sql_upper):
|
||||
unsupported.append("JOIN")
|
||||
if re.search(r"\bGROUP\s+BY\b", sql_upper):
|
||||
unsupported.append("GROUP BY")
|
||||
if re.search(r"\b(MAX|MIN|SUM|AVG|COUNT)\s*\(", sql_upper):
|
||||
unsupported.append("aggregate functions")
|
||||
if re.search(r"\bLEFT\s+JOIN\b", sql_upper):
|
||||
unsupported.append("LEFT JOIN")
|
||||
if re.search(r"\(\s*SELECT\b", sql_upper):
|
||||
unsupported.append("subquery")
|
||||
|
||||
if unsupported:
|
||||
raise UnsupportedQueryError(
|
||||
f"FileBackend cannot execute queries with {', '.join(unsupported)}. "
|
||||
f"Configure MySQL or use 'backend: mysql' in pipeline_config.yaml. "
|
||||
f"Query: {sql[:80]}..."
|
||||
)
|
||||
|
||||
|
||||
# --- SQL parsing helpers (minimal, covers common patterns) ---
|
||||
|
||||
def _extract_table_name(sql: str) -> str:
|
||||
"""Extract the primary table name from a SQL statement."""
|
||||
import re
|
||||
sql_clean = sql.strip()
|
||||
|
||||
# INSERT INTO table
|
||||
m = re.search(r"(?:insert\s+into|update|delete\s+from|from)\s+(\w+)", sql_clean, re.I)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_insert_columns(sql: str) -> list[str]:
|
||||
"""Extract column names from INSERT INTO table (col1, col2, ...)."""
|
||||
import re
|
||||
m = re.search(r"\(([^)]+)\)\s*VALUES", sql, re.I)
|
||||
if m:
|
||||
return [c.strip() for c in m.group(1).split(",")]
|
||||
return []
|
||||
|
||||
|
||||
def _extract_set_columns(sql: str) -> list[str]:
|
||||
"""Extract column names from UPDATE ... SET col1 = %s, col2 = %s."""
|
||||
import re
|
||||
m = re.search(r"SET\s+(.+?)(?:\s+WHERE|$)", sql, re.I | re.S)
|
||||
if m:
|
||||
return [c.strip().split("=")[0].strip() for c in m.group(1).split(",")]
|
||||
return []
|
||||
|
||||
|
||||
def _extract_where_conditions(sql: str, params: Sequence) -> list[tuple[str, Any]]:
|
||||
"""Extract simple col = %s conditions from WHERE clause."""
|
||||
import re
|
||||
m = re.search(r"WHERE\s+(.+?)(?:\s+ORDER|\s+LIMIT|$)", sql, re.I | re.S)
|
||||
if not m:
|
||||
return []
|
||||
|
||||
where_clause = m.group(1)
|
||||
cols = re.findall(r"(\w+)\s*=\s*%s", where_clause)
|
||||
# Map to params (taking from the end of params for UPDATE, all for SELECT)
|
||||
return list(zip(cols, params[-len(cols):] if cols else []))
|
||||
|
||||
|
||||
def _extract_order_by(sql: str) -> str:
|
||||
"""Extract the first ORDER BY column."""
|
||||
import re
|
||||
m = re.search(r"ORDER\s+BY\s+(\w+)", sql, re.I)
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
def _extract_limit(sql: str) -> int | None:
|
||||
"""Extract LIMIT value."""
|
||||
import re
|
||||
m = re.search(r"LIMIT\s+(\d+)", sql, re.I)
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _matches_conditions(row: dict, conditions: list[tuple[str, Any]]) -> bool:
|
||||
"""Check if a row matches all WHERE conditions."""
|
||||
return all(str(row.get(col)) == str(val) for col, val in conditions)
|
||||
|
||||
|
||||
def _primary_key_for(table: str) -> str:
|
||||
"""Return the auto-increment primary key name for a table."""
|
||||
pk_map = {
|
||||
"sources": "source_id",
|
||||
"documents": "doc_id",
|
||||
"distilled_content": "distill_id",
|
||||
"review_logs": "review_id",
|
||||
"topics": "topic_id",
|
||||
"export_jobs": "export_id",
|
||||
"pipeline_runs": "run_id",
|
||||
"pipeline_iteration_tracker": "tracker_id",
|
||||
"crawl_schedule": "schedule_id",
|
||||
"change_detection": "change_id",
|
||||
}
|
||||
return pk_map.get(table, "")
|
||||
|
||||
|
||||
# --- Factory ---
|
||||
|
||||
def get_backend(backend_type: str | None = None) -> DatabaseBackend:
|
||||
"""Create and return the appropriate database backend.
|
||||
|
||||
Args:
|
||||
backend_type: 'mysql' or 'file'. If None, reads from pipeline config.
|
||||
|
||||
Returns:
|
||||
A DatabaseBackend instance.
|
||||
"""
|
||||
if backend_type is None:
|
||||
backend_type = get_state_backend()
|
||||
|
||||
if backend_type == "mysql":
|
||||
return MySQLBackend()
|
||||
|
||||
return FileBackend()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db_session(backend_type: str | None = None):
|
||||
"""Context manager for database sessions.
|
||||
|
||||
Usage:
|
||||
with db_session() as db:
|
||||
rows = db.fetch_all("SELECT * FROM documents")
|
||||
"""
|
||||
db = get_backend(backend_type)
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Gemini CLI wrapper for independent content quality evaluation.
|
||||
|
||||
Uses the Gemini CLI (google/gemini-cli) to evaluate raw crawled content
|
||||
before distillation, providing third-party quality assessment.
|
||||
|
||||
Requires: `npm install -g @google/gemini-cli` and Google auth configured.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("refcurator.gemini")
|
||||
|
||||
GEMINI_CMD = "gemini"
|
||||
TIMEOUT_SECONDS = 60
|
||||
|
||||
EVALUATION_PROMPT = """You are evaluating a raw reference document for inclusion in a curated knowledge base.
|
||||
|
||||
Topic: {topic}
|
||||
Source URL: {source_url}
|
||||
|
||||
Score each criterion from 0.0 to 1.0:
|
||||
- relevance: Does this content actually relate to the topic "{topic}"?
|
||||
- authority: Is this from an authoritative, official source (official docs, research paper) or low-quality (scraped blog, forum post, SEO spam)?
|
||||
- completeness: Is this a complete article with substance, or a navigation fragment, error page, stub, or boilerplate?
|
||||
- freshness: Does the information appear current and not outdated? Look for version numbers, dates, deprecated APIs.
|
||||
- distill_value: Does this contain unique, valuable information worth summarizing, or is it redundant with what official docs already cover?
|
||||
|
||||
Return ONLY a valid JSON object with no markdown formatting, no code fences, no explanation:
|
||||
{{"relevance": 0.0, "authority": 0.0, "completeness": 0.0, "freshness": 0.0, "distill_value": 0.0, "verdict": "approve", "reason": "brief explanation"}}
|
||||
|
||||
The verdict must be one of: "approve", "reject", "deep_research"
|
||||
- approve: source is worth distilling (score >= 0.75 typical)
|
||||
- reject: not worth distilling (low quality, irrelevant, or fragment)
|
||||
- deep_research: partially relevant but needs supplementary sources"""
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Check if the Gemini CLI is installed and authenticated."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[GEMINI_CMD, "--help"],
|
||||
capture_output=True, timeout=10,
|
||||
)
|
||||
return result.returncode == 0
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_content(
|
||||
content: str,
|
||||
topic: str,
|
||||
source_url: str = "",
|
||||
timeout: int = TIMEOUT_SECONDS,
|
||||
) -> Optional[dict]:
|
||||
"""Evaluate raw content using Gemini CLI.
|
||||
|
||||
Args:
|
||||
content: Raw document content (markdown/text)
|
||||
topic: The curation topic for relevance scoring
|
||||
source_url: Original URL of the content
|
||||
timeout: Subprocess timeout in seconds
|
||||
|
||||
Returns:
|
||||
Parsed evaluation dict with scores, verdict, and reason.
|
||||
Returns None if Gemini is unavailable or evaluation fails.
|
||||
"""
|
||||
# Truncate very long content to avoid overwhelming the model
|
||||
max_chars = 50_000
|
||||
if len(content) > max_chars:
|
||||
content = content[:max_chars] + "\n\n[... content truncated for evaluation ...]"
|
||||
|
||||
prompt = EVALUATION_PROMPT.format(topic=topic, source_url=source_url)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[GEMINI_CMD, prompt],
|
||||
input=content,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.warning("Gemini CLI failed (exit %d): %s", result.returncode, result.stderr[:200])
|
||||
return None
|
||||
|
||||
return _parse_response(result.stdout)
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.warning("Gemini CLI not found. Install with: npm install -g @google/gemini-cli")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Gemini CLI timed out after %ds", timeout)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("Gemini evaluation failed: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_response(output: str) -> Optional[dict]:
|
||||
"""Parse Gemini's response, handling markdown-wrapped JSON."""
|
||||
text = output.strip()
|
||||
|
||||
# Try direct JSON parse first
|
||||
try:
|
||||
return _validate_evaluation(json.loads(text))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Try extracting JSON from markdown code fences
|
||||
m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
|
||||
if m:
|
||||
try:
|
||||
return _validate_evaluation(json.loads(m.group(1).strip()))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Try finding a JSON object anywhere in the output
|
||||
m = re.search(r"\{[^{}]*\"relevance\"[^{}]*\}", text, re.DOTALL)
|
||||
if m:
|
||||
try:
|
||||
return _validate_evaluation(json.loads(m.group(0)))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
logger.warning("Could not parse Gemini response as JSON: %s", text[:200])
|
||||
return None
|
||||
|
||||
|
||||
def _validate_evaluation(data: dict) -> Optional[dict]:
|
||||
"""Validate that the evaluation has required fields and reasonable values."""
|
||||
required_scores = ["relevance", "authority", "completeness", "freshness", "distill_value"]
|
||||
|
||||
for field in required_scores:
|
||||
if field not in data:
|
||||
logger.warning("Missing required field: %s", field)
|
||||
return None
|
||||
score = data[field]
|
||||
if not isinstance(score, (int, float)) or score < 0 or score > 1:
|
||||
logger.warning("Invalid score for %s: %s", field, score)
|
||||
return None
|
||||
|
||||
if "verdict" not in data or data["verdict"] not in ("approve", "reject", "deep_research"):
|
||||
data["verdict"] = _derive_verdict(data)
|
||||
|
||||
if "reason" not in data:
|
||||
data["reason"] = ""
|
||||
|
||||
# Add weighted score
|
||||
data["weighted_score"] = round(
|
||||
data["relevance"] * 0.25
|
||||
+ data["authority"] * 0.25
|
||||
+ data["completeness"] * 0.20
|
||||
+ data["freshness"] * 0.15
|
||||
+ data["distill_value"] * 0.15,
|
||||
4,
|
||||
)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _derive_verdict(data: dict) -> str:
|
||||
"""Derive verdict from scores if Gemini didn't provide one."""
|
||||
score = (
|
||||
data.get("relevance", 0) * 0.25
|
||||
+ data.get("authority", 0) * 0.25
|
||||
+ data.get("completeness", 0) * 0.20
|
||||
+ data.get("freshness", 0) * 0.15
|
||||
+ data.get("distill_value", 0) * 0.15
|
||||
)
|
||||
if score >= 0.75:
|
||||
return "approve"
|
||||
elif score >= 0.50:
|
||||
return "deep_research"
|
||||
else:
|
||||
return "reject"
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Manifest I/O for reference discovery and crawl results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from refcurator.models import CrawlResult, CrawlResultEntry, Manifest, ManifestURL
|
||||
from refcurator.utils import normalize_url
|
||||
|
||||
|
||||
def read_manifest(path: Path) -> Manifest:
|
||||
"""Read a manifest JSON file."""
|
||||
data = json.loads(path.read_text())
|
||||
return Manifest(**data)
|
||||
|
||||
|
||||
def write_manifest(manifest: Manifest, path: Path) -> None:
|
||||
"""Write a manifest to a JSON file."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(manifest.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def merge_manifests(manifests: list[Manifest]) -> Manifest:
|
||||
"""Merge multiple manifests, deduplicating URLs."""
|
||||
seen: dict[str, ManifestURL] = {}
|
||||
topic_parts = []
|
||||
|
||||
for m in manifests:
|
||||
if m.topic:
|
||||
topic_parts.append(m.topic)
|
||||
for url_entry in m.urls:
|
||||
normalized = normalize_url(url_entry.url)
|
||||
existing = seen.get(normalized)
|
||||
if existing is None or (
|
||||
url_entry.credibility_score
|
||||
and (existing.credibility_score or 0) < url_entry.credibility_score
|
||||
):
|
||||
seen[normalized] = url_entry
|
||||
|
||||
urls = list(seen.values())
|
||||
return Manifest(
|
||||
discovery_date=datetime.now().isoformat(),
|
||||
topic=" + ".join(topic_parts) if topic_parts else None,
|
||||
total_urls=len(urls),
|
||||
urls=urls,
|
||||
)
|
||||
|
||||
|
||||
def dedup_manifest_urls(manifest: Manifest, existing_urls: set[str]) -> Manifest:
|
||||
"""Remove URLs already in the existing set (normalized comparison)."""
|
||||
existing_normalized = {normalize_url(u) for u in existing_urls}
|
||||
filtered = [u for u in manifest.urls if normalize_url(u.url) not in existing_normalized]
|
||||
return Manifest(
|
||||
discovery_date=manifest.discovery_date,
|
||||
topic=manifest.topic,
|
||||
total_urls=len(filtered),
|
||||
urls=filtered,
|
||||
)
|
||||
|
||||
|
||||
def read_crawl_result(path: Path) -> CrawlResult:
|
||||
"""Read a crawl result JSON file."""
|
||||
data = json.loads(path.read_text())
|
||||
return CrawlResult(**data)
|
||||
|
||||
|
||||
def write_crawl_result(result: CrawlResult, path: Path) -> None:
|
||||
"""Write a crawl result to a JSON file."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(result.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def create_crawl_result(
|
||||
entries: list[dict],
|
||||
crawler: str = "firecrawl",
|
||||
) -> CrawlResult:
|
||||
"""Create a CrawlResult from a list of crawl entry dicts."""
|
||||
docs = [CrawlResultEntry(**e) for e in entries]
|
||||
completed = [d for d in docs if d.status == "completed"]
|
||||
failed = [d for d in docs if d.status != "completed"]
|
||||
|
||||
return CrawlResult(
|
||||
crawl_date=datetime.now().isoformat(),
|
||||
crawler_used=crawler,
|
||||
total_crawled=len(completed),
|
||||
total_failed=len(failed),
|
||||
documents=docs,
|
||||
)
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Pydantic v2 models matching the reference_library MySQL schema."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, computed_field
|
||||
|
||||
|
||||
# --- Enums matching MySQL ENUMs ---
|
||||
|
||||
class SourceType(str, Enum):
|
||||
official_docs = "official_docs"
|
||||
engineering_blog = "engineering_blog"
|
||||
research_paper = "research_paper"
|
||||
github_repo = "github_repo"
|
||||
community_guide = "community_guide"
|
||||
pdf_document = "pdf_document"
|
||||
api_reference = "api_reference"
|
||||
|
||||
|
||||
class CredibilityTier(str, Enum):
|
||||
tier1_official = "tier1_official"
|
||||
tier2_verified = "tier2_verified"
|
||||
tier3_community = "tier3_community"
|
||||
|
||||
|
||||
class DocType(str, Enum):
|
||||
webpage = "webpage"
|
||||
pdf = "pdf"
|
||||
markdown = "markdown"
|
||||
api_spec = "api_spec"
|
||||
code_sample = "code_sample"
|
||||
|
||||
|
||||
class Language(str, Enum):
|
||||
en = "en"
|
||||
ko = "ko"
|
||||
mixed = "mixed"
|
||||
|
||||
|
||||
class CrawlMethod(str, Enum):
|
||||
firecrawl = "firecrawl"
|
||||
scrapy = "scrapy"
|
||||
aiohttp = "aiohttp"
|
||||
nodejs = "nodejs"
|
||||
manual = "manual"
|
||||
api = "api"
|
||||
|
||||
|
||||
class CrawlStatus(str, Enum):
|
||||
pending = "pending"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
stale = "stale"
|
||||
|
||||
|
||||
class ReviewStatus(str, Enum):
|
||||
pending = "pending"
|
||||
in_review = "in_review"
|
||||
approved = "approved"
|
||||
needs_refactor = "needs_refactor"
|
||||
rejected = "rejected"
|
||||
|
||||
|
||||
class ReviewerType(str, Enum):
|
||||
auto_qa = "auto_qa"
|
||||
human = "human"
|
||||
claude_review = "claude_review"
|
||||
gemini_review = "gemini_review"
|
||||
|
||||
|
||||
class Decision(str, Enum):
|
||||
approve = "approve"
|
||||
refactor = "refactor"
|
||||
deep_research = "deep_research"
|
||||
reject = "reject"
|
||||
|
||||
|
||||
class PipelineStatus(str, Enum):
|
||||
running = "running"
|
||||
completed = "completed"
|
||||
failed = "failed"
|
||||
paused = "paused"
|
||||
|
||||
|
||||
class PipelineStage(str, Enum):
|
||||
discovery = "discovery"
|
||||
crawling = "crawling"
|
||||
storing = "storing"
|
||||
evaluating = "evaluating"
|
||||
distilling = "distilling"
|
||||
exporting = "exporting"
|
||||
|
||||
|
||||
class RunType(str, Enum):
|
||||
topic = "topic"
|
||||
urls = "urls"
|
||||
manifest = "manifest"
|
||||
|
||||
|
||||
class ExportType(str, Enum):
|
||||
project_files = "project_files"
|
||||
fine_tuning = "fine_tuning"
|
||||
training_dataset = "training_dataset"
|
||||
knowledge_base = "knowledge_base"
|
||||
|
||||
|
||||
class OutputFormat(str, Enum):
|
||||
markdown = "markdown"
|
||||
jsonl = "jsonl"
|
||||
parquet = "parquet"
|
||||
sqlite = "sqlite"
|
||||
|
||||
|
||||
class Frequency(str, Enum):
|
||||
daily = "daily"
|
||||
weekly = "weekly"
|
||||
biweekly = "biweekly"
|
||||
monthly = "monthly"
|
||||
on_demand = "on_demand"
|
||||
|
||||
|
||||
class ChangeType(str, Enum):
|
||||
content_updated = "content_updated"
|
||||
url_moved = "url_moved"
|
||||
deleted = "deleted"
|
||||
new_version = "new_version"
|
||||
|
||||
|
||||
class FinalDecision(str, Enum):
|
||||
approved = "approved"
|
||||
rejected = "rejected"
|
||||
needs_manual_review = "needs_manual_review"
|
||||
|
||||
|
||||
# --- Core Table Models ---
|
||||
|
||||
class Source(BaseModel):
|
||||
source_id: Optional[int] = None
|
||||
source_name: str
|
||||
source_type: SourceType
|
||||
base_url: Optional[str] = None
|
||||
credibility_tier: CredibilityTier = CredibilityTier.tier3_community
|
||||
vendor: Optional[str] = None
|
||||
is_active: bool = True
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class Document(BaseModel):
|
||||
doc_id: Optional[int] = None
|
||||
source_id: int
|
||||
title: str
|
||||
url: Optional[str] = None
|
||||
url_hash: Optional[str] = None # Generated column in MySQL
|
||||
doc_type: DocType
|
||||
language: Language = Language.en
|
||||
original_publish_date: Optional[date] = None
|
||||
last_modified_date: Optional[date] = None
|
||||
crawl_date: Optional[datetime] = None
|
||||
crawl_method: CrawlMethod = CrawlMethod.firecrawl
|
||||
crawl_status: CrawlStatus = CrawlStatus.pending
|
||||
raw_content_path: Optional[str] = None
|
||||
raw_content_size: Optional[int] = None
|
||||
version: int = 1
|
||||
previous_version_id: Optional[int] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class DistilledContent(BaseModel):
|
||||
distill_id: Optional[int] = None
|
||||
doc_id: int
|
||||
summary: Optional[str] = None
|
||||
key_concepts: Optional[list[dict[str, Any]]] = None
|
||||
code_snippets: Optional[list[dict[str, Any]]] = None
|
||||
structured_content: Optional[str] = None
|
||||
token_count_original: Optional[int] = None
|
||||
token_count_distilled: Optional[int] = None
|
||||
distill_model: Optional[str] = None
|
||||
distill_date: Optional[datetime] = None
|
||||
review_status: ReviewStatus = ReviewStatus.pending
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def compression_ratio(self) -> Optional[float]:
|
||||
if self.token_count_original and self.token_count_distilled:
|
||||
return round(self.token_count_distilled / self.token_count_original * 100, 2)
|
||||
return None
|
||||
|
||||
|
||||
class ReviewLog(BaseModel):
|
||||
review_id: Optional[int] = None
|
||||
distill_id: int
|
||||
review_round: int = 1
|
||||
reviewer_type: ReviewerType
|
||||
quality_score: Optional[float] = None
|
||||
assessment: Optional[dict[str, float]] = None
|
||||
decision: Decision
|
||||
feedback: Optional[str] = None
|
||||
refactor_instructions: Optional[str] = None
|
||||
research_queries: Optional[list[str]] = None
|
||||
reviewed_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class Topic(BaseModel):
|
||||
topic_id: Optional[int] = None
|
||||
topic_name: str
|
||||
topic_slug: str
|
||||
parent_topic_id: Optional[int] = None
|
||||
description: Optional[str] = None
|
||||
|
||||
|
||||
class DocumentTopic(BaseModel):
|
||||
doc_id: int
|
||||
topic_id: int
|
||||
relevance_score: float = 1.0
|
||||
|
||||
|
||||
class ExportJob(BaseModel):
|
||||
export_id: Optional[int] = None
|
||||
export_name: str
|
||||
export_type: ExportType
|
||||
output_format: OutputFormat = OutputFormat.markdown
|
||||
topic_filter: Optional[list[int]] = None
|
||||
date_range_start: Optional[date] = None
|
||||
date_range_end: Optional[date] = None
|
||||
min_quality_score: float = 0.80
|
||||
output_path: Optional[str] = None
|
||||
total_documents: Optional[int] = None
|
||||
total_tokens: Optional[int] = None
|
||||
status: str = "pending"
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
created_at: Optional[datetime] = None
|
||||
|
||||
|
||||
class PipelineRun(BaseModel):
|
||||
run_id: Optional[int] = None
|
||||
run_type: RunType
|
||||
input_value: str
|
||||
status: PipelineStatus = PipelineStatus.running
|
||||
current_stage: PipelineStage = PipelineStage.discovery
|
||||
options: Optional[dict[str, Any]] = None
|
||||
stats: Optional[dict[str, int]] = Field(default_factory=lambda: {
|
||||
"sources_discovered": 0,
|
||||
"pages_crawled": 0,
|
||||
"documents_stored": 0,
|
||||
"documents_distilled": 0,
|
||||
"approved": 0,
|
||||
"refactored": 0,
|
||||
"deep_researched": 0,
|
||||
"rejected": 0,
|
||||
"needs_manual_review": 0,
|
||||
})
|
||||
export_path: Optional[str] = None
|
||||
export_document_count: Optional[int] = None
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
error_stage: Optional[str] = None
|
||||
|
||||
|
||||
class PipelineIterationTracker(BaseModel):
|
||||
tracker_id: Optional[int] = None
|
||||
run_id: int
|
||||
doc_id: int
|
||||
refactor_count: int = 0
|
||||
deep_research_count: int = 0
|
||||
final_decision: Optional[FinalDecision] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
# --- Non-DB Models (manifest/crawl/assessment) ---
|
||||
|
||||
class ManifestURL(BaseModel):
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
credibility_tier: Optional[str] = None
|
||||
credibility_score: Optional[float] = None
|
||||
source_type: Optional[str] = None
|
||||
vendor: Optional[str] = None
|
||||
|
||||
|
||||
class Manifest(BaseModel):
|
||||
discovery_date: Optional[str] = None
|
||||
topic: Optional[str] = None
|
||||
total_urls: int = 0
|
||||
urls: list[ManifestURL] = Field(default_factory=list)
|
||||
|
||||
|
||||
class CrawlResultEntry(BaseModel):
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
raw_path: str
|
||||
content_size: int = 0
|
||||
status: str = "completed"
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class CrawlResult(BaseModel):
|
||||
crawl_date: Optional[str] = None
|
||||
crawler_used: str = "firecrawl"
|
||||
total_crawled: int = 0
|
||||
total_failed: int = 0
|
||||
documents: list[CrawlResultEntry] = Field(default_factory=list)
|
||||
|
||||
|
||||
class QAAssessment(BaseModel):
|
||||
"""Legacy model for post-distillation Claude self-review (deprecated)."""
|
||||
accuracy: float = 0.0
|
||||
completeness: float = 0.0
|
||||
clarity: float = 0.0
|
||||
prompt_engineering_quality: float = 0.0
|
||||
usability: float = 0.0
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def weighted_score(self) -> float:
|
||||
return round(
|
||||
self.accuracy * 0.25
|
||||
+ self.completeness * 0.20
|
||||
+ self.clarity * 0.20
|
||||
+ self.prompt_engineering_quality * 0.25
|
||||
+ self.usability * 0.10,
|
||||
4,
|
||||
)
|
||||
|
||||
|
||||
class SourceQAAssessment(BaseModel):
|
||||
"""Pre-distillation source quality assessment via Gemini."""
|
||||
relevance: float = 0.0
|
||||
authority: float = 0.0
|
||||
completeness: float = 0.0
|
||||
freshness: float = 0.0
|
||||
distill_value: float = 0.0
|
||||
verdict: str = ""
|
||||
reason: str = ""
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def weighted_score(self) -> float:
|
||||
return round(
|
||||
self.relevance * 0.25
|
||||
+ self.authority * 0.25
|
||||
+ self.completeness * 0.20
|
||||
+ self.freshness * 0.15
|
||||
+ self.distill_value * 0.15,
|
||||
4,
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Common utilities for the reference curator pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
"""Normalize a URL for deduplication.
|
||||
|
||||
- Lowercase scheme and host
|
||||
- Remove trailing slashes
|
||||
- Sort query parameters
|
||||
- Remove common tracking params (utm_*, ref, fbclid)
|
||||
- Remove fragment
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
scheme = parsed.scheme.lower()
|
||||
netloc = parsed.netloc.lower()
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
|
||||
# Sort query params, removing tracking params
|
||||
tracking_params = {"utm_source", "utm_medium", "utm_campaign", "utm_content",
|
||||
"utm_term", "ref", "fbclid", "gclid", "mc_cid", "mc_eid"}
|
||||
params = parse_qs(parsed.query, keep_blank_values=True)
|
||||
filtered = {k: v for k, v in sorted(params.items()) if k not in tracking_params}
|
||||
query = urlencode(filtered, doseq=True)
|
||||
|
||||
return urlunparse((scheme, netloc, path, "", query, ""))
|
||||
|
||||
|
||||
def url_hash(url: str) -> str:
|
||||
"""SHA-256 hash of normalized URL. Matches the url_hash column in schema.sql."""
|
||||
return hashlib.sha256(normalize_url(url).encode()).hexdigest()
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
"""Convert text to a URL/folder-friendly slug.
|
||||
|
||||
>>> slugify("Prompt Engineering Best Practices")
|
||||
'prompt-engineering-best-practices'
|
||||
"""
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = text.encode("ascii", "ignore").decode()
|
||||
text = text.lower()
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
text = text.strip("-")
|
||||
return text or "untitled"
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""Approximate token count using chars/4 heuristic.
|
||||
|
||||
Good enough for compression ratio calculations without requiring tiktoken.
|
||||
"""
|
||||
return max(1, len(text) // 4)
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO", run_id: int | None = None) -> logging.Logger:
|
||||
"""Configure and return a logger for the reference curator pipeline."""
|
||||
logger = logging.getLogger("refcurator")
|
||||
if not logger.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
fmt = "[refcurator]"
|
||||
if run_id:
|
||||
fmt += f" [run:{run_id}]"
|
||||
fmt += " %(levelname)s: %(message)s"
|
||||
handler.setFormatter(logging.Formatter(fmt))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
return logger
|
||||
Reference in New Issue
Block a user