Andrew Yim
f215c11c32
Build the refcurator shared Python package and 7 CLI scripts that were previously specification-only. Add Gemini CLI as an independent pre-distillation quality evaluator, replacing the circular Claude-self-review pattern. Key changes: - shared/lib/src/refcurator/: 7-module package (config, db, models, utils, manifest, gemini) with PyMySQL + JSON file dual backend - 7 Click CLI scripts: discover, crawl_mgr, repo, distiller, reviewer, exporter, pipeline — each with subcommands for data management - Gemini quality gate: evaluates raw content BEFORE distillation using 5 criteria (relevance, authority, completeness, freshness, distill_value) - Pipeline reordered: discovery → crawl → store → evaluate → distill → export - Bug fixes from Codex adversarial review: - FileBackend now hard-fails on JOIN/aggregate/GROUP BY queries - Exporter uses MAX(review_id) to prevent shipping stale approvals - Distiller updates existing rows on refactor instead of forking - Updated all 7 CLAUDE.md directives with real script references - install.sh updated with refcurator package install step 51/51 E2E tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
91 lines
2.8 KiB
Python
91 lines
2.8 KiB
Python
"""Manifest I/O for reference discovery and crawl results."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from refcurator.models import CrawlResult, CrawlResultEntry, Manifest, ManifestURL
|
|
from refcurator.utils import normalize_url
|
|
|
|
|
|
def read_manifest(path: Path) -> Manifest:
|
|
"""Read a manifest JSON file."""
|
|
data = json.loads(path.read_text())
|
|
return Manifest(**data)
|
|
|
|
|
|
def write_manifest(manifest: Manifest, path: Path) -> None:
|
|
"""Write a manifest to a JSON file."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(manifest.model_dump_json(indent=2))
|
|
|
|
|
|
def merge_manifests(manifests: list[Manifest]) -> Manifest:
|
|
"""Merge multiple manifests, deduplicating URLs."""
|
|
seen: dict[str, ManifestURL] = {}
|
|
topic_parts = []
|
|
|
|
for m in manifests:
|
|
if m.topic:
|
|
topic_parts.append(m.topic)
|
|
for url_entry in m.urls:
|
|
normalized = normalize_url(url_entry.url)
|
|
existing = seen.get(normalized)
|
|
if existing is None or (
|
|
url_entry.credibility_score
|
|
and (existing.credibility_score or 0) < url_entry.credibility_score
|
|
):
|
|
seen[normalized] = url_entry
|
|
|
|
urls = list(seen.values())
|
|
return Manifest(
|
|
discovery_date=datetime.now().isoformat(),
|
|
topic=" + ".join(topic_parts) if topic_parts else None,
|
|
total_urls=len(urls),
|
|
urls=urls,
|
|
)
|
|
|
|
|
|
def dedup_manifest_urls(manifest: Manifest, existing_urls: set[str]) -> Manifest:
|
|
"""Remove URLs already in the existing set (normalized comparison)."""
|
|
existing_normalized = {normalize_url(u) for u in existing_urls}
|
|
filtered = [u for u in manifest.urls if normalize_url(u.url) not in existing_normalized]
|
|
return Manifest(
|
|
discovery_date=manifest.discovery_date,
|
|
topic=manifest.topic,
|
|
total_urls=len(filtered),
|
|
urls=filtered,
|
|
)
|
|
|
|
|
|
def read_crawl_result(path: Path) -> CrawlResult:
|
|
"""Read a crawl result JSON file."""
|
|
data = json.loads(path.read_text())
|
|
return CrawlResult(**data)
|
|
|
|
|
|
def write_crawl_result(result: CrawlResult, path: Path) -> None:
|
|
"""Write a crawl result to a JSON file."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(result.model_dump_json(indent=2))
|
|
|
|
|
|
def create_crawl_result(
|
|
entries: list[dict],
|
|
crawler: str = "firecrawl",
|
|
) -> CrawlResult:
|
|
"""Create a CrawlResult from a list of crawl entry dicts."""
|
|
docs = [CrawlResultEntry(**e) for e in entries]
|
|
completed = [d for d in docs if d.status == "completed"]
|
|
failed = [d for d in docs if d.status != "completed"]
|
|
|
|
return CrawlResult(
|
|
crawl_date=datetime.now().isoformat(),
|
|
crawler_used=crawler,
|
|
total_crawled=len(completed),
|
|
total_failed=len(failed),
|
|
documents=docs,
|
|
)
|