feat(reference-curator): Add pipeline orchestrator and refactor skill format
Pipeline Orchestrator: - Add 07-pipeline-orchestrator skill with code/CLAUDE.md and desktop/SKILL.md - Add /reference-curator-pipeline slash command for full workflow automation - Add pipeline_runs and pipeline_iteration_tracker tables to schema.sql - Add v_pipeline_status and v_pipeline_iterations views - Add pipeline_config.yaml configuration template - Update AGENTS.md with Reference Curator Skills section - Update claude-project files with pipeline documentation Skill Format Refactoring: - Extract YAML frontmatter from SKILL.md files to separate skill.yaml - Add tools/ directories with MCP tool documentation - Update SKILL-FORMAT-REQUIREMENTS.md with new structure - Add migrate-skill-structure.py script for format conversion Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
330
scripts/migrate-skill-structure.py
Executable file
330
scripts/migrate-skill-structure.py
Executable file
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate SKILL.md files to new structure:
|
||||
1. Extract YAML frontmatter → skill.yaml
|
||||
2. Strip frontmatter from SKILL.md
|
||||
3. Create tools/ directory
|
||||
|
||||
Usage:
|
||||
python migrate-skill-structure.py --dry-run # Preview changes
|
||||
python migrate-skill-structure.py # Execute migration
|
||||
python migrate-skill-structure.py --verbose # Verbose output
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def find_skill_files(base_path: Path) -> list[Path]:
|
||||
"""Find all desktop/SKILL.md files."""
|
||||
skill_files = []
|
||||
|
||||
for skill_md in base_path.rglob("desktop/SKILL.md"):
|
||||
# Skip archived skills
|
||||
if "99_archive" in str(skill_md):
|
||||
continue
|
||||
skill_files.append(skill_md)
|
||||
|
||||
return sorted(skill_files)
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> tuple[Optional[dict], str]:
|
||||
"""
|
||||
Parse YAML frontmatter from content.
|
||||
Returns (frontmatter_dict, remaining_content).
|
||||
"""
|
||||
# Match frontmatter between --- delimiters
|
||||
pattern = r'^---\s*\n(.*?)\n---\s*\n(.*)$'
|
||||
match = re.match(pattern, content, re.DOTALL)
|
||||
|
||||
if not match:
|
||||
return None, content
|
||||
|
||||
frontmatter_text = match.group(1)
|
||||
remaining_content = match.group(2)
|
||||
|
||||
try:
|
||||
frontmatter = yaml.safe_load(frontmatter_text)
|
||||
return frontmatter, remaining_content
|
||||
except yaml.YAMLError as e:
|
||||
print(f" YAML parse error: {e}")
|
||||
return None, content
|
||||
|
||||
|
||||
def generate_skill_yaml(frontmatter: dict) -> str:
|
||||
"""Generate skill.yaml content from frontmatter."""
|
||||
# Use a custom dumper to preserve formatting
|
||||
result = "# Skill metadata (extracted from SKILL.md frontmatter)\n\n"
|
||||
|
||||
# Required fields first
|
||||
if "name" in frontmatter:
|
||||
result += f"name: {frontmatter['name']}\n"
|
||||
|
||||
if "description" in frontmatter:
|
||||
desc = frontmatter["description"]
|
||||
# Check if description is multi-line
|
||||
if "\n" in str(desc) or len(str(desc)) > 80:
|
||||
result += "description: |\n"
|
||||
for line in str(desc).strip().split("\n"):
|
||||
result += f" {line}\n"
|
||||
else:
|
||||
# Single line - use quoted string if contains special chars
|
||||
if any(c in str(desc) for c in [":", "{", "}", "[", "]", ",", "#", "&", "*", "!", "|", ">", "'", '"', "%", "@", "`"]):
|
||||
result += f'description: "{desc}"\n'
|
||||
else:
|
||||
result += f"description: {desc}\n"
|
||||
|
||||
result += "\n# Optional fields\n"
|
||||
|
||||
# Optional fields
|
||||
if "allowed-tools" in frontmatter:
|
||||
tools = frontmatter["allowed-tools"]
|
||||
if isinstance(tools, str):
|
||||
# Parse comma-separated tools
|
||||
tool_list = [t.strip() for t in tools.split(",")]
|
||||
else:
|
||||
tool_list = tools
|
||||
|
||||
result += "allowed-tools:\n"
|
||||
for tool in tool_list:
|
||||
result += f" - {tool}\n"
|
||||
|
||||
if "license" in frontmatter:
|
||||
result += f"\nlicense: {frontmatter['license']}\n"
|
||||
|
||||
# Add triggers section (placeholder - can be extracted from description)
|
||||
result += "\n# triggers: [] # TODO: Extract from description\n"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_tool_stubs(frontmatter: dict) -> dict[str, str]:
|
||||
"""
|
||||
Determine which tool stub files to create based on allowed-tools.
|
||||
Returns dict of {filename: content}.
|
||||
"""
|
||||
stubs = {}
|
||||
|
||||
if "allowed-tools" not in frontmatter:
|
||||
return stubs
|
||||
|
||||
tools_str = frontmatter["allowed-tools"]
|
||||
if isinstance(tools_str, str):
|
||||
tools = [t.strip() for t in tools_str.split(",")]
|
||||
else:
|
||||
tools = tools_str
|
||||
|
||||
# Map tools to stub files
|
||||
tool_mappings = {
|
||||
"mcp__firecrawl__*": "firecrawl",
|
||||
"mcp__perplexity__*": "perplexity",
|
||||
"mcp__notion__*": "notion",
|
||||
"mcp__chrome-devtools__*": "chrome-devtools",
|
||||
}
|
||||
|
||||
claude_core_tools = {"Read", "Write", "Edit", "Bash", "Glob", "Grep"}
|
||||
webfetch_tools = {"WebFetch"}
|
||||
|
||||
has_claude_core = False
|
||||
has_webfetch = False
|
||||
mcp_tools = set()
|
||||
|
||||
for tool in tools:
|
||||
tool = tool.strip()
|
||||
|
||||
# Check MCP tool patterns
|
||||
for pattern, stub_name in tool_mappings.items():
|
||||
if pattern.replace("*", "") in tool or tool == pattern:
|
||||
mcp_tools.add(stub_name)
|
||||
break
|
||||
else:
|
||||
# Check Claude core tools
|
||||
if tool in claude_core_tools:
|
||||
has_claude_core = True
|
||||
elif tool in webfetch_tools:
|
||||
has_webfetch = True
|
||||
|
||||
# Generate stub content
|
||||
stub_template = """# {tool_name}
|
||||
|
||||
> TODO: Document tool usage for this skill
|
||||
|
||||
## Available Commands
|
||||
|
||||
- [ ] List commands
|
||||
|
||||
## Configuration
|
||||
|
||||
- [ ] Add configuration details
|
||||
|
||||
## Examples
|
||||
|
||||
- [ ] Add usage examples
|
||||
"""
|
||||
|
||||
for mcp_tool in mcp_tools:
|
||||
title = mcp_tool.replace("-", " ").title()
|
||||
stubs[f"{mcp_tool}.md"] = stub_template.format(tool_name=title)
|
||||
|
||||
if has_claude_core:
|
||||
stubs["claude-core.md"] = stub_template.format(tool_name="Claude Core Tools (Read, Write, Edit, Bash, Glob, Grep)")
|
||||
|
||||
if has_webfetch:
|
||||
stubs["webfetch.md"] = stub_template.format(tool_name="WebFetch")
|
||||
|
||||
return stubs
|
||||
|
||||
|
||||
def migrate_skill(skill_md_path: Path, dry_run: bool = True, verbose: bool = False) -> dict:
|
||||
"""
|
||||
Migrate a single SKILL.md file.
|
||||
Returns status dict.
|
||||
"""
|
||||
desktop_dir = skill_md_path.parent
|
||||
skill_yaml_path = desktop_dir / "skill.yaml"
|
||||
tools_dir = desktop_dir / "tools"
|
||||
|
||||
status = {
|
||||
"path": str(skill_md_path),
|
||||
"success": False,
|
||||
"skill_yaml_created": False,
|
||||
"frontmatter_stripped": False,
|
||||
"tools_dir_created": False,
|
||||
"tool_stubs": [],
|
||||
"error": None
|
||||
}
|
||||
|
||||
try:
|
||||
# Read original content
|
||||
content = skill_md_path.read_text(encoding="utf-8")
|
||||
|
||||
# Parse frontmatter
|
||||
frontmatter, remaining_content = parse_frontmatter(content)
|
||||
|
||||
if frontmatter is None:
|
||||
status["error"] = "No frontmatter found"
|
||||
return status
|
||||
|
||||
# Generate skill.yaml content
|
||||
skill_yaml_content = generate_skill_yaml(frontmatter)
|
||||
|
||||
# Get tool stubs
|
||||
tool_stubs = get_tool_stubs(frontmatter)
|
||||
status["tool_stubs"] = list(tool_stubs.keys())
|
||||
|
||||
if verbose:
|
||||
print(f"\n Frontmatter fields: {list(frontmatter.keys())}")
|
||||
print(f" Tool stubs to create: {status['tool_stubs']}")
|
||||
|
||||
if dry_run:
|
||||
print(f" [DRY-RUN] Would create: {skill_yaml_path}")
|
||||
print(f" [DRY-RUN] Would strip frontmatter from: {skill_md_path}")
|
||||
print(f" [DRY-RUN] Would create directory: {tools_dir}")
|
||||
for stub_name in tool_stubs:
|
||||
print(f" [DRY-RUN] Would create stub: {tools_dir / stub_name}")
|
||||
status["success"] = True
|
||||
else:
|
||||
# Create skill.yaml
|
||||
skill_yaml_path.write_text(skill_yaml_content, encoding="utf-8")
|
||||
status["skill_yaml_created"] = True
|
||||
|
||||
# Strip frontmatter from SKILL.md
|
||||
skill_md_path.write_text(remaining_content.lstrip(), encoding="utf-8")
|
||||
status["frontmatter_stripped"] = True
|
||||
|
||||
# Create tools/ directory
|
||||
tools_dir.mkdir(exist_ok=True)
|
||||
status["tools_dir_created"] = True
|
||||
|
||||
# Create tool stubs
|
||||
for stub_name, stub_content in tool_stubs.items():
|
||||
stub_path = tools_dir / stub_name
|
||||
stub_path.write_text(stub_content, encoding="utf-8")
|
||||
|
||||
status["success"] = True
|
||||
|
||||
except Exception as e:
|
||||
status["error"] = str(e)
|
||||
|
||||
return status
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Migrate SKILL.md files to new structure")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview changes without executing")
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
||||
parser.add_argument("--path", type=str, default=None, help="Base path to search (default: auto-detect)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Find base path
|
||||
if args.path:
|
||||
base_path = Path(args.path)
|
||||
else:
|
||||
# Auto-detect from script location
|
||||
script_dir = Path(__file__).parent
|
||||
base_path = script_dir.parent / "custom-skills"
|
||||
|
||||
if not base_path.exists():
|
||||
print(f"Error: Path not found: {base_path}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Searching for SKILL.md files in: {base_path}")
|
||||
print(f"Mode: {'DRY-RUN' if args.dry_run else 'EXECUTE'}")
|
||||
print("-" * 60)
|
||||
|
||||
# Find all skill files
|
||||
skill_files = find_skill_files(base_path)
|
||||
print(f"Found {len(skill_files)} desktop/SKILL.md files\n")
|
||||
|
||||
# Process each file
|
||||
results = {
|
||||
"total": len(skill_files),
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
"skill_yaml_created": 0,
|
||||
"tools_dirs_created": 0,
|
||||
"tool_stubs_created": 0
|
||||
}
|
||||
|
||||
for i, skill_path in enumerate(skill_files, 1):
|
||||
rel_path = skill_path.relative_to(base_path)
|
||||
print(f"[{i}/{len(skill_files)}] Processing: {rel_path}")
|
||||
|
||||
status = migrate_skill(skill_path, dry_run=args.dry_run, verbose=args.verbose)
|
||||
|
||||
if status["success"]:
|
||||
results["success"] += 1
|
||||
if status["skill_yaml_created"] or args.dry_run:
|
||||
results["skill_yaml_created"] += 1
|
||||
if status["tools_dir_created"] or args.dry_run:
|
||||
results["tools_dirs_created"] += 1
|
||||
results["tool_stubs_created"] += len(status["tool_stubs"])
|
||||
print(f" ✓ OK")
|
||||
else:
|
||||
results["failed"] += 1
|
||||
print(f" ✗ FAILED: {status['error']}")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("MIGRATION SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f"Total files processed: {results['total']}")
|
||||
print(f"Successful: {results['success']}")
|
||||
print(f"Failed: {results['failed']}")
|
||||
print(f"skill.yaml files {'to create' if args.dry_run else 'created'}: {results['skill_yaml_created']}")
|
||||
print(f"tools/ directories {'to create' if args.dry_run else 'created'}: {results['tools_dirs_created']}")
|
||||
print(f"Tool stub files {'to create' if args.dry_run else 'created'}: {results['tool_stubs_created']}")
|
||||
|
||||
if args.dry_run:
|
||||
print("\n[DRY-RUN] No files were modified. Run without --dry-run to execute.")
|
||||
|
||||
return 0 if results["failed"] == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user