ourdigital/our-claude-skills
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(reference-curator): Add portable skill suite for reference documentation curation

Browse Source 
6 modular skills for curating, processing, and exporting reference docs:
- reference-discovery: Search and validate authoritative sources
- web-crawler-orchestrator: Multi-backend crawling (Firecrawl/Node/aiohttp/Scrapy)
- content-repository: MySQL storage with version tracking
- content-distiller: Summarization and key concept extraction
- quality-reviewer: QA loop with approve/refactor/research routing
- markdown-exporter: Structured output for Claude Projects or fine-tuning

Cross-machine installation support:
- Environment-based config (~/.reference-curator.env)
- Commands tracked in repo, symlinked during install
- install.sh with --minimal, --check, --uninstall modes
- Firecrawl MCP as default (always available)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Yim
2026-01-29 00:20:27 +07:00
parent e80056ae8a
commit 6d7a6d7a88
26 changed files with 4486 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

747
custom-skills/90-reference-curator/install.sh Executable file
View File

@@ -0,0 +1,747 @@
#!/bin/bash
#
# Reference Curator - Portable Installation Script
#
# Installs the Reference Curator skill suite on any machine.
# Handles environment configuration, database setup, and skill registration.
#
# Usage:
# ./install.sh # Interactive installation
# ./install.sh --check # Verify installation status
# ./install.sh --minimal # Firecrawl-only mode (no local crawlers)
# ./install.sh --uninstall # Remove installation
#
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m' # No Color
# Paths
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="$HOME/.reference-curator.env"
CONFIG_DIR="$HOME/.config/reference-curator"
CLAUDE_COMMANDS_DIR="$HOME/.claude/commands"
CLAUDE_SKILLS_DIR="$HOME/.claude/skills"
# Default values
DEFAULT_LIBRARY_PATH="$HOME/reference-library"
DEFAULT_CRAWLER="firecrawl"
# Skills and commands to register
SKILLS=(
"reference-discovery:01-reference-discovery"
"web-crawler-orchestrator:02-web-crawler-orchestrator"
"content-repository:03-content-repository"
"content-distiller:04-content-distiller"
"quality-reviewer:05-quality-reviewer"
"markdown-exporter:06-markdown-exporter"
)
COMMANDS=(
"reference-discovery"
"web-crawler"
"content-repository"
"content-distiller"
"quality-reviewer"
"markdown-exporter"
)
# ============================================================================
# Helper Functions
# ============================================================================
print_header() {
echo ""
echo -e "${BLUE}╔══════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ ${BOLD}Reference Curator - Portable Installation${NC}${BLUE} ║${NC}"
echo -e "${BLUE}╚══════════════════════════════════════════════════════╝${NC}"
echo ""
}
print_step() {
echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BOLD}$1${NC}"
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
}
print_substep() {
echo -e "\n${BLUE}▶${NC} $1"
}
print_success() {
echo -e " ${GREEN}✓${NC} $1"
}
print_warning() {
echo -e " ${YELLOW}⚠${NC} $1"
}
print_error() {
echo -e " ${RED}✗${NC} $1"
}
print_info() {
echo -e " ${BLUE}ℹ${NC} $1"
}
prompt_with_default() {
local prompt="$1"
local default="$2"
local var_name="$3"
if [[ -n "$default" ]]; then
read -p " $prompt [$default]: " input
eval "$var_name=\"${input:-$default}\""
else
read -p " $prompt: " input
eval "$var_name=\"$input\""
fi
}
prompt_password() {
local prompt="$1"
local var_name="$2"
read -sp " $prompt: " input
echo ""
eval "$var_name=\"$input\""
}
prompt_yes_no() {
local prompt="$1"
local default="$2"
if [[ "$default" == "y" ]]; then
read -p " $prompt [Y/n]: " input
[[ ! "$input" =~ ^[Nn] ]]
else
read -p " $prompt [y/N]: " input
[[ "$input" =~ ^[Yy] ]]
fi
}
# ============================================================================
# Check Prerequisites
# ============================================================================
check_prerequisites() {
print_step "Step 1: Checking Prerequisites"
local all_ok=true
# Check OS
print_substep "Operating System"
case "$(uname -s)" in
Darwin)
print_success "macOS detected"
;;
Linux)
print_success "Linux detected"
;;
*)
print_error "Unsupported OS: $(uname -s)"
all_ok=false
;;
esac
# Check MySQL
print_substep "MySQL Client"
if command -v mysql &> /dev/null; then
local mysql_version=$(mysql --version 2>/dev/null | head -1)
print_success "MySQL client installed: $mysql_version"
else
print_warning "MySQL client not found - database features will be limited"
print_info "Install with: brew install mysql (macOS) or apt install mysql-client (Linux)"
fi
# Check Claude Code directories
print_substep "Claude Code Directories"
if [[ -d "$HOME/.claude" ]]; then
print_success "~/.claude directory exists"
else
print_warning "~/.claude directory not found - will be created"
fi
# Check for existing installation
print_substep "Existing Installation"
if [[ -f "$ENV_FILE" ]]; then
print_warning "Existing installation found at $ENV_FILE"
if prompt_yes_no "Overwrite existing configuration?" "n"; then
rm -f "$ENV_FILE"
print_info "Existing configuration removed"
else
print_info "Will merge with existing configuration"
fi
else
print_success "No existing installation found"
fi
echo ""
if $all_ok; then
return 0
else
return 1
fi
}
# ============================================================================
# Configure Environment
# ============================================================================
configure_environment() {
print_step "Step 2: Environment Configuration"
echo -e "\n ${BOLD}Environment variables will be saved to:${NC} $ENV_FILE"
echo ""
# Library path
print_substep "Reference Library Path"
prompt_with_default "Storage directory for reference library" "$DEFAULT_LIBRARY_PATH" "LIBRARY_PATH"
# MySQL configuration
print_substep "MySQL Configuration"
if prompt_yes_no "Configure MySQL database?" "y"; then
SETUP_MYSQL=true
prompt_with_default "MySQL host" "localhost" "MYSQL_HOST"
prompt_with_default "MySQL port" "3306" "MYSQL_PORT"
prompt_with_default "MySQL username" "" "MYSQL_USER"
prompt_password "MySQL password" "MYSQL_PASSWORD"
if [[ -z "$MYSQL_PASSWORD" ]]; then
print_warning "No password provided - MySQL connection may fail"
fi
else
SETUP_MYSQL=false
print_info "Skipping MySQL setup - database features will be unavailable"
fi
# Crawler configuration
print_substep "Crawler Configuration"
echo ""
echo " Available crawler backends:"
echo " 1) Firecrawl MCP only (recommended for most users)"
echo " 2) Local crawlers + Firecrawl (requires our-seo-agent project)"
echo ""
read -p " Select option [1]: " crawler_option
crawler_option=${crawler_option:-1}
if [[ "$crawler_option" == "2" ]]; then
prompt_with_default "Path to crawler project (our-seo-agent)" "$HOME/Project/our-seo-agent" "CRAWLER_PATH"
if [[ -d "$CRAWLER_PATH" ]]; then
print_success "Crawler project found at $CRAWLER_PATH"
ENABLE_LOCAL_CRAWLERS=true
DEFAULT_CRAWLER="nodejs"
else
print_warning "Crawler project not found - using Firecrawl only"
ENABLE_LOCAL_CRAWLERS=false
DEFAULT_CRAWLER="firecrawl"
fi
else
ENABLE_LOCAL_CRAWLERS=false
DEFAULT_CRAWLER="firecrawl"
print_info "Using Firecrawl MCP as default crawler"
fi
# Save environment file
print_substep "Saving Configuration"
cat > "$ENV_FILE" << EOF
# Reference Curator Environment Configuration
# Generated: $(date -Iseconds)
#
# Source this file in your shell profile:
# echo 'source ~/.reference-curator.env' >> ~/.zshrc
#
# Storage paths
export REFERENCE_LIBRARY_PATH="$LIBRARY_PATH"
# MySQL configuration
export MYSQL_HOST="${MYSQL_HOST:-localhost}"
export MYSQL_PORT="${MYSQL_PORT:-3306}"
export MYSQL_USER="$MYSQL_USER"
export MYSQL_PASSWORD='$MYSQL_PASSWORD'
# Crawler configuration
export DEFAULT_CRAWLER="$DEFAULT_CRAWLER"
export CRAWLER_PROJECT_PATH="${CRAWLER_PATH:-}"
export NODEJS_CRAWLER_ENABLED=${ENABLE_LOCAL_CRAWLERS:-false}
export AIOHTTP_CRAWLER_ENABLED=${ENABLE_LOCAL_CRAWLERS:-false}
export SCRAPY_CRAWLER_ENABLED=${ENABLE_LOCAL_CRAWLERS:-false}
EOF
chmod 600 "$ENV_FILE"
print_success "Environment saved to $ENV_FILE"
# Source the environment
source "$ENV_FILE"
}
# ============================================================================
# Install Configuration Files
# ============================================================================
install_configs() {
print_step "Step 3: Installing Configuration Files"
print_substep "Creating directories"
mkdir -p "$CONFIG_DIR"
print_success "$CONFIG_DIR"
print_substep "Copying configuration files"
for config in db_config.yaml crawl_config.yaml export_config.yaml; do
if [[ -f "$SCRIPT_DIR/shared/config/$config" ]]; then
cp "$SCRIPT_DIR/shared/config/$config" "$CONFIG_DIR/"
print_success "$config"
else
print_error "$config - source not found"
fi
done
}
# ============================================================================
# Create Storage Directories
# ============================================================================
create_directories() {
print_step "Step 4: Creating Storage Directories"
local lib_path="${REFERENCE_LIBRARY_PATH:-$DEFAULT_LIBRARY_PATH}"
for dir in raw processed exports backups; do
mkdir -p "$lib_path/$dir"
print_success "$lib_path/$dir"
done
}
# ============================================================================
# Setup MySQL Database
# ============================================================================
setup_database() {
print_step "Step 5: Setting Up MySQL Database"
if [[ "$SETUP_MYSQL" != "true" ]]; then
print_info "MySQL setup skipped"
return 0
fi
# Test connection
print_substep "Testing MySQL connection"
if mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "SELECT 1" &>/dev/null; then
print_success "MySQL connection successful"
else
print_error "MySQL connection failed"
print_info "Check your credentials and try again"
return 1
fi
# Check if database exists
print_substep "Checking database"
if mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "USE reference_library" &>/dev/null; then
print_info "Database 'reference_library' already exists"
local table_count=$(mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -N -e \
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='reference_library'" 2>/dev/null)
if [[ "$table_count" -ge 9 ]]; then
print_success "Schema already applied ($table_count tables)"
return 0
fi
else
# Try to create database
print_substep "Creating database"
if mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e \
"CREATE DATABASE IF NOT EXISTS reference_library CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" &>/dev/null; then
print_success "Database created"
else
print_warning "Could not create database - may need admin privileges"
echo ""
if prompt_yes_no "Do you have MySQL root access to create the database?" "n"; then
prompt_password "MySQL root password" "ROOT_PASSWORD"
mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u root -p"$ROOT_PASSWORD" << EOF
CREATE DATABASE IF NOT EXISTS reference_library CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
GRANT ALL PRIVILEGES ON reference_library.* TO '$MYSQL_USER'@'%';
FLUSH PRIVILEGES;
EOF
print_success "Database created and privileges granted"
else
print_error "Cannot proceed without database"
return 1
fi
fi
fi
# Apply schema
print_substep "Applying schema"
if [[ -f "$SCRIPT_DIR/shared/schema.sql" ]]; then
if mysql -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" reference_library < "$SCRIPT_DIR/shared/schema.sql" 2>/dev/null; then
print_success "Schema applied successfully"
else
print_error "Failed to apply schema"
return 1
fi
else
print_error "Schema file not found: $SCRIPT_DIR/shared/schema.sql"
return 1
fi
}
# ============================================================================
# Register Claude Code Commands
# ============================================================================
register_commands() {
print_step "Step 6: Registering Claude Code Commands"
mkdir -p "$CLAUDE_COMMANDS_DIR"
for cmd in "${COMMANDS[@]}"; do
local source="$SCRIPT_DIR/commands/${cmd}.md"
local target="$CLAUDE_COMMANDS_DIR/${cmd}.md"
if [[ -f "$source" ]]; then
ln -sf "$source" "$target"
print_success "/$cmd"
else
print_warning "/$cmd - source not found: $source"
fi
done
}
# ============================================================================
# Register Claude Desktop Skills
# ============================================================================
register_skills() {
print_step "Step 7: Registering Claude Desktop Skills"
mkdir -p "$CLAUDE_SKILLS_DIR"
for skill_entry in "${SKILLS[@]}"; do
IFS=':' read -r skill_name skill_folder <<< "$skill_entry"
local target="$SCRIPT_DIR/$skill_folder/desktop"
if [[ -d "$target" ]]; then
ln -sf "$target" "$CLAUDE_SKILLS_DIR/$skill_name"
print_success "$skill_name"
else
print_warning "$skill_name - target not found: $target"
fi
done
}
# ============================================================================
# Post-Installation
# ============================================================================
post_install() {
print_step "Installation Complete!"
echo ""
echo -e "${GREEN}╔══════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Installation Successful! ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "${BOLD}Next Steps:${NC}"
echo ""
echo " 1. Add environment to your shell profile:"
echo -e " ${CYAN}echo 'source ~/.reference-curator.env' >> ~/.zshrc${NC}"
echo ""
echo " 2. Reload your shell or run:"
echo -e " ${CYAN}source ~/.reference-curator.env${NC}"
echo ""
echo " 3. Verify installation:"
echo -e " ${CYAN}./install.sh --check${NC}"
echo ""
echo " 4. Start using the skills:"
echo -e " ${CYAN}/reference-discovery Claude system prompts${NC}"
echo ""
if [[ "$DEFAULT_CRAWLER" == "firecrawl" ]]; then
echo -e "${YELLOW}Note:${NC} Using Firecrawl MCP as default crawler."
echo " Make sure firecrawl MCP server is configured in Claude Code settings."
echo ""
fi
}
# ============================================================================
# Check Installation Status
# ============================================================================
check_status() {
print_header
echo -e "${BOLD}Installation Status Check${NC}"
echo ""
local all_ok=true
# Check environment file
print_substep "Environment Configuration"
if [[ -f "$ENV_FILE" ]]; then
print_success "$ENV_FILE"
source "$ENV_FILE"
else
print_error "$ENV_FILE - not found"
all_ok=false
fi
# Check config files
print_substep "Configuration Files ($CONFIG_DIR)"
for config in db_config.yaml crawl_config.yaml export_config.yaml; do
if [[ -f "$CONFIG_DIR/$config" ]]; then
print_success "$config"
else
print_error "$config - missing"
all_ok=false
fi
done
# Check storage directories
print_substep "Storage Directories"
local lib_path="${REFERENCE_LIBRARY_PATH:-$DEFAULT_LIBRARY_PATH}"
for dir in raw processed exports; do
if [[ -d "$lib_path/$dir" ]]; then
print_success "$lib_path/$dir"
else
print_error "$lib_path/$dir - missing"
all_ok=false
fi
done
# Check MySQL
print_substep "MySQL Database"
if [[ -n "$MYSQL_USER" ]] && command -v mysql &>/dev/null; then
if mysql -h "${MYSQL_HOST:-localhost}" -P "${MYSQL_PORT:-3306}" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "USE reference_library" &>/dev/null; then
local table_count=$(mysql -h "${MYSQL_HOST:-localhost}" -P "${MYSQL_PORT:-3306}" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -N -e \
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='reference_library'" 2>/dev/null)
print_success "Database accessible ($table_count tables)"
else
print_warning "Database 'reference_library' not accessible"
all_ok=false
fi
else
print_info "MySQL not configured or client not installed"
fi
# Check Claude Code commands
print_substep "Claude Code Commands ($CLAUDE_COMMANDS_DIR)"
for cmd in "${COMMANDS[@]}"; do
if [[ -L "$CLAUDE_COMMANDS_DIR/${cmd}.md" ]] || [[ -f "$CLAUDE_COMMANDS_DIR/${cmd}.md" ]]; then
print_success "/$cmd"
else
print_error "/$cmd - not registered"
all_ok=false
fi
done
# Check Claude Desktop skills
print_substep "Claude Desktop Skills ($CLAUDE_SKILLS_DIR)"
for skill_entry in "${SKILLS[@]}"; do
IFS=':' read -r skill_name skill_folder <<< "$skill_entry"
if [[ -L "$CLAUDE_SKILLS_DIR/$skill_name" ]]; then
print_success "$skill_name"
else
print_error "$skill_name - not registered"
all_ok=false
fi
done
echo ""
if $all_ok; then
echo -e "${GREEN}All components installed correctly.${NC}"
return 0
else
echo -e "${YELLOW}Some components need attention. Run ./install.sh to fix.${NC}"
return 1
fi
}
# ============================================================================
# Uninstall
# ============================================================================
uninstall() {
print_header
echo -e "${YELLOW}${BOLD}Uninstall Reference Curator${NC}"
echo ""
echo "This will remove:"
echo " • Environment file: $ENV_FILE"
echo " • Config directory: $CONFIG_DIR"
echo " • Claude Code commands"
echo " • Claude Desktop skill symlinks"
echo ""
echo -e "${GREEN}This will NOT remove:${NC}"
echo " • Storage directories (your data is safe)"
echo " • MySQL database (your data is safe)"
echo ""
if ! prompt_yes_no "Proceed with uninstall?" "n"; then
echo "Uninstall cancelled."
exit 0
fi
echo ""
# Remove environment file
if [[ -f "$ENV_FILE" ]]; then
rm -f "$ENV_FILE"
print_success "Removed $ENV_FILE"
fi
# Remove config directory
if [[ -d "$CONFIG_DIR" ]]; then
rm -rf "$CONFIG_DIR"
print_success "Removed $CONFIG_DIR"
fi
# Remove Claude Code commands
for cmd in "${COMMANDS[@]}"; do
local target="$CLAUDE_COMMANDS_DIR/${cmd}.md"
if [[ -L "$target" ]] || [[ -f "$target" ]]; then
rm -f "$target"
print_success "Removed /$cmd command"
fi
done
# Remove Claude Desktop skills
for skill_entry in "${SKILLS[@]}"; do
IFS=':' read -r skill_name skill_folder <<< "$skill_entry"
local target="$CLAUDE_SKILLS_DIR/$skill_name"
if [[ -L "$target" ]]; then
rm -f "$target"
print_success "Removed $skill_name skill"
fi
done
echo ""
echo -e "${GREEN}Uninstall complete.${NC}"
echo ""
echo "To reinstall, run: ./install.sh"
}
# ============================================================================
# Main Installation
# ============================================================================
install() {
print_header
echo "This installer will set up the Reference Curator skill suite."
echo ""
echo "Components to install:"
echo " • Environment configuration"
echo " • Configuration files"
echo " • Storage directories"
echo " • MySQL database (optional)"
echo " • Claude Code commands (6)"
echo " • Claude Desktop skills (6)"
echo ""
if ! prompt_yes_no "Continue with installation?" "y"; then
echo "Installation cancelled."
exit 0
fi
check_prerequisites
configure_environment
install_configs
create_directories
if [[ "$SETUP_MYSQL" == "true" ]]; then
setup_database
fi
register_commands
register_skills
post_install
}
# ============================================================================
# Minimal Installation (Firecrawl only)
# ============================================================================
install_minimal() {
print_header
echo -e "${BOLD}Minimal Installation Mode${NC}"
echo ""
echo "This will install Reference Curator with Firecrawl MCP only."
echo "No local crawlers or MySQL database will be configured."
echo ""
if ! prompt_yes_no "Continue?" "y"; then
echo "Installation cancelled."
exit 0
fi
# Minimal environment
SETUP_MYSQL=false
ENABLE_LOCAL_CRAWLERS=false
DEFAULT_CRAWLER="firecrawl"
LIBRARY_PATH="$DEFAULT_LIBRARY_PATH"
cat > "$ENV_FILE" << EOF
# Reference Curator Environment Configuration (Minimal)
# Generated: $(date -Iseconds)
export REFERENCE_LIBRARY_PATH="$LIBRARY_PATH"
export DEFAULT_CRAWLER="firecrawl"
EOF
chmod 600 "$ENV_FILE"
source "$ENV_FILE"
install_configs
create_directories
register_commands
register_skills
post_install
}
# ============================================================================
# Entry Point
# ============================================================================
case "${1:-}" in
--check)
check_status
;;
--uninstall)
uninstall
;;
--minimal)
install_minimal
;;
--help|-h)
echo "Reference Curator - Portable Installation Script"
echo ""
echo "Usage:"
echo " ./install.sh Interactive installation"
echo " ./install.sh --check Check installation status"
echo " ./install.sh --minimal Firecrawl-only mode (no MySQL)"
echo " ./install.sh --uninstall Remove installation"
echo " ./install.sh --help Show this help"
;;
*)
install
;;
esac