| #!/usr/bin/env bash | | | # Firecrawl Skills & CLI Auto-Update Script | | | # Updates all Firecrawl components: CLI, skills, workflows, and LLM context files | | | # Run via scheduled task or manually | | | LOG="$HOME/startup_stack.log" | | | SKILLS_DIR="$HOME/AppData/Local/hermes/skills/firecrawl" | | | CONTEXT_DIR="$HOME/AppData/Local/hermes/llm-context/crawl4ai" | |
| echo "========================================" >> "$LOG" | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Firecrawl Skills Auto-Update Starting" >> "$LOG" | |
| echo "========================================" >> "$LOG" | |
| # Update Firecrawl CLI (npm global package) | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Updating Firecrawl CLI..." >> "$LOG" | |
| if command -v npm &> /dev/null; then | |
| npm update -g firecrawl-cli >> "$LOG" 2>&1 | |
| CLI_VERSION=$(firecrawl --version 2>/dev/null || echo "unknown") | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Firecrawl CLI updated to: $CLI_VERSION" >> "$LOG" | |
| else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - npm not found, skipping CLI update" >> "$LOG" | | | fi | | | # Update CLI Skills (firecrawl/cli repo) | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Updating CLI skills..." >> "$LOG" | |
| if [ -d "/c/Tools/firecrawl-cli-skills/.git" ]; then | |
| cd /c/Tools/firecrawl-cli-skills | |
| BEFORE=$(git rev-parse HEAD) | |
| git pull origin main >> "$LOG" 2>&1 | |
| AFTER=$(git rev-parse HEAD) | |
| if [ "$BEFORE" != "$AFTER" ]; then | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - CLI skills updated: $BEFORE -> $AFTER" >> "$LOG" | |
| # Copy updated skills to Hermes | |
| cp -r skills/* "$SKILLS_DIR/" >> "$LOG" 2>&1 | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - CLI skills copied to Hermes" >> "$LOG" | |
| else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - CLI skills already up-to-date" >> "$LOG" | | | fi | | | else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - CLI skills repo not found, skipping" >> "$LOG" | | | fi | | | # Update Build Skills (firecrawl/skills repo) | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Updating build skills..." >> "$LOG" | |
| if [ -d "/c/Tools/firecrawl-build-skills/.git" ]; then | |
| cd /c/Tools/firecrawl-build-skills | |
| BEFORE=$(git rev-parse HEAD) | |
| git pull origin main >> "$LOG" 2>&1 | |
| AFTER=$(git rev-parse HEAD) | |
| if [ "$BEFORE" != "$AFTER" ]; then | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Build skills updated: $BEFORE -> $AFTER" >> "$LOG" | |
| cp -r skills/* "$SKILLS_DIR/" >> "$LOG" 2>&1 | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Build skills copied to Hermes" >> "$LOG" | |
| else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Build skills already up-to-date" >> "$LOG" | | | fi | | | else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Build skills repo not found, skipping" >> "$LOG" | | | fi | | | # Update Workflow Skills (firecrawl/firecrawl-workflows repo) | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Updating workflow skills..." >> "$LOG" | |
| if [ -d "/c/Tools/firecrawl-workflows/.git" ]; then | |
| cd /c/Tools/firecrawl-workflows | |
| BEFORE=$(git rev-parse HEAD) | |
| git pull origin main >> "$LOG" 2>&1 | |
| AFTER=$(git rev-parse HEAD) | |
| if [ "$BEFORE" != "$AFTER" ]; then | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Workflow skills updated: $BEFORE -> $AFTER" >> "$LOG" | |
| cp -r skills/* "$SKILLS_DIR/" >> "$LOG" 2>&1 | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Workflow skills copied to Hermes" >> "$LOG" | |
| else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Workflow skills already up-to-date" >> "$LOG" | | | fi | | | else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Workflow skills repo not found, skipping" >> "$LOG" | | | fi | | | # Update LLM Context Files (Crawl4AI llm.txt approach) | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Updating LLM context files..." >> "$LOG" | | | mkdir -p "$CONTEXT_DIR" | | | cd "$CONTEXT_DIR" | | | COMPONENTS=( | | | "installation" | | | "simple_crawling" | | | "config_objects" | | | "extraction-llm" | | | "extraction-no-llm" | | | "multi_urls_crawling" | | | "deep_crawling" | | | "docker" | | | "cli" | | | "http_based_crawler_strategy" | | | "url_seeder" | | | "deep_crawl_advanced_filters_scorers" | | | ) | | | UPDATED_COUNT=0 | |
| for component in "${COMPONENTS[@]}"; do | |
| # Download memory (what) context | |
| MEMORY_URL="https://docs.crawl4ai.com/assets/llm.txt/txt/${component}.txt" | |
| MEMORY_FILE="${component}_memory.txt" | |
| if curl -s -o "$MEMORY_FILE.new" "$MEMORY_URL" 2>> "$LOG"; then | |
| if [ ! -f "$MEMORY_FILE" ] || ! cmp -s "$MEMORY_FILE" "$MEMORY_FILE.new"; then | |
| mv "$MEMORY_FILE.new" "$MEMORY_FILE" | |
| ((UPDATED_COUNT++)) | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Updated: $MEMORY_FILE" >> "$LOG" | |
| else | | | rm "$MEMORY_FILE.new" | | | fi | | | fi | | | # Download reasoning (how/why) context | | | REASONING_URL="https://docs.crawl4ai.com/assets/llm.txt/diagrams/${component}.txt" | |
| REASONING_FILE="${component}_reasoning.txt" | |
| if curl -s -o "$REASONING_FILE.new" "$REASONING_URL" 2>> "$LOG"; then | |
| if [ ! -f "$REASONING_FILE" ] || ! cmp -s "$REASONING_FILE" "$REASONING_FILE.new"; then | |
| mv "$REASONING_FILE.new" "$REASONING_FILE" | |
| ((UPDATED_COUNT++)) | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Updated: $REASONING_FILE" >> "$LOG" | |
| else | | | rm "$REASONING_FILE.new" | | | fi | | | fi | | | done | |
| if [ $UPDATED_COUNT -gt 0 ]; then | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - LLM context: $UPDATED_COUNT files updated" >> "$LOG" | |
| else | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - LLM context: all files up-to-date" >> "$LOG" | | | fi | | | # Update Crawl4AI Skill Package | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Updating Crawl4AI skill package..." >> "$LOG" | | | CRAWL4AI_SKILL="$HOME/AppData/Local/hermes/skills/web/crawl4ai" | | | cd /tmp | |
| if curl -L -s -o crawl4ai-skill.zip.new "https://docs.crawl4ai.com/assets/crawl4ai-skill.zip" 2>> "$LOG"; then | |
| if [ ! -f "crawl4ai-skill.zip" ] || ! cmp -s "crawl4ai-skill.zip" "crawl4ai-skill.zip.new"; then | |
| mv crawl4ai-skill.zip.new crawl4ai-skill.zip | | | unzip -q -o crawl4ai-skill.zip -d /tmp/crawl4ai-new | | | # Backup existing skill | | | if [ -d "$CRAWL4AI_SKILL" ]; then | | | cp -r "$CRAWL4AI_SKILL" "$CRAWL4AI_SKILL.backup" 2>> "$LOG" | | | fi | | | # Replace with updated skill | | | mkdir -p "$CRAWL4AI_SKILL" | |
| cp -r /tmp/crawl4ai-new/crawl4ai/* "$CRAWL4AI_SKILL/" 2>> "$LOG" | |
| rm -rf /tmp/crawl4ai-new | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Crawl4AI skill package updated" >> "$LOG" | |
| else | | | rm crawl4ai-skill.zip.new | | | echo "$(date '+%Y-%m-%d %H:%M:%S') - Crawl4AI skill package already up-to-date" >> "$LOG" | | | fi | | | fi | |
| echo "========================================" >> "$LOG" | |
| echo "$(date '+%Y-%m-%d %H:%M:%S') - Firecrawl Skills Auto-Update Complete" >> "$LOG" | |
| echo "========================================" >> "$LOG" | |
| exit 0 |