diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index ac0ee0e..0c5e9d1 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,243 +6,57 @@ }, "metadata": { "description": "Official Apify Agent Skills for web scraping, data extraction, and automation", - "version": "1.6.1" + "version": "2.0.0" }, "plugins": [ - { - "name": "apify-lead-generation", - "source": "./skills/apify-lead-generation", - "skills": "./", - "description": "Generate B2B/B2C leads by scraping Google Maps, websites, Instagram, TikTok, Facebook, LinkedIn, YouTube, and Google Search using Apify Actors", - "keywords": [ - "leads", - "sales", - "prospecting", - "b2b", - "b2c", - "scraping", - "apify", - "google-maps", - "instagram", - "tiktok", - "facebook", - "linkedin", - "youtube" - ], - "category": "data-extraction", - "version": "1.1.12" - }, - { - "name": "apify-brand-reputation-monitoring", - "source": "./skills/apify-brand-reputation-monitoring", - "skills": "./", - "description": "Track reviews, ratings, sentiment, and brand mentions across Google Maps, Booking.com, TripAdvisor, Facebook, Instagram, YouTube, and TikTok", - "keywords": [ - "reputation", - "reviews", - "sentiment", - "monitoring", - "brand", - "ratings", - "google-maps", - "booking", - "tripadvisor", - "facebook", - "instagram", - "youtube", - "tiktok" - ], - "category": "data-extraction", - "version": "1.0.2" - }, - { - "name": "apify-competitor-intelligence", - "source": "./skills/apify-competitor-intelligence", - "skills": "./", - "description": "Analyze competitor strategies, content, pricing, ads, and market positioning across Google Maps, Booking.com, Facebook, Instagram, YouTube, and TikTok", - "keywords": [ - "competitor", - "intelligence", - "analysis", - "benchmarking", - "strategy", - "ads", - "google-maps", - "booking", - "facebook", - "instagram", - "youtube", - "tiktok" - ], - "category": "data-extraction", - "version": "1.1.2" - }, - { - "name": "apify-market-research", - "source": "./skills/apify-market-research", - "skills": "./", - "description": "Analyze market conditions, geographic opportunities, pricing, consumer behavior, and product validation across Google Maps, Facebook, Instagram, Booking.com, and TripAdvisor", - "keywords": [ - "market", - "research", - "analysis", - "pricing", - "geographic", - "validation", - "trends", - "google-maps", - "facebook", - "instagram", - "booking", - "tripadvisor" - ], - "category": "data-extraction", - "version": "1.0.2" - }, - { - "name": "apify-influencer-discovery", - "source": "./skills/apify-influencer-discovery", - "skills": "./", - "description": "Find and evaluate influencers for brand partnerships, verify authenticity, and track collaboration performance across Instagram, Facebook, YouTube, and TikTok", - "keywords": [ - "influencer", - "discovery", - "partnership", - "creator", - "collaboration", - "authenticity", - "instagram", - "facebook", - "youtube", - "tiktok" - ], - "category": "data-extraction", - "version": "1.0.1" - }, - { - "name": "apify-trend-analysis", - "source": "./skills/apify-trend-analysis", - "skills": "./", - "description": "Discover and track emerging trends across Google Trends, Instagram, Facebook, YouTube, and TikTok to inform content strategy", - "keywords": [ - "trends", - "analysis", - "hashtags", - "viral", - "discovery", - "content", - "google-trends", - "instagram", - "facebook", - "youtube", - "tiktok" - ], - "category": "data-extraction", - "version": "1.0.1" - }, - { - "name": "apify-content-analytics", - "source": "./skills/apify-content-analytics", - "skills": "./", - "description": "Track engagement metrics, measure campaign ROI, and analyze content performance across Instagram, Facebook, YouTube, and TikTok", - "keywords": [ - "analytics", - "engagement", - "performance", - "metrics", - "ROI", - "content", - "instagram", - "facebook", - "youtube", - "tiktok" - ], - "category": "data-extraction", - "version": "1.0.1" - }, - { - "name": "apify-audience-analysis", - "source": "./skills/apify-audience-analysis", - "skills": "./", - "description": "Understand audience demographics, preferences, behavior patterns, and engagement quality across Facebook, Instagram, YouTube, and TikTok", - "keywords": [ - "audience", - "demographics", - "behavior", - "engagement", - "analysis", - "followers", - "facebook", - "instagram", - "youtube", - "tiktok" - ], - "category": "data-extraction", - "version": "1.0.1" - }, { "name": "apify-ultimate-scraper", "source": "./skills/apify-ultimate-scraper", "skills": "./", - "description": "Universal AI-powered web scraper for any platform. Scrape data from Instagram, Facebook, TikTok, YouTube, Google Maps, Google Search, Google Trends, Booking.com, and TripAdvisor for lead generation, brand monitoring, competitor analysis, influencer discovery, trend research, and more", + "description": "Universal AI-powered web scraper for 55+ platforms. Scrape data from Instagram, Facebook, TikTok, YouTube, Google Maps, Google Search, Google Trends, Booking.com, TripAdvisor, Amazon, Walmart, eBay, and more for lead generation, brand monitoring, competitor analysis, influencer discovery, trend research, content analytics, audience analysis, e-commerce pricing, and reviews", "keywords": [ - "scraper", - "universal", + "scraping", + "web-scraper", + "data-extraction", + "apify", "instagram", "facebook", "tiktok", "youtube", - "google-maps", - "leads", - "monitoring", - "competitor", - "trends", - "influencer" + "google-maps" ], "category": "data-extraction", - "version": "1.4.1" + "version": "2.0.0" }, { - "name": "apify-ecommerce", - "source": "./skills/apify-ecommerce", + "name": "apify-actor-development", + "source": "./skills/apify-actor-development", "skills": "./", - "description": "Scrape e-commerce data for pricing intelligence, customer sentiment, product research, quality analysis, and supply chain monitoring across Amazon, Walmart, eBay, IKEA, and 50+ marketplaces", + "description": "Develop, debug, and deploy Apify Actors - serverless cloud programs for web scraping, automation, and data processing", "keywords": [ - "ecommerce", - "pricing", - "reviews", - "sentiment", - "products", - "sellers", - "amazon", - "walmart", - "ebay", - "MAP", - "competitor", - "research", - "supply-chain" + "apify", + "actor", + "development", + "deploy", + "serverless" ], - "category": "data-extraction", - "version": "1.0.0" + "category": "development", + "version": "2.0.0" }, { - "name": "apify-actor-development", - "source": "./skills/apify-actor-development", + "name": "apify-actorization", + "source": "./skills/apify-actorization", "skills": "./", - "description": "Develop, debug, and deploy Apify Actors - serverless cloud programs for web scraping, automation, and data processing", + "description": "Convert existing projects into Apify Actors - serverless cloud programs. Actorize JavaScript/TypeScript (SDK with Actor.init/exit), Python (async context manager), or any language (CLI wrapper)", "keywords": [ "apify", "actor", - "web-scraping", - "automation", - "crawlee", - "playwright", - "cheerio", - "serverless", - "development" + "actorization", + "migration", + "convert" ], "category": "development", - "version": "1.0.0" + "version": "2.0.0" }, { "name": "apify-generate-output-schema", @@ -252,19 +66,17 @@ "keywords": [ "apify", "actor", - "output-schema", - "dataset-schema", - "key-value-store", - "schema-generation", - "development" + "schema", + "output", + "dataset" ], "category": "development", - "version": "1.0.0" + "version": "2.0.0" }, { "name": "apify-actor-commands", "description": "Commands for Apify Actor development workflow", - "version": "1.0.0", + "version": "2.0.0", "author": { "name": "Apify", "email": "support@apify.com" @@ -274,25 +86,6 @@ "commands": [ "./commands/create-actor.md" ] - }, - { - "name": "apify-actorization", - "source": "./skills/apify-actorization", - "skills": "./", - "description": "Convert existing projects into Apify Actors - serverless cloud programs. Actorize JavaScript/TypeScript (SDK with Actor.init/exit), Python (async context manager), or any language (CLI wrapper). Use when migrating code to Apify, wrapping CLI tools as Actors, or adding Actor SDK to existing projects.", - "keywords": [ - "actorization", - "convert", - "migrate", - "actor", - "apify", - "sdk", - "deployment", - "crawlee", - "serverless" - ], - "category": "development", - "version": "1.0.0" } ] } diff --git a/README.md b/README.md index 0966e2b..4dac599 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,20 @@ # Apify Agent Skills -Official Apify Agent Skills for web scraping, data extraction, and automation. Works with Claude Code, Cursor, Codex, Gemini CLI, and other AI coding assistants. - -## Available skills - - -| Name | Description | Documentation | -|------|-------------|---------------| -| `apify-actor-development` | Develop, debug, and deploy Apify Actors - serverless cloud programs for web scraping, automation, and data processing | [SKILL.md](skills/apify-actor-development/SKILL.md) | -| `apify-actorization` | Convert existing projects into Apify Actors - serverless cloud programs. Actorize JavaScript/TypeScript (SDK with Actor.init/exit), Python (async context manager), or any language (CLI wrapper). Use when migrating code to Apify, wrapping CLI tools as Actors, or adding Actor SDK to existing projects. | [SKILL.md](skills/apify-actorization/SKILL.md) | -| `apify-audience-analysis` | Understand audience demographics, preferences, behavior patterns, and engagement quality across Facebook, Instagram, YouTube, and TikTok | [SKILL.md](skills/apify-audience-analysis/SKILL.md) | -| `apify-brand-reputation-monitoring` | Track reviews, ratings, sentiment, and brand mentions across Google Maps, Booking.com, TripAdvisor, Facebook, Instagram, YouTube, and TikTok | [SKILL.md](skills/apify-brand-reputation-monitoring/SKILL.md) | -| `apify-competitor-intelligence` | Analyze competitor strategies, content, pricing, ads, and market positioning across Google Maps, Booking.com, Facebook, Instagram, YouTube, and TikTok | [SKILL.md](skills/apify-competitor-intelligence/SKILL.md) | -| `apify-content-analytics` | Track engagement metrics, measure campaign ROI, and analyze content performance across Instagram, Facebook, YouTube, and TikTok | [SKILL.md](skills/apify-content-analytics/SKILL.md) | -| `apify-ecommerce` | Scrape e-commerce data for pricing intelligence, customer sentiment, product research, quality analysis, and supply chain monitoring across Amazon, Walmart, eBay, IKEA, and 50+ marketplaces | [SKILL.md](skills/apify-ecommerce/SKILL.md) | -| `apify-generate-output-schema` | Generate output schemas (dataset_schema.json, output_schema.json, key_value_store_schema.json) for an Apify Actor by analyzing its source code | [SKILL.md](skills/apify-generate-output-schema/SKILL.md) | -| `apify-influencer-discovery` | Find and evaluate influencers for brand partnerships, verify authenticity, and track collaboration performance across Instagram, Facebook, YouTube, and TikTok | [SKILL.md](skills/apify-influencer-discovery/SKILL.md) | -| `apify-lead-generation` | Generate B2B/B2C leads by scraping Google Maps, websites, Instagram, TikTok, Facebook, LinkedIn, YouTube, and Google Search using Apify Actors | [SKILL.md](skills/apify-lead-generation/SKILL.md) | -| `apify-market-research` | Analyze market conditions, geographic opportunities, pricing, consumer behavior, and product validation across Google Maps, Facebook, Instagram, Booking.com, and TripAdvisor | [SKILL.md](skills/apify-market-research/SKILL.md) | -| `apify-trend-analysis` | Discover and track emerging trends across Google Trends, Instagram, Facebook, YouTube, and TikTok to inform content strategy | [SKILL.md](skills/apify-trend-analysis/SKILL.md) | -| `apify-ultimate-scraper` | Universal AI-powered web scraper for any platform. Scrape data from Instagram, Facebook, TikTok, YouTube, Google Maps, Google Search, Google Trends, Booking.com, and TripAdvisor for lead generation, brand monitoring, competitor analysis, influencer discovery, trend research, and more | [SKILL.md](skills/apify-ultimate-scraper/SKILL.md) | - +A collection of AI agent skills for web scraping, data extraction, and Actor development on the Apify platform. + +> Looking for more specialized skills? Check out [apify/awesome-skills](https://github.com/apify/awesome-skills) — a community collection of domain-specific skills for lead generation, brand monitoring, competitor intelligence, and more. + +## Skills + +### Scraping + +- **[Ultimate scraper](skills/apify-ultimate-scraper/)** (`apify-ultimate-scraper`) — AI-powered web scraper for 55+ platforms including Instagram, Facebook, TikTok, YouTube, Google Maps, Amazon, Walmart, eBay, Booking.com, TripAdvisor, and more. Can also search the [Apify Store](https://apify.com/store) to find the right Actor for any platform not listed here. + +### Development + +- **[Actor development](skills/apify-actor-development/)** (`apify-actor-development`) — create, debug, and deploy Apify Actors from scratch in JavaScript, TypeScript, or Python. +- **[Actorization](skills/apify-actorization/)** (`apify-actorization`) — convert existing projects into Apify Actors. Supports JS/TS (SDK), Python (async context manager), and any language (CLI wrapper). +- **[Generate output schema](skills/apify-generate-output-schema/)** (`apify-generate-output-schema`) — generate output schemas (`dataset_schema.json`, `output_schema.json`, `key_value_store_schema.json`) for an Apify Actor by analyzing its source code. ## Installation @@ -59,44 +53,19 @@ Any AI tool that supports Markdown context can use the skills by pointing to: ## Prerequisites -1. **Apify account** - [apify.com](https://apify.com) -2. **API token** - get from [Apify Console](https://console.apify.com/account/integrations), add `APIFY_TOKEN=your_token` to `.env` -3. **Node.js 20.6+** -4. **[mcpc CLI](https://github.com/apify/mcp-cli)** - `npm install -g @apify/mcpc` - -## Output formats - -- **Quick answer** - top 5 results displayed in chat (no file saved) -- **CSV** - full export with all fields -- **JSON** - full data export +1. **Apify account** — [apify.com](https://apify.com) +2. **API token** — get from [Apify Console](https://console.apify.com/account/integrations), add `APIFY_TOKEN=your_token` to `.env` +3. **Node.js 20.6+** (for the scraper skill) ## Pricing Apify Actors use pay-per-result pricing. Check individual Actor pricing on the [Apify platform](https://apify.com). -## Contributing - -1. Fork this repository. -2. Create your skill in `skills/your-skill-name/`. -3. Add `SKILL.md` with proper frontmatter: - ```yaml - --- - name: your-skill-name - description: What your skill does and when to use it - --- - ``` -4. Add entry to `.claude-plugin/marketplace.json`. -5. Run `uv run scripts/generate_agents.py` to update AGENTS.md. -6. Submit a pull request. - -## Development - -```bash -# Regenerate AGENTS.md and validate marketplace.json -uv run scripts/generate_agents.py -``` - ## Support - [Apify Documentation](https://docs.apify.com) - [Apify Discord](https://discord.gg/jyEM2PRvMU) + +## License + +[Apache-2.0](LICENSE) diff --git a/agents/AGENTS.md b/agents/AGENTS.md index 8c168df..4bd726d 100644 --- a/agents/AGENTS.md +++ b/agents/AGENTS.md @@ -5,16 +5,7 @@ You have additional SKILLs documented in directories containing a "SKILL.md" fil These skills are: - apify-actor-development -> "skills/apify-actor-development/SKILL.md" - apify-actorization -> "skills/apify-actorization/SKILL.md" - - apify-audience-analysis -> "skills/apify-audience-analysis/SKILL.md" - - apify-brand-reputation-monitoring -> "skills/apify-brand-reputation-monitoring/SKILL.md" - - apify-competitor-intelligence -> "skills/apify-competitor-intelligence/SKILL.md" - - apify-content-analytics -> "skills/apify-content-analytics/SKILL.md" - - apify-ecommerce -> "skills/apify-ecommerce/SKILL.md" - apify-generate-output-schema -> "skills/apify-generate-output-schema/SKILL.md" - - apify-influencer-discovery -> "skills/apify-influencer-discovery/SKILL.md" - - apify-lead-generation -> "skills/apify-lead-generation/SKILL.md" - - apify-market-research -> "skills/apify-market-research/SKILL.md" - - apify-trend-analysis -> "skills/apify-trend-analysis/SKILL.md" - apify-ultimate-scraper -> "skills/apify-ultimate-scraper/SKILL.md" IMPORTANT: You MUST read the SKILL.md file whenever the description of the skills matches the user intent, or may help accomplish their task. @@ -23,16 +14,7 @@ IMPORTANT: You MUST read the SKILL.md file whenever the description of the skill apify-actor-development: `Develop, debug, and deploy Apify Actors - serverless cloud programs for web scraping, automation, and data processing. Use when creating new Actors, modifying existing ones, or troubleshooting Actor code.` apify-actorization: `Convert existing projects into Apify Actors - serverless cloud programs. Actorize JavaScript/TypeScript (SDK with Actor.init/exit), Python (async context manager), or any language (CLI wrapper). Use when migrating code to Apify, wrapping CLI tools as Actors, or adding Actor SDK to existing projects.` -apify-audience-analysis: `Understand audience demographics, preferences, behavior patterns, and engagement quality across Facebook, Instagram, YouTube, and TikTok.` -apify-brand-reputation-monitoring: `Track reviews, ratings, sentiment, and brand mentions across Google Maps, Booking.com, TripAdvisor, Facebook, Instagram, YouTube, and TikTok. Use when user asks to monitor brand reputation, analyze reviews, track mentions, or gather customer feedback.` -apify-competitor-intelligence: `Analyze competitor strategies, content, pricing, ads, and market positioning across Google Maps, Booking.com, Facebook, Instagram, YouTube, and TikTok.` -apify-content-analytics: `Track engagement metrics, measure campaign ROI, and analyze content performance across Instagram, Facebook, YouTube, and TikTok.` -apify-ecommerce: `Scrape e-commerce data for pricing intelligence, customer reviews, and seller discovery across Amazon, Walmart, eBay, IKEA, and 50+ marketplaces. Use when user asks to monitor prices, track competitors, analyze reviews, research products, or find sellers.` apify-generate-output-schema: `Generate output schemas (dataset_schema.json, output_schema.json, key_value_store_schema.json) for an Apify Actor by analyzing its source code. Use when creating or updating Actor output schemas.` -apify-influencer-discovery: `Find and evaluate influencers for brand partnerships, verify authenticity, and track collaboration performance across Instagram, Facebook, YouTube, and TikTok.` -apify-lead-generation: `Generates B2B/B2C leads by scraping Google Maps, websites, Instagram, TikTok, Facebook, LinkedIn, YouTube, and Google Search. Use when user asks to find leads, prospects, businesses, build lead lists, enrich contacts, or scrape profiles for sales outreach.` -apify-market-research: `Analyze market conditions, geographic opportunities, pricing, consumer behavior, and product validation across Google Maps, Facebook, Instagram, Booking.com, and TripAdvisor.` -apify-trend-analysis: `Discover and track emerging trends across Google Trends, Instagram, Facebook, YouTube, and TikTok to inform content strategy.` apify-ultimate-scraper: `Universal AI-powered web scraper for any platform. Scrape data from Instagram, Facebook, TikTok, YouTube, Google Maps, Google Search, Google Trends, Booking.com, and TripAdvisor. Use for lead generation, brand monitoring, competitor analysis, influencer discovery, trend research, content analytics, audience analysis, or any data extraction task.` diff --git a/scripts/generate_agents.py b/scripts/generate_agents.py index d463bf9..434c2fe 100644 --- a/scripts/generate_agents.py +++ b/scripts/generate_agents.py @@ -5,25 +5,16 @@ # /// """Generate AGENTS.md from AGENTS_TEMPLATE.md and SKILL.md frontmatter. -Also validates that marketplace.json is in sync with discovered skills, -updates the skills table in README.md, and handles version bumping. - -Version bumping (conventional commits): - - BREAKING CHANGE: or feat!: → major bump (1.0.0 → 2.0.0) - - feat: → minor bump (1.0.0 → 1.1.0) - - fix:, docs:, chore:, etc. → patch bump (1.0.0 → 1.0.1) +Also validates that marketplace.json is in sync with discovered skills. Usage: - uv run scripts/generate_agents.py # Just regenerate - uv run scripts/generate_agents.py --bump "feat: X" # Bump based on commit msg + uv run scripts/generate_agents.py """ from __future__ import annotations -import argparse import json import re -import subprocess import sys from pathlib import Path @@ -32,13 +23,6 @@ TEMPLATE_PATH = ROOT / "scripts" / "AGENTS_TEMPLATE.md" OUTPUT_PATH = ROOT / "agents" / "AGENTS.md" MARKETPLACE_PATH = ROOT / ".claude-plugin" / "marketplace.json" -PLUGIN_PATH = ROOT / ".claude-plugin" / "plugin.json" -README_PATH = ROOT / "README.md" -SKILLS_DIR = ROOT / "skills" - -# Markers for the auto-generated skills table in README -README_TABLE_START = "" -README_TABLE_END = "" def load_template() -> str: @@ -97,286 +81,39 @@ def repl(match: re.Match[str]) -> str: return content -def load_marketplace() -> dict: - """Load marketplace.json and return parsed structure.""" +def validate_marketplace(skills: list[dict[str, str]]) -> list[str]: + """Validate marketplace.json against discovered skills. Returns error messages.""" if not MARKETPLACE_PATH.exists(): - raise FileNotFoundError(f"marketplace.json not found at {MARKETPLACE_PATH}") - return json.loads(MARKETPLACE_PATH.read_text(encoding="utf-8")) - - -def generate_readme_table(skills: list[dict[str, str]]) -> str: - """Generate the skills table for README.md using marketplace.json names.""" - marketplace = load_marketplace() - plugins = {p["source"]: p for p in marketplace.get("plugins", [])} - - lines = [ - "| Name | Description | Documentation |", - "|------|-------------|---------------|", - ] - - for skill in skills: - source = f"./{skill['path']}" - plugin = plugins.get(source, {}) - name = plugin.get("name", skill["name"]) - description = plugin.get("description", skill["description"]) - doc_link = f"[SKILL.md]({skill['path']}/SKILL.md)" - lines.append(f"| `{name}` | {description} | {doc_link} |") - - return "\n".join(lines) - - -def update_readme(skills: list[dict[str, str]]) -> bool: - """ - Update the README.md skills table between markers. - Returns True if the file was updated, False if markers not found. - """ - if not README_PATH.exists(): - print(f"Warning: README.md not found at {README_PATH}", file=sys.stderr) - return False - - content = README_PATH.read_text(encoding="utf-8") - - start_idx = content.find(README_TABLE_START) - end_idx = content.find(README_TABLE_END) - - if start_idx == -1 or end_idx == -1: - print( - f"Warning: README.md markers not found. Add {README_TABLE_START} and " - f"{README_TABLE_END} to enable table generation.", - file=sys.stderr, - ) - return False - - if end_idx < start_idx: - print("Warning: README.md markers are in wrong order.", file=sys.stderr) - return False - - table = generate_readme_table(skills) - new_content = ( - content[: start_idx + len(README_TABLE_START)] - + "\n" - + table - + "\n" - + content[end_idx:] - ) - - README_PATH.write_text(new_content, encoding="utf-8") - return True + return [f"marketplace.json not found at {MARKETPLACE_PATH}"] - -def validate_marketplace(skills: list[dict[str, str]]) -> list[str]: - """ - Validate marketplace.json against discovered skills. - Returns list of error messages (empty = passed). - """ - errors: list[str] = [] - marketplace = load_marketplace() + marketplace = json.loads(MARKETPLACE_PATH.read_text(encoding="utf-8")) plugins = marketplace.get("plugins", []) + errors: list[str] = [] - # Build lookups (normalize paths: skill uses "skills/x", marketplace uses "./skills/x") - skill_by_source = {f"./{s['path']}": s for s in skills} - plugin_by_source = {p["source"]: p for p in plugins} - - # Check: every skill has a marketplace entry with matching name - for skill in skills: - expected_source = f"./{skill['path']}" - if expected_source not in plugin_by_source: - errors.append( - f"Skill '{skill['name']}' at '{skill['path']}' is missing from marketplace.json" - ) - elif plugin_by_source[expected_source]["name"] != skill["name"]: - errors.append( - f"Name mismatch at '{expected_source}': " - f"SKILL.md='{skill['name']}', marketplace.json='{plugin_by_source[expected_source]['name']}'" - ) - - # Check: every marketplace plugin with skills has a corresponding skill + # Every plugin with skills should have at least one SKILL.md for plugin in plugins: - # Skip plugins that don't have skills (e.g., commands-only plugins) - if "skills" not in plugin: - continue - if plugin["source"] not in skill_by_source: + source = plugin.get("source", "").lstrip("./") + plugin_skills = [s for s in skills if s["path"].startswith(source)] + if not plugin_skills: errors.append( - f"Marketplace plugin '{plugin['name']}' at '{plugin['source']}' has no SKILL.md" + f"Plugin '{plugin['name']}' at '{source}' has no SKILL.md files" ) - return errors - - -def parse_version(version: str) -> tuple[int, int, int]: - """Parse semver string to tuple.""" - match = re.match(r"(\d+)\.(\d+)\.(\d+)", version) - if not match: - return (1, 0, 0) - return (int(match.group(1)), int(match.group(2)), int(match.group(3))) - - -def format_version(version: tuple[int, int, int]) -> str: - """Format version tuple to string.""" - return f"{version[0]}.{version[1]}.{version[2]}" - - -def get_bump_type(commit_msg: str) -> str: - """ - Determine version bump type from conventional commit message. - Returns: 'major', 'minor', 'patch', or 'none' - """ - msg_lower = commit_msg.lower() - - # Major: BREAKING CHANGE or ! after type - if "breaking change" in msg_lower or re.match(r"^\w+!:", commit_msg): - return "major" - - # Minor: feat - if re.match(r"^feat(\(.+\))?:", commit_msg, re.IGNORECASE): - return "minor" - - # Patch: fix, docs, chore, refactor, style, test, perf, ci, build - patch_types = ["fix", "docs", "chore", "refactor", "style", "test", "perf", "ci", "build"] - for t in patch_types: - if re.match(rf"^{t}(\(.+\))?:", commit_msg, re.IGNORECASE): - return "patch" - - return "none" - - -def bump_version(version: str, bump_type: str) -> str: - """Bump version based on type.""" - major, minor, patch = parse_version(version) - - if bump_type == "major": - return format_version((major + 1, 0, 0)) - elif bump_type == "minor": - return format_version((major, minor + 1, 0)) - elif bump_type == "patch": - return format_version((major, minor, patch + 1)) - return version - - -def update_user_agent_in_skill(skill_name: str, new_version: str) -> bool: - """ - Update USER_AGENT version in skill's run_actor.py script. - Returns True if updated, False otherwise. - """ - script_path = SKILLS_DIR / skill_name / "reference" / "scripts" / "run_actor.py" - if not script_path.exists(): - return False - - content = script_path.read_text(encoding="utf-8") - - # Pattern: USER_AGENT = "apify-agent-skills/skill-name-X.Y.Z" - pattern = rf'(USER_AGENT\s*=\s*"apify-agent-skills/{re.escape(skill_name)}-)\d+\.\d+\.\d+"' - replacement = rf'\g<1>{new_version}"' - - new_content, count = re.subn(pattern, replacement, content) - - if count > 0: - script_path.write_text(new_content, encoding="utf-8") - print(f"Updated USER_AGENT in {script_path.relative_to(ROOT)}: {new_version}") - return True - - return False - - -def get_changed_skills() -> set[str]: - """Get list of skill names that have staged changes.""" - try: - result = subprocess.run( - ["git", "diff", "--cached", "--name-only"], - capture_output=True, - text=True, - cwd=ROOT, - ) - changed_files = result.stdout.strip().split("\n") if result.stdout.strip() else [] - except Exception: - return set() - - changed_skills = set() - for f in changed_files: - # Match skills/skill-name/... pattern - match = re.match(r"skills/([^/]+)/", f) - if match: - changed_skills.add(match.group(1)) - - return changed_skills - - -def update_versions(commit_msg: str) -> bool: - """ - Update versions based on commit message and changed files. - Returns True if any version was bumped. - """ - bump_type = get_bump_type(commit_msg) - if bump_type == "none": - print(f"No version bump needed for commit: {commit_msg[:50]}...") - return False - - changed_skills = get_changed_skills() - bumped = False - - # Load marketplace.json - marketplace = load_marketplace() - - # Bump individual skill versions if they changed - for plugin in marketplace.get("plugins", []): - skill_name = plugin["source"].replace("./skills/", "") - if skill_name in changed_skills: - old_version = plugin.get("version", "1.0.0") - new_version = bump_version(old_version, bump_type) - if old_version != new_version: - plugin["version"] = new_version - print(f"Bumped {skill_name}: {old_version} → {new_version} ({bump_type})") - bumped = True - # Also update USER_AGENT in skill's run_actor.py - update_user_agent_in_skill(skill_name, new_version) - - # Bump marketplace version if any skill changed - if changed_skills or bumped: - old_version = marketplace.get("metadata", {}).get("version", "1.0.0") - new_version = bump_version(old_version, bump_type) - if old_version != new_version: - if "metadata" not in marketplace: - marketplace["metadata"] = {} - marketplace["metadata"]["version"] = new_version - print(f"Bumped marketplace: {old_version} → {new_version} ({bump_type})") - bumped = True - - # Save marketplace.json - if bumped: - MARKETPLACE_PATH.write_text( - json.dumps(marketplace, indent=2) + "\n", - encoding="utf-8" + # Every discovered skill should be covered by a plugin + for skill in skills: + found = any( + skill["path"].startswith(p.get("source", "").lstrip("./")) + for p in plugins ) - - # Also bump plugin.json version - if bumped and PLUGIN_PATH.exists(): - plugin_data = json.loads(PLUGIN_PATH.read_text(encoding="utf-8")) - old_version = plugin_data.get("version", "1.0.0") - new_version = bump_version(old_version, bump_type) - if old_version != new_version: - plugin_data["version"] = new_version - PLUGIN_PATH.write_text( - json.dumps(plugin_data, indent=2) + "\n", - encoding="utf-8" + if not found: + errors.append( + f"Skill '{skill['name']}' at '{skill['path']}' is not covered by any plugin" ) - print(f"Bumped plugin.json: {old_version} → {new_version}") - return bumped + return errors def main() -> None: - parser = argparse.ArgumentParser(description="Generate AGENTS.md and manage versions") - parser.add_argument( - "--bump", - metavar="COMMIT_MSG", - help="Bump versions based on conventional commit message" - ) - args = parser.parse_args() - - # Handle version bumping if requested - if args.bump: - update_versions(args.bump) - template = load_template() skills = collect_skills() output = render(template, skills) @@ -393,10 +130,6 @@ def main() -> None: sys.exit(1) print("Marketplace.json validation passed.") - # Update README.md skills table - if update_readme(skills): - print(f"Updated {README_PATH} skills table.") - if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/skills/apify-audience-analysis/SKILL.md b/skills/apify-audience-analysis/SKILL.md deleted file mode 100644 index 7ce31aa..0000000 --- a/skills/apify-audience-analysis/SKILL.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -name: apify-audience-analysis -description: Understand audience demographics, preferences, behavior patterns, and engagement quality across Facebook, Instagram, YouTube, and TikTok. ---- - -# Audience Analysis - -Analyze and understand your audience using Apify Actors to extract follower demographics, engagement patterns, and behavior data from multiple platforms. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Identify audience analysis type (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the analysis script -- [ ] Step 5: Summarize findings -``` - -### Step 1: Identify Audience Analysis Type - -Select the appropriate Actor based on analysis needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Facebook follower demographics | `apify/facebook-followers-following-scraper` | FB followers/following lists | -| Facebook engagement behavior | `apify/facebook-likes-scraper` | FB post likes analysis | -| Facebook video audience | `apify/facebook-reels-scraper` | FB Reels viewers | -| Facebook comment analysis | `apify/facebook-comments-scraper` | FB post/video comments | -| Facebook content engagement | `apify/facebook-posts-scraper` | FB post engagement metrics | -| Instagram audience sizing | `apify/instagram-profile-scraper` | IG profile demographics | -| Instagram location-based | `apify/instagram-search-scraper` | IG geo-tagged audience | -| Instagram tagged network | `apify/instagram-tagged-scraper` | IG tag network analysis | -| Instagram comprehensive | `apify/instagram-scraper` | Full IG audience data | -| Instagram API-based | `apify/instagram-api-scraper` | IG API access | -| Instagram follower counts | `apify/instagram-followers-count-scraper` | IG follower tracking | -| Instagram comment export | `apify/export-instagram-comments-posts` | IG comment bulk export | -| Instagram comment analysis | `apify/instagram-comment-scraper` | IG comment sentiment | -| YouTube viewer feedback | `streamers/youtube-comments-scraper` | YT comment analysis | -| YouTube channel audience | `streamers/youtube-channel-scraper` | YT channel subscribers | -| TikTok follower demographics | `clockworks/tiktok-followers-scraper` | TT follower lists | -| TikTok profile analysis | `clockworks/tiktok-profile-scraper` | TT profile demographics | -| TikTok comment analysis | `clockworks/tiktok-comments-scraper` | TT comment engagement | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `apify/facebook-followers-following-scraper`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Findings - -After completion, report: -- Number of audience members/profiles analyzed -- File location and name -- Key demographic insights -- Suggested next steps (deeper analysis, segmentation) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-audience-analysis/reference/scripts/run_actor.js b/skills/apify-audience-analysis/reference/scripts/run_actor.js deleted file mode 100644 index 1a28392..0000000 --- a/skills/apify-audience-analysis/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-audience-analysis-1.0.1'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-brand-reputation-monitoring/SKILL.md b/skills/apify-brand-reputation-monitoring/SKILL.md deleted file mode 100644 index 5fc0ee5..0000000 --- a/skills/apify-brand-reputation-monitoring/SKILL.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -name: apify-brand-reputation-monitoring -description: Track reviews, ratings, sentiment, and brand mentions across Google Maps, Booking.com, TripAdvisor, Facebook, Instagram, YouTube, and TikTok. Use when user asks to monitor brand reputation, analyze reviews, track mentions, or gather customer feedback. ---- - -# Brand Reputation Monitoring - -Scrape reviews, ratings, and brand mentions from multiple platforms using Apify Actors. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Determine data source (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the monitoring script -- [ ] Step 5: Summarize results -``` - -### Step 1: Determine Data Source - -Select the appropriate Actor based on user needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Google Maps reviews | `compass/crawler-google-places` | Business reviews, ratings | -| Google Maps review export | `compass/Google-Maps-Reviews-Scraper` | Dedicated review scraping | -| Booking.com hotels | `voyager/booking-scraper` | Hotel data, scores | -| Booking.com reviews | `voyager/booking-reviews-scraper` | Detailed hotel reviews | -| TripAdvisor reviews | `maxcopell/tripadvisor-reviews` | Attraction/restaurant reviews | -| Facebook reviews | `apify/facebook-reviews-scraper` | Page reviews | -| Facebook comments | `apify/facebook-comments-scraper` | Post comment monitoring | -| Facebook page metrics | `apify/facebook-pages-scraper` | Page ratings overview | -| Facebook reactions | `apify/facebook-likes-scraper` | Reaction type analysis | -| Instagram comments | `apify/instagram-comment-scraper` | Comment sentiment | -| Instagram hashtags | `apify/instagram-hashtag-scraper` | Brand hashtag monitoring | -| Instagram search | `apify/instagram-search-scraper` | Brand mention discovery | -| Instagram tagged posts | `apify/instagram-tagged-scraper` | Brand tag tracking | -| Instagram export | `apify/export-instagram-comments-posts` | Bulk comment export | -| Instagram comprehensive | `apify/instagram-scraper` | Full Instagram monitoring | -| Instagram API | `apify/instagram-api-scraper` | API-based monitoring | -| YouTube comments | `streamers/youtube-comments-scraper` | Video comment sentiment | -| TikTok comments | `clockworks/tiktok-comments-scraper` | TikTok sentiment | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `compass/crawler-google-places`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Results - -After completion, report: -- Number of reviews/mentions found -- File location and name -- Key fields available -- Suggested next steps (sentiment analysis, filtering) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-brand-reputation-monitoring/reference/scripts/run_actor.js b/skills/apify-brand-reputation-monitoring/reference/scripts/run_actor.js deleted file mode 100644 index edc49c6..0000000 --- a/skills/apify-brand-reputation-monitoring/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-brand-reputation-monitoring-1.1.1'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-competitor-intelligence/SKILL.md b/skills/apify-competitor-intelligence/SKILL.md deleted file mode 100644 index eb5bdc3..0000000 --- a/skills/apify-competitor-intelligence/SKILL.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -name: apify-competitor-intelligence -description: Analyze competitor strategies, content, pricing, ads, and market positioning across Google Maps, Booking.com, Facebook, Instagram, YouTube, and TikTok. ---- - -# Competitor Intelligence - -Analyze competitors using Apify Actors to extract data from multiple platforms. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Identify competitor analysis type (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the analysis script -- [ ] Step 5: Summarize findings -``` - -### Step 1: Identify Competitor Analysis Type - -Select the appropriate Actor based on analysis needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Competitor business data | `compass/crawler-google-places` | Location analysis | -| Competitor contact discovery | `poidata/google-maps-email-extractor` | Email extraction | -| Feature benchmarking | `compass/google-maps-extractor` | Detailed business data | -| Competitor review analysis | `compass/Google-Maps-Reviews-Scraper` | Review comparison | -| Hotel competitor data | `voyager/booking-scraper` | Hotel benchmarking | -| Hotel review comparison | `voyager/booking-reviews-scraper` | Review analysis | -| Competitor ad strategies | `apify/facebook-ads-scraper` | Ad creative analysis | -| Competitor page metrics | `apify/facebook-pages-scraper` | Page performance | -| Competitor content analysis | `apify/facebook-posts-scraper` | Post strategies | -| Competitor reels performance | `apify/facebook-reels-scraper` | Reels analysis | -| Competitor audience analysis | `apify/facebook-comments-scraper` | Comment sentiment | -| Competitor event monitoring | `apify/facebook-events-scraper` | Event tracking | -| Competitor audience overlap | `apify/facebook-followers-following-scraper` | Follower analysis | -| Competitor review benchmarking | `apify/facebook-reviews-scraper` | Review comparison | -| Competitor ad monitoring | `apify/facebook-search-scraper` | Ad discovery | -| Competitor profile metrics | `apify/instagram-profile-scraper` | Profile analysis | -| Competitor content monitoring | `apify/instagram-post-scraper` | Post tracking | -| Competitor engagement analysis | `apify/instagram-comment-scraper` | Comment analysis | -| Competitor reel performance | `apify/instagram-reel-scraper` | Reel metrics | -| Competitor growth tracking | `apify/instagram-followers-count-scraper` | Follower tracking | -| Comprehensive competitor data | `apify/instagram-scraper` | Full analysis | -| API-based competitor analysis | `apify/instagram-api-scraper` | API access | -| Competitor video analysis | `streamers/youtube-scraper` | Video metrics | -| Competitor sentiment analysis | `streamers/youtube-comments-scraper` | Comment sentiment | -| Competitor channel metrics | `streamers/youtube-channel-scraper` | Channel analysis | -| TikTok competitor analysis | `clockworks/tiktok-scraper` | TikTok data | -| Competitor video strategies | `clockworks/tiktok-video-scraper` | Video analysis | -| Competitor TikTok profiles | `clockworks/tiktok-profile-scraper` | Profile data | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `compass/crawler-google-places`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Findings - -After completion, report: -- Number of competitors analyzed -- File location and name -- Key competitive insights -- Suggested next steps (deeper analysis, benchmarking) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-competitor-intelligence/reference/scripts/run_actor.js b/skills/apify-competitor-intelligence/reference/scripts/run_actor.js deleted file mode 100644 index 6f373dd..0000000 --- a/skills/apify-competitor-intelligence/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-competitor-intelligence-1.0.1'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-content-analytics/SKILL.md b/skills/apify-content-analytics/SKILL.md deleted file mode 100644 index 021eeb5..0000000 --- a/skills/apify-content-analytics/SKILL.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -name: apify-content-analytics -description: Track engagement metrics, measure campaign ROI, and analyze content performance across Instagram, Facebook, YouTube, and TikTok. ---- - -# Content Analytics - -Track and analyze content performance using Apify Actors to extract engagement metrics from multiple platforms. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Identify content analytics type (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the analytics script -- [ ] Step 5: Summarize findings -``` - -### Step 1: Identify Content Analytics Type - -Select the appropriate Actor based on analytics needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Post engagement metrics | `apify/instagram-post-scraper` | Post performance | -| Reel performance | `apify/instagram-reel-scraper` | Reel analytics | -| Follower growth tracking | `apify/instagram-followers-count-scraper` | Growth metrics | -| Comment engagement | `apify/instagram-comment-scraper` | Comment analysis | -| Hashtag performance | `apify/instagram-hashtag-scraper` | Branded hashtags | -| Mention tracking | `apify/instagram-tagged-scraper` | Tag tracking | -| Comprehensive metrics | `apify/instagram-scraper` | Full data | -| API-based analytics | `apify/instagram-api-scraper` | API access | -| Facebook post performance | `apify/facebook-posts-scraper` | Post metrics | -| Reaction analysis | `apify/facebook-likes-scraper` | Engagement types | -| Facebook Reels metrics | `apify/facebook-reels-scraper` | Reels performance | -| Ad performance tracking | `apify/facebook-ads-scraper` | Ad analytics | -| Facebook comment analysis | `apify/facebook-comments-scraper` | Comment engagement | -| Page performance audit | `apify/facebook-pages-scraper` | Page metrics | -| YouTube video metrics | `streamers/youtube-scraper` | Video performance | -| YouTube Shorts analytics | `streamers/youtube-shorts-scraper` | Shorts performance | -| TikTok content metrics | `clockworks/tiktok-scraper` | TikTok analytics | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `apify/instagram-post-scraper`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Findings - -After completion, report: -- Number of content pieces analyzed -- File location and name -- Key performance insights -- Suggested next steps (deeper analysis, content optimization) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-content-analytics/reference/scripts/run_actor.js b/skills/apify-content-analytics/reference/scripts/run_actor.js deleted file mode 100644 index 418bc07..0000000 --- a/skills/apify-content-analytics/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-content-analytics-1.0.0'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-ecommerce/SKILL.md b/skills/apify-ecommerce/SKILL.md deleted file mode 100644 index 9735163..0000000 --- a/skills/apify-ecommerce/SKILL.md +++ /dev/null @@ -1,263 +0,0 @@ ---- -name: apify-ecommerce -description: Scrape e-commerce data for pricing intelligence, customer reviews, and seller discovery across Amazon, Walmart, eBay, IKEA, and 50+ marketplaces. Use when user asks to monitor prices, track competitors, analyze reviews, research products, or find sellers. ---- - -# E-commerce Data Extraction - -Extract product data, prices, reviews, and seller information from any e-commerce platform using Apify's E-commerce Scraping Tool. - -## Prerequisites - -- `.env` file with `APIFY_TOKEN` (at `~/.claude/.env`) -- Node.js 20.6+ (for native `--env-file` support) - -## Workflow Selection - -| User Need | Workflow | Best For | -|-----------|----------|----------| -| Track prices, compare products | Workflow 1: Products & Pricing | Price monitoring, MAP compliance, competitor analysis. Add AI summary for insights. | -| Analyze reviews (sentiment or quality) | Workflow 2: Reviews | Brand perception, customer sentiment, quality issues, defect patterns | -| Find sellers across stores | Workflow 3: Sellers | Unauthorized resellers, vendor discovery via Google Shopping | - -## Progress Tracking - -``` -Task Progress: -- [ ] Step 1: Select workflow and determine data source -- [ ] Step 2: Configure Actor input -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the extraction script -- [ ] Step 5: Summarize results -``` - ---- - -## Workflow 1: Products & Pricing - -**Use case:** Extract product data, prices, and stock status. Track competitor prices, detect MAP violations, benchmark products, or research markets. - -**Best for:** Pricing analysts, product managers, market researchers. - -### Input Options - -| Input Type | Field | Description | -|------------|-------|-------------| -| Product URLs | `detailsUrls` | Direct URLs to product pages (use object format) | -| Category URLs | `listingUrls` | URLs to category/search result pages | -| Keyword Search | `keyword` + `marketplaces` | Search term across selected marketplaces | - -### Example - Product URLs -```json -{ - "detailsUrls": [ - {"url": "https://www.amazon.com/dp/B09V3KXJPB"}, - {"url": "https://www.walmart.com/ip/123456789"} - ], - "additionalProperties": true -} -``` - -### Example - Keyword Search -```json -{ - "keyword": "Samsung Galaxy S24", - "marketplaces": ["www.amazon.com", "www.walmart.com"], - "additionalProperties": true, - "maxProductResults": 50 -} -``` - -### Optional: AI Summary - -Add these fields to get AI-generated insights: - -| Field | Description | -|-------|-------------| -| `fieldsToAnalyze` | Data points to analyze: `["name", "offers", "brand", "description"]` | -| `customPrompt` | Custom analysis instructions | - -**Example with AI summary:** -```json -{ - "keyword": "robot vacuum", - "marketplaces": ["www.amazon.com"], - "maxProductResults": 50, - "additionalProperties": true, - "fieldsToAnalyze": ["name", "offers", "brand"], - "customPrompt": "Summarize price range and identify top brands" -} -``` - -### Output Fields -- `name` - Product name -- `url` - Product URL -- `offers.price` - Current price -- `offers.priceCurrency` - Currency code (may vary by seller region) -- `brand.slogan` - Brand name (nested in object) -- `image` - Product image URL -- Additional seller/stock info when `additionalProperties: true` - -> **Note:** Currency may vary in results even for US searches, as prices reflect different seller regions. - ---- - -## Workflow 2: Customer Reviews - -**Use case:** Extract reviews for sentiment analysis, brand perception monitoring, or quality issue detection. - -**Best for:** Brand managers, customer experience teams, QA teams, product managers. - -### Input Options - -| Input Type | Field | Description | -|------------|-------|-------------| -| Product URLs | `reviewListingUrls` | Product pages to extract reviews from | -| Keyword Search | `keywordReviews` + `marketplacesReviews` | Search for product reviews by keyword | - -### Example - Extract Reviews from Product -```json -{ - "reviewListingUrls": [ - {"url": "https://www.amazon.com/dp/B09V3KXJPB"} - ], - "sortReview": "Most recent", - "additionalReviewProperties": true, - "maxReviewResults": 500 -} -``` - -### Example - Keyword Search -```json -{ - "keywordReviews": "wireless earbuds", - "marketplacesReviews": ["www.amazon.com"], - "sortReview": "Most recent", - "additionalReviewProperties": true, - "maxReviewResults": 200 -} -``` - -### Sort Options -- `Most recent` - Latest reviews first (recommended) -- `Most relevant` - Platform default relevance -- `Most helpful` - Highest voted reviews -- `Highest rated` - 5-star reviews first -- `Lowest rated` - 1-star reviews first - -> **Note:** The `sortReview: "Lowest rated"` option may not work consistently across all marketplaces. For quality analysis, collect a large sample and filter by rating in post-processing. - -### Quality Analysis Tips -- Set high `maxReviewResults` for statistical significance -- Look for recurring keywords: "broke", "defect", "quality", "returned" -- Filter results by rating if sorting doesn't work as expected -- Cross-reference with competitor products for benchmarking - ---- - -## Workflow 3: Seller Intelligence - -**Use case:** Find sellers across stores, discover unauthorized resellers, evaluate vendor options. - -**Best for:** Brand protection teams, procurement, supply chain managers. - -> **Note:** This workflow uses Google Shopping to find sellers across stores. Direct seller profile URLs are not reliably supported. - -### Input Configuration -```json -{ - "googleShoppingSearchKeyword": "Nike Air Max 90", - "scrapeSellersFromGoogleShopping": true, - "countryCode": "us", - "maxGoogleShoppingSellersPerProduct": 20, - "maxGoogleShoppingResults": 100 -} -``` - -### Options -| Field | Description | -|-------|-------------| -| `googleShoppingSearchKeyword` | Product name to search | -| `scrapeSellersFromGoogleShopping` | Set to `true` to extract sellers | -| `scrapeProductsFromGoogleShopping` | Set to `true` to also extract product details | -| `countryCode` | Target country (e.g., `us`, `uk`, `de`) | -| `maxGoogleShoppingSellersPerProduct` | Max sellers per product | -| `maxGoogleShoppingResults` | Total result limit | - ---- - -## Supported Marketplaces - -### Amazon (20+ regions) -`www.amazon.com`, `www.amazon.co.uk`, `www.amazon.de`, `www.amazon.fr`, `www.amazon.it`, `www.amazon.es`, `www.amazon.ca`, `www.amazon.com.au`, `www.amazon.co.jp`, `www.amazon.in`, `www.amazon.com.br`, `www.amazon.com.mx`, `www.amazon.nl`, `www.amazon.pl`, `www.amazon.se`, `www.amazon.ae`, `www.amazon.sa`, `www.amazon.sg`, `www.amazon.com.tr`, `www.amazon.eg` - -### Major US Retailers -`www.walmart.com`, `www.costco.com`, `www.costco.ca`, `www.homedepot.com` - -### European Retailers -`allegro.pl`, `allegro.cz`, `allegro.sk`, `www.alza.cz`, `www.alza.sk`, `www.alza.de`, `www.alza.at`, `www.alza.hu`, `www.kaufland.de`, `www.kaufland.pl`, `www.kaufland.cz`, `www.kaufland.sk`, `www.kaufland.at`, `www.kaufland.fr`, `www.kaufland.it`, `www.cdiscount.com` - -### IKEA (40+ country/language combinations) -Supports all major IKEA regional sites with multiple language options. - -### Google Shopping -Use for seller discovery across multiple stores. - ---- - -## Running the Extraction - -### Step 1: Set Skill Path -```bash -SKILL_PATH=~/.claude/skills/apify-ecommerce -``` - -### Step 2: Run Script - -**Quick answer (display in chat):** -```bash -node --env-file=~/.claude/.env $SKILL_PATH/reference/scripts/run_actor.js \ - --actor "apify/e-commerce-scraping-tool" \ - --input 'JSON_INPUT' -``` - -**CSV export:** -```bash -node --env-file=~/.claude/.env $SKILL_PATH/reference/scripts/run_actor.js \ - --actor "apify/e-commerce-scraping-tool" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_filename.csv \ - --format csv -``` - -**JSON export:** -```bash -node --env-file=~/.claude/.env $SKILL_PATH/reference/scripts/run_actor.js \ - --actor "apify/e-commerce-scraping-tool" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_filename.json \ - --format json -``` - -### Step 3: Summarize Results - -Report: -- Number of items extracted -- File location (if exported) -- Key insights based on workflow: - - **Products:** Price range, outliers, MAP violations - - **Reviews:** Average rating, sentiment trends, quality issues - - **Sellers:** Seller count, unauthorized sellers found - ---- - -## Error Handling - -| Error | Solution | -|-------|----------| -| `APIFY_TOKEN not found` | Ensure `~/.claude/.env` contains `APIFY_TOKEN=your_token` | -| `Actor not found` | Verify Actor ID: `apify/e-commerce-scraping-tool` | -| `Run FAILED` | Check Apify console link in error output | -| `Timeout` | Reduce `maxProductResults` or increase `--timeout` | -| `No results` | Verify URLs are valid and accessible | -| `Invalid marketplace` | Check marketplace value matches supported list exactly | diff --git a/skills/apify-ecommerce/reference/scripts/package.json b/skills/apify-ecommerce/reference/scripts/package.json deleted file mode 100644 index 3dbc1ca..0000000 --- a/skills/apify-ecommerce/reference/scripts/package.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "type": "module" -} diff --git a/skills/apify-ecommerce/reference/scripts/run_actor.js b/skills/apify-ecommerce/reference/scripts/run_actor.js deleted file mode 100644 index 9c67d2e..0000000 --- a/skills/apify-ecommerce/reference/scripts/run_actor.js +++ /dev/null @@ -1,369 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output data.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-ecommerce-1.0.0'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., apify/e-commerce-scraping-tool) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 products - node --env-file=.env scripts/run_actor.js \\ - --actor "apify/e-commerce-scraping-tool" \\ - --input '{"keyword": "bluetooth headphones", "marketplaces": ["www.amazon.com"], "maxProductResults": 10}' - - # Export prices to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "apify/e-commerce-scraping-tool" \\ - --input '{"detailsUrls": ["https://amazon.com/dp/B09V3KXJPB"]}' \\ - --output prices.csv --format csv - - # Export reviews to JSON - node --env-file=.env scripts/run_actor.js \\ - --actor "apify/e-commerce-scraping-tool" \\ - --input '{"reviewListingUrls": ["https://amazon.com/dp/B09V3KXJPB"], "maxReviewResults": 100}' \\ - --output reviews.json --format json -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-influencer-discovery/SKILL.md b/skills/apify-influencer-discovery/SKILL.md deleted file mode 100644 index 12404a0..0000000 --- a/skills/apify-influencer-discovery/SKILL.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -name: apify-influencer-discovery -description: Find and evaluate influencers for brand partnerships, verify authenticity, and track collaboration performance across Instagram, Facebook, YouTube, and TikTok. ---- - -# Influencer Discovery - -Discover and analyze influencers across multiple platforms using Apify Actors. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Determine discovery source (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the discovery script -- [ ] Step 5: Summarize results -``` - -### Step 1: Determine Discovery Source - -Select the appropriate Actor based on user needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Influencer profiles | `apify/instagram-profile-scraper` | Profile metrics, bio, follower counts | -| Find by hashtag | `apify/instagram-hashtag-scraper` | Discover influencers using specific hashtags | -| Reel engagement | `apify/instagram-reel-scraper` | Analyze reel performance and engagement | -| Discovery by niche | `apify/instagram-search-scraper` | Search for influencers by keyword/niche | -| Brand mentions | `apify/instagram-tagged-scraper` | Track who tags brands/products | -| Comprehensive data | `apify/instagram-scraper` | Full profile, posts, comments analysis | -| API-based discovery | `apify/instagram-api-scraper` | Fast API-based data extraction | -| Engagement analysis | `apify/export-instagram-comments-posts` | Export comments for sentiment analysis | -| Facebook content | `apify/facebook-posts-scraper` | Analyze Facebook post performance | -| Micro-influencers | `apify/facebook-groups-scraper` | Find influencers in niche groups | -| Influential pages | `apify/facebook-search-scraper` | Search for influential pages | -| YouTube creators | `streamers/youtube-channel-scraper` | Channel metrics and subscriber data | -| TikTok influencers | `clockworks/tiktok-scraper` | Comprehensive TikTok data extraction | -| TikTok (free) | `clockworks/free-tiktok-scraper` | Free TikTok data extractor | -| Live streamers | `clockworks/tiktok-live-scraper` | Discover live streaming influencers | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `apify/instagram-profile-scraper`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Results - -After completion, report: -- Number of influencers found -- File location and name -- Key metrics available (followers, engagement rate, etc.) -- Suggested next steps (filtering, outreach, deeper analysis) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-influencer-discovery/reference/scripts/run_actor.js b/skills/apify-influencer-discovery/reference/scripts/run_actor.js deleted file mode 100644 index e600ded..0000000 --- a/skills/apify-influencer-discovery/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-influencer-discovery-1.0.0'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-lead-generation/SKILL.md b/skills/apify-lead-generation/SKILL.md deleted file mode 100644 index 45e1d40..0000000 --- a/skills/apify-lead-generation/SKILL.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -name: apify-lead-generation -description: Generates B2B/B2C leads by scraping Google Maps, websites, Instagram, TikTok, Facebook, LinkedIn, YouTube, and Google Search. Use when user asks to find leads, prospects, businesses, build lead lists, enrich contacts, or scrape profiles for sales outreach. ---- - -# Lead Generation - -Scrape leads from multiple platforms using Apify Actors. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Determine lead source (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the lead finder script -- [ ] Step 5: Summarize results -``` - -### Step 1: Determine Lead Source - -Select the appropriate Actor based on user needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Local businesses | `compass/crawler-google-places` | Restaurants, gyms, shops | -| Contact enrichment | `vdrmota/contact-info-scraper` | Emails, phones from URLs | -| Instagram profiles | `apify/instagram-profile-scraper` | Influencer discovery | -| Instagram posts/comments | `apify/instagram-scraper` | Posts, comments, hashtags, places | -| Instagram search | `apify/instagram-search-scraper` | Places, users, hashtags discovery | -| TikTok videos/hashtags | `clockworks/tiktok-scraper` | Comprehensive TikTok data extraction | -| TikTok hashtags/profiles | `clockworks/free-tiktok-scraper` | Free TikTok data extractor | -| TikTok user search | `clockworks/tiktok-user-search-scraper` | Find users by keywords | -| TikTok profiles | `clockworks/tiktok-profile-scraper` | Creator outreach | -| TikTok followers/following | `clockworks/tiktok-followers-scraper` | Audience analysis, segmentation | -| Facebook pages | `apify/facebook-pages-scraper` | Business contacts | -| Facebook page contacts | `apify/facebook-page-contact-information` | Extract emails, phones, addresses | -| Facebook groups | `apify/facebook-groups-scraper` | Buying intent signals | -| Facebook events | `apify/facebook-events-scraper` | Event networking, partnerships | -| Google Search | `apify/google-search-scraper` | Broad lead discovery | -| YouTube channels | `streamers/youtube-scraper` | Creator partnerships | -| Google Maps emails | `poidata/google-maps-email-extractor` | Direct email extraction | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `compass/crawler-google-places`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Results - -After completion, report: -- Number of leads found -- File location and name -- Key fields available -- Suggested next steps (filtering, enrichment) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-lead-generation/reference/scripts/run_actor.js b/skills/apify-lead-generation/reference/scripts/run_actor.js deleted file mode 100644 index 6cd4acc..0000000 --- a/skills/apify-lead-generation/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-lead-generation-1.1.11'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-market-research/SKILL.md b/skills/apify-market-research/SKILL.md deleted file mode 100644 index 95e926b..0000000 --- a/skills/apify-market-research/SKILL.md +++ /dev/null @@ -1,119 +0,0 @@ ---- -name: apify-market-research -description: Analyze market conditions, geographic opportunities, pricing, consumer behavior, and product validation across Google Maps, Facebook, Instagram, Booking.com, and TripAdvisor. ---- - -# Market Research - -Conduct market research using Apify Actors to extract data from multiple platforms. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Identify market research type (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the analysis script -- [ ] Step 5: Summarize findings -``` - -### Step 1: Identify Market Research Type - -Select the appropriate Actor based on research needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Market density | `compass/crawler-google-places` | Location analysis | -| Geospatial analysis | `compass/google-maps-extractor` | Business mapping | -| Regional interest | `apify/google-trends-scraper` | Trend data | -| Pricing and demand | `apify/facebook-marketplace-scraper` | Market pricing | -| Event market | `apify/facebook-events-scraper` | Event analysis | -| Consumer needs | `apify/facebook-groups-scraper` | Group research | -| Market landscape | `apify/facebook-pages-scraper` | Business pages | -| Business density | `apify/facebook-page-contact-information` | Contact data | -| Cultural insights | `apify/facebook-photos-scraper` | Visual research | -| Niche targeting | `apify/instagram-hashtag-scraper` | Hashtag research | -| Hashtag stats | `apify/instagram-hashtag-stats` | Market sizing | -| Market activity | `apify/instagram-reel-scraper` | Activity analysis | -| Market intelligence | `apify/instagram-scraper` | Full data | -| Product launch research | `apify/instagram-api-scraper` | API access | -| Hospitality market | `voyager/booking-scraper` | Hotel data | -| Tourism insights | `maxcopell/tripadvisor-reviews` | Review analysis | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `compass/crawler-google-places`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Findings - -After completion, report: -- Number of results found -- File location and name -- Key market insights -- Suggested next steps (deeper analysis, validation) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-market-research/reference/scripts/run_actor.js b/skills/apify-market-research/reference/scripts/run_actor.js deleted file mode 100644 index 7a0a904..0000000 --- a/skills/apify-market-research/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-market-research-1.0.0'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-trend-analysis/SKILL.md b/skills/apify-trend-analysis/SKILL.md deleted file mode 100644 index 7692cde..0000000 --- a/skills/apify-trend-analysis/SKILL.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -name: apify-trend-analysis -description: Discover and track emerging trends across Google Trends, Instagram, Facebook, YouTube, and TikTok to inform content strategy. ---- - -# Trend Analysis - -Discover and track emerging trends using Apify Actors to extract data from multiple platforms. - -## Prerequisites -(No need to check it upfront) - -- `.env` file with `APIFY_TOKEN` -- Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` - -## Workflow - -Copy this checklist and track progress: - -``` -Task Progress: -- [ ] Step 1: Identify trend type (select Actor) -- [ ] Step 2: Fetch Actor schema via mcpc -- [ ] Step 3: Ask user preferences (format, filename) -- [ ] Step 4: Run the analysis script -- [ ] Step 5: Summarize findings -``` - -### Step 1: Identify Trend Type - -Select the appropriate Actor based on research needs: - -| User Need | Actor ID | Best For | -|-----------|----------|----------| -| Search trends | `apify/google-trends-scraper` | Google Trends data | -| Hashtag tracking | `apify/instagram-hashtag-scraper` | Hashtag content | -| Hashtag metrics | `apify/instagram-hashtag-stats` | Performance stats | -| Visual trends | `apify/instagram-post-scraper` | Post analysis | -| Trending discovery | `apify/instagram-search-scraper` | Search trends | -| Comprehensive tracking | `apify/instagram-scraper` | Full data | -| API-based trends | `apify/instagram-api-scraper` | API access | -| Engagement trends | `apify/export-instagram-comments-posts` | Comment tracking | -| Product trends | `apify/facebook-marketplace-scraper` | Marketplace data | -| Visual analysis | `apify/facebook-photos-scraper` | Photo trends | -| Community trends | `apify/facebook-groups-scraper` | Group monitoring | -| YouTube Shorts | `streamers/youtube-shorts-scraper` | Short-form trends | -| YouTube hashtags | `streamers/youtube-video-scraper-by-hashtag` | Hashtag videos | -| TikTok hashtags | `clockworks/tiktok-hashtag-scraper` | Hashtag content | -| Trending sounds | `clockworks/tiktok-sound-scraper` | Audio trends | -| TikTok ads | `clockworks/tiktok-ads-scraper` | Ad trends | -| Discover page | `clockworks/tiktok-discover-scraper` | Discover trends | -| Explore trends | `clockworks/tiktok-explore-scraper` | Explore content | -| Trending content | `clockworks/tiktok-trends-scraper` | Viral content | - -### Step 2: Fetch Actor Schema - -Fetch the Actor's input schema and details dynamically using mcpc: - -```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" -``` - -Replace `ACTOR_ID` with the selected Actor (e.g., `apify/google-trends-scraper`). - -This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) - -### Step 3: Ask User Preferences - -Before running, ask: -1. **Output format**: - - **Quick answer** - Display top few results in chat (no file saved) - - **CSV** - Full export with all fields - - **JSON** - Full export in JSON format -2. **Number of results**: Based on character of use case - -### Step 4: Run the Script - -**Quick answer (display in chat, no file):** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' -``` - -**CSV:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.csv \ - --format csv -``` - -**JSON:** -```bash -node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/run_actor.js \ - --actor "ACTOR_ID" \ - --input 'JSON_INPUT' \ - --output YYYY-MM-DD_OUTPUT_FILE.json \ - --format json -``` - -### Step 5: Summarize Findings - -After completion, report: -- Number of results found -- File location and name -- Key trend insights -- Suggested next steps (deeper analysis, content opportunities) - - -## Error Handling - -`APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` -`Actor not found` - Check Actor ID spelling -`Run FAILED` - Ask user to check Apify console link in error output -`Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-trend-analysis/reference/scripts/run_actor.js b/skills/apify-trend-analysis/reference/scripts/run_actor.js deleted file mode 100644 index 5512427..0000000 --- a/skills/apify-trend-analysis/reference/scripts/run_actor.js +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env node -/** - * Apify Actor Runner - Runs Apify actors and exports results. - * - * Usage: - * # Quick answer (display in chat, no file saved) - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - * - * # Export to file - * node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' --output leads.csv --format csv - */ - -import { parseArgs } from 'node:util'; -import { writeFileSync, statSync } from 'node:fs'; - -// User-Agent for tracking skill usage in Apify analytics -const USER_AGENT = 'apify-agent-skills/apify-trend-analysis-1.0.0'; - -// Parse command-line arguments -function parseCliArgs() { - const options = { - actor: { type: 'string', short: 'a' }, - input: { type: 'string', short: 'i' }, - output: { type: 'string', short: 'o' }, - format: { type: 'string', short: 'f', default: 'csv' }, - timeout: { type: 'string', short: 't', default: '600' }, - 'poll-interval': { type: 'string', default: '5' }, - help: { type: 'boolean', short: 'h' }, - }; - - const { values } = parseArgs({ options, allowPositionals: false }); - - if (values.help) { - printHelp(); - process.exit(0); - } - - if (!values.actor) { - console.error('Error: --actor is required'); - printHelp(); - process.exit(1); - } - - if (!values.input) { - console.error('Error: --input is required'); - printHelp(); - process.exit(1); - } - - return { - actor: values.actor, - input: values.input, - output: values.output, - format: values.format || 'csv', - timeout: parseInt(values.timeout, 10), - pollInterval: parseInt(values['poll-interval'], 10), - }; -} - -function printHelp() { - console.log(` -Apify Actor Runner - Run Apify actors and export results - -Usage: - node --env-file=.env scripts/run_actor.js --actor ACTOR_ID --input '{}' - -Options: - --actor, -a Actor ID (e.g., compass/crawler-google-places) [required] - --input, -i Actor input as JSON string [required] - --output, -o Output file path (optional - if not provided, displays quick answer) - --format, -f Output format: csv, json (default: csv) - --timeout, -t Max wait time in seconds (default: 600) - --poll-interval Seconds between status checks (default: 5) - --help, -h Show this help message - -Output Formats: - JSON (all data) --output file.json --format json - CSV (all data) --output file.csv --format csv - Quick answer (no --output) - displays top 5 in chat - -Examples: - # Quick answer - display top 5 in chat - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' - - # Export all data to CSV - node --env-file=.env scripts/run_actor.js \\ - --actor "compass/crawler-google-places" \\ - --input '{"searchStringsArray": ["coffee shops"], "locationQuery": "Seattle, USA"}' \\ - --output leads.csv --format csv -`); -} - -// Start an actor run and return { runId, datasetId } -async function startActor(token, actorId, inputJson) { - // Convert "author/actor" format to "author~actor" for API compatibility - const apiActorId = actorId.replace('/', '~'); - const url = `https://api.apify.com/v2/acts/${apiActorId}/runs?token=${encodeURIComponent(token)}`; - - let data; - try { - data = JSON.parse(inputJson); - } catch (e) { - console.error(`Error: Invalid JSON input: ${e.message}`); - process.exit(1); - } - - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${USER_AGENT}/start_actor`, - }, - body: JSON.stringify(data), - }); - - if (response.status === 404) { - console.error(`Error: Actor '${actorId}' not found`); - process.exit(1); - } - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: API request failed (${response.status}): ${text}`); - process.exit(1); - } - - const result = await response.json(); - return { - runId: result.data.id, - datasetId: result.data.defaultDatasetId, - }; -} - -// Poll run status until complete or timeout -async function pollUntilComplete(token, runId, timeout, interval) { - const url = `https://api.apify.com/v2/actor-runs/${runId}?token=${encodeURIComponent(token)}`; - const startTime = Date.now(); - let lastStatus = null; - - while (true) { - const response = await fetch(url); - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to get run status: ${text}`); - process.exit(1); - } - - const result = await response.json(); - const status = result.data.status; - - // Only print when status changes - if (status !== lastStatus) { - console.log(`Status: ${status}`); - lastStatus = status; - } - - if (['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(status)) { - return status; - } - - const elapsed = (Date.now() - startTime) / 1000; - if (elapsed > timeout) { - console.error(`Warning: Timeout after ${timeout}s, actor still running`); - return 'TIMED-OUT'; - } - - await sleep(interval * 1000); - } -} - -// Download dataset items -async function downloadResults(token, datasetId, outputPath, format) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/download_${format}`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - - if (format === 'json') { - writeFileSync(outputPath, JSON.stringify(data, null, 2)); - } else { - // CSV output - if (data.length > 0) { - const fieldnames = Object.keys(data[0]); - const csvLines = [fieldnames.join(',')]; - - for (const row of data) { - const values = fieldnames.map((key) => { - let value = row[key]; - - // Truncate long text fields - if (typeof value === 'string' && value.length > 200) { - value = value.slice(0, 200) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - value = JSON.stringify(value) || ''; - } - - // CSV escape: wrap in quotes if contains comma, quote, or newline - if (value === null || value === undefined) { - return ''; - } - const strValue = String(value); - if (strValue.includes(',') || strValue.includes('"') || strValue.includes('\n')) { - return `"${strValue.replace(/"/g, '""')}"`; - } - return strValue; - }); - csvLines.push(values.join(',')); - } - - writeFileSync(outputPath, csvLines.join('\n')); - } else { - writeFileSync(outputPath, ''); - } - } - - console.log(`Saved to: ${outputPath}`); -} - -// Display top 5 results in chat format -async function displayQuickAnswer(token, datasetId) { - const url = `https://api.apify.com/v2/datasets/${datasetId}/items?token=${encodeURIComponent(token)}&format=json`; - - const response = await fetch(url, { - headers: { - 'User-Agent': `${USER_AGENT}/quick_answer`, - }, - }); - - if (!response.ok) { - const text = await response.text(); - console.error(`Error: Failed to download results: ${text}`); - process.exit(1); - } - - const data = await response.json(); - const total = data.length; - - if (total === 0) { - console.log('\nNo results found.'); - return; - } - - // Display top 5 - console.log(`\n${'='.repeat(60)}`); - console.log(`TOP 5 RESULTS (of ${total} total)`); - console.log('='.repeat(60)); - - for (let i = 0; i < Math.min(5, data.length); i++) { - const item = data[i]; - console.log(`\n--- Result ${i + 1} ---`); - - for (const [key, value] of Object.entries(item)) { - let displayValue = value; - - // Truncate long values - if (typeof value === 'string' && value.length > 100) { - displayValue = value.slice(0, 100) + '...'; - } else if (Array.isArray(value) || (typeof value === 'object' && value !== null)) { - const jsonStr = JSON.stringify(value); - displayValue = jsonStr.length > 100 ? jsonStr.slice(0, 100) + '...' : jsonStr; - } - - console.log(` ${key}: ${displayValue}`); - } - } - - console.log(`\n${'='.repeat(60)}`); - if (total > 5) { - console.log(`Showing 5 of ${total} results.`); - } - console.log(`Full data available at: https://console.apify.com/storage/datasets/${datasetId}`); - console.log('='.repeat(60)); -} - -// Report summary of downloaded data -function reportSummary(outputPath, format) { - const stats = statSync(outputPath); - const size = stats.size; - - let count; - try { - const content = require('fs').readFileSync(outputPath, 'utf-8'); - if (format === 'json') { - const data = JSON.parse(content); - count = Array.isArray(data) ? data.length : 1; - } else { - // CSV - count lines minus header - const lines = content.split('\n').filter((line) => line.trim()); - count = Math.max(0, lines.length - 1); - } - } catch { - count = 'unknown'; - } - - console.log(`Records: ${count}`); - console.log(`Size: ${size.toLocaleString()} bytes`); -} - -// Helper: sleep for ms -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -// Main function -async function main() { - // Parse args first so --help works without token - const args = parseCliArgs(); - - // Check for APIFY_TOKEN - const token = process.env.APIFY_TOKEN; - if (!token) { - console.error('Error: APIFY_TOKEN not found in .env file'); - console.error(''); - console.error('Add your token to .env file:'); - console.error(' APIFY_TOKEN=your_token_here'); - console.error(''); - console.error('Get your token: https://console.apify.com/account/integrations'); - process.exit(1); - } - - // Start the actor run - console.log(`Starting actor: ${args.actor}`); - const { runId, datasetId } = await startActor(token, args.actor, args.input); - console.log(`Run ID: ${runId}`); - console.log(`Dataset ID: ${datasetId}`); - - // Poll for completion - const status = await pollUntilComplete(token, runId, args.timeout, args.pollInterval); - - if (status !== 'SUCCEEDED') { - console.error(`Error: Actor run ${status}`); - console.error(`Details: https://console.apify.com/actors/runs/${runId}`); - process.exit(1); - } - - // Determine output mode - if (args.output) { - // File output mode - await downloadResults(token, datasetId, args.output, args.format); - reportSummary(args.output, args.format); - } else { - // Quick answer mode - display in chat - await displayQuickAnswer(token, datasetId); - } -} - -main().catch((err) => { - console.error(`Error: ${err.message}`); - process.exit(1); -}); diff --git a/skills/apify-ultimate-scraper/SKILL.md b/skills/apify-ultimate-scraper/SKILL.md index 5c62d30..4da4583 100644 --- a/skills/apify-ultimate-scraper/SKILL.md +++ b/skills/apify-ultimate-scraper/SKILL.md @@ -12,7 +12,6 @@ AI-driven data extraction from 55+ Actors across all major platforms. This skill - `.env` file with `APIFY_TOKEN` - Node.js 20.6+ (for native `--env-file` support) -- `mcpc` CLI tool: `npm install -g @apify/mcpc` ## Workflow @@ -21,7 +20,7 @@ Copy this checklist and track progress: ``` Task Progress: - [ ] Step 1: Understand user goal and select Actor -- [ ] Step 2: Fetch Actor schema via mcpc +- [ ] Step 2: Fetch Actor schema - [ ] Step 3: Ask user preferences (format, filename) - [ ] Step 4: Run the scraper script - [ ] Step 5: Summarize results and offer follow-ups @@ -149,35 +148,39 @@ For complex tasks, chain multiple Actors: If none of the Actors above match the user's request, search the Apify Store directly: ```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call search-actors keywords:="SEARCH_KEYWORDS" limit:=10 offset:=0 category:="" | jq -r '.content[0].text' +node ${CLAUDE_PLUGIN_ROOT}/reference/scripts/search_actors.js --query "SEARCH_KEYWORDS" ``` Replace `SEARCH_KEYWORDS` with 1-3 simple terms (e.g., "LinkedIn profiles", "Amazon products", "Twitter"). ### Step 2: Fetch Actor Schema -Fetch the Actor's input schema and details dynamically using mcpc: +Fetch the Actor's input schema and details: ```bash -export $(grep APIFY_TOKEN .env | xargs) && mcpc --json mcp.apify.com --header "Authorization: Bearer $APIFY_TOKEN" tools-call fetch-actor-details actor:="ACTOR_ID" | jq -r ".content" +node --env-file=.env ${CLAUDE_PLUGIN_ROOT}/reference/scripts/fetch_actor_details.js --actor "ACTOR_ID" ``` Replace `ACTOR_ID` with the selected Actor (e.g., `compass/crawler-google-places`). This returns: -- Actor description and README -- Required and optional input parameters -- Output fields (if available) +- Actor info (title, description, URL, categories, stats, rating) +- README summary +- Input schema (required and optional parameters) ### Step 3: Ask User Preferences -Before running, ask: +**Skip this step** for simple lookups (e.g., "what's Nike's follower count?", "find me 5 coffee shops in Prague") — just use quick answer mode and move to Step 4. + +For larger scraping tasks, ask: 1. **Output format**: - **Quick answer** - Display top few results in chat (no file saved) - **CSV** - Full export with all fields - **JSON** - Full export in JSON format 2. **Number of results**: Based on character of use case +**Cost safety**: Always set a sensible result limit in the Actor input (e.g., `maxResults`, `resultsLimit`, `maxCrawledPages`, or equivalent field from the input schema). Default to 100 results unless the user explicitly asks for more. Warn the user before running large scrapes (1000+ results) as they consume more Apify credits. + ### Step 4: Run the Script **Quick answer (display in chat, no file):** @@ -224,7 +227,6 @@ After completion, report: ## Error Handling `APIFY_TOKEN not found` - Ask user to create `.env` with `APIFY_TOKEN=your_token` -`mcpc not found` - Ask user to install `npm install -g @apify/mcpc` `Actor not found` - Check Actor ID spelling `Run FAILED` - Ask user to check Apify console link in error output `Timeout` - Reduce input size or increase `--timeout` diff --git a/skills/apify-ultimate-scraper/reference/scripts/fetch_actor_details.js b/skills/apify-ultimate-scraper/reference/scripts/fetch_actor_details.js new file mode 100644 index 0000000..3d0a1a0 --- /dev/null +++ b/skills/apify-ultimate-scraper/reference/scripts/fetch_actor_details.js @@ -0,0 +1,136 @@ +#!/usr/bin/env node +/** + * Fetch Apify Actor details: README, input schema, and description. + * + * Usage: + * node --env-file=.env scripts/fetch_actor_details.js --actor "apify/instagram-profile-scraper" + */ + +import { parseArgs } from 'node:util'; + +const USER_AGENT = 'apify-agent-skills/apify-ultimate-scraper-1.3.0'; + +function parseCliArgs() { + const options = { + actor: { type: 'string', short: 'a' }, + help: { type: 'boolean', short: 'h' }, + }; + + const { values } = parseArgs({ options, allowPositionals: false }); + + if (values.help) { + console.log(` +Fetch Apify Actor details (README, input schema, description) + +Usage: + node --env-file=.env scripts/fetch_actor_details.js --actor "ACTOR_ID" + +Options: + --actor, -a Actor ID (e.g., apify/instagram-profile-scraper) [required] + --help, -h Show this help message +`); + process.exit(0); + } + + if (!values.actor) { + console.error('Error: --actor is required'); + process.exit(1); + } + + return { actor: values.actor }; +} + +async function fetchActorInfo(token, actorId) { + const apiActorId = actorId.replace('/', '~'); + const url = `https://api.apify.com/v2/acts/${apiActorId}?token=${encodeURIComponent(token)}`; + + const response = await fetch(url, { + headers: { 'User-Agent': `${USER_AGENT}/fetch_actor_info` }, + }); + + if (response.status === 404) { + console.error(`Error: Actor '${actorId}' not found`); + process.exit(1); + } + + if (!response.ok) { + const text = await response.text(); + console.error(`Error: Failed to fetch actor info (${response.status}): ${text}`); + process.exit(1); + } + + return (await response.json()).data; +} + +async function fetchBuildDetails(token, actorId, buildId) { + const apiActorId = actorId.replace('/', '~'); + const url = `https://api.apify.com/v2/acts/${apiActorId}/builds/${buildId}?token=${encodeURIComponent(token)}`; + + const response = await fetch(url, { + headers: { 'User-Agent': `${USER_AGENT}/fetch_build` }, + }); + + if (!response.ok) { + return null; + } + + return (await response.json()).data; +} + +async function main() { + const args = parseCliArgs(); + + const token = process.env.APIFY_TOKEN; + if (!token) { + console.error('Error: APIFY_TOKEN not found in .env file'); + console.error('Add your token to .env: APIFY_TOKEN=your_token_here'); + console.error('Get your token: https://console.apify.com/account/integrations'); + process.exit(1); + } + + // Step 1: Get actor info (includes readmeSummary, taggedBuilds) + const actorInfo = await fetchActorInfo(token, args.actor); + + // Step 2: Get build details for input schema + const buildId = actorInfo.taggedBuilds?.latest?.buildId; + let inputSchema = null; + + if (buildId) { + const build = await fetchBuildDetails(token, args.actor, buildId); + if (build) { + const schemaRaw = build.inputSchema; + if (schemaRaw) { + inputSchema = typeof schemaRaw === 'string' ? JSON.parse(schemaRaw) : schemaRaw; + } + } + } + + // Compose output + const stats = actorInfo.stats || {}; + const output = { + actorId: args.actor, + title: actorInfo.title || null, + url: `https://apify.com/${args.actor}`, + description: actorInfo.description || null, + categories: actorInfo.categories || [], + isDeprecated: actorInfo.isDeprecated || false, + stats: { + totalUsers: stats.totalUsers || 0, + monthlyUsers: stats.totalUsers30Days || 0, + bookmarks: stats.bookmarkCount || 0, + }, + rating: { + average: stats.actorReviewRating || null, + count: stats.actorReviewCount || 0, + }, + readmeSummary: actorInfo.readmeSummary || null, + inputSchema: inputSchema || null, + }; + + console.log(JSON.stringify(output, null, 2)); +} + +main().catch((err) => { + console.error(`Error: ${err.message}`); + process.exit(1); +}); diff --git a/skills/apify-ultimate-scraper/reference/scripts/search_actors.js b/skills/apify-ultimate-scraper/reference/scripts/search_actors.js new file mode 100644 index 0000000..e96823b --- /dev/null +++ b/skills/apify-ultimate-scraper/reference/scripts/search_actors.js @@ -0,0 +1,103 @@ +#!/usr/bin/env node +/** + * Search Apify Store for Actors matching keywords. + * + * Usage: + * node --env-file=.env scripts/search_actors.js --query "instagram" + * node --env-file=.env scripts/search_actors.js --query "amazon products" --limit 5 + */ + +import { parseArgs } from 'node:util'; + +const USER_AGENT = 'apify-agent-skills/apify-ultimate-scraper-1.3.0'; + +function parseCliArgs() { + const options = { + query: { type: 'string', short: 'q' }, + limit: { type: 'string', short: 'l', default: '10' }, + help: { type: 'boolean', short: 'h' }, + }; + + const { values } = parseArgs({ options, allowPositionals: false }); + + if (values.help) { + console.log(` +Search Apify Store for Actors + +Usage: + node --env-file=.env scripts/search_actors.js --query "KEYWORDS" + +Options: + --query, -q Search keywords (e.g., "instagram", "amazon products") [required] + --limit, -l Max results to return (default: 10) + --help, -h Show this help message +`); + process.exit(0); + } + + if (!values.query) { + console.error('Error: --query is required'); + process.exit(1); + } + + return { + query: values.query, + limit: parseInt(values.limit, 10) || 10, + }; +} + +async function searchStore(query, limit) { + const params = new URLSearchParams({ search: query, limit: String(limit) }); + const url = `https://api.apify.com/v2/store?${params}`; + + const response = await fetch(url, { + headers: { 'User-Agent': `${USER_AGENT}/search_actors` }, + }); + + if (!response.ok) { + const text = await response.text(); + console.error(`Error: Store search failed (${response.status}): ${text}`); + process.exit(1); + } + + const result = await response.json(); + return result.data?.items || []; +} + +function formatResults(actors) { + if (actors.length === 0) { + console.log('No actors found.'); + return; + } + + console.log(`Found ${actors.length} actor(s):\n`); + + for (const actor of actors) { + const id = `${actor.username}/${actor.name}`; + const title = actor.title || id; + const desc = actor.description + ? actor.description.length > 120 + ? actor.description.slice(0, 120) + '...' + : actor.description + : 'No description'; + const runs = actor.stats?.totalRuns?.toLocaleString() || '0'; + const users = actor.stats?.totalUsers?.toLocaleString() || '0'; + + console.log(` ${id}`); + console.log(` Title: ${title}`); + console.log(` ${desc}`); + console.log(` Runs: ${runs} | Users: ${users}`); + console.log(); + } +} + +async function main() { + const args = parseCliArgs(); + const actors = await searchStore(args.query, args.limit); + formatResults(actors); +} + +main().catch((err) => { + console.error(`Error: ${err.message}`); + process.exit(1); +});