diff --git a/CHANGELOG.md b/CHANGELOG.md index 68bfccd..0784492 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,4 +80,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- -For more details about changes, see the [commit history](https://github.com/anthropics/crawailer/commits/main). \ No newline at end of file +For more details about changes, see the [commit history](https://git.supported.systems/MCP/crawailer/commits/branch/main). \ No newline at end of file diff --git a/README.md b/README.md index cd4dfa9..4bb76d9 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,65 @@ # ๐Ÿ•ท๏ธ Crawailer -**The JavaScript-first web scraper that actually works with modern websites** +**The web scraper that doesn't suck at JavaScript** โœจ -> **Finally!** A Python library that handles React, Vue, Angular, and dynamic content without the headaches. When `requests` fails and Selenium feels like overkill, Crawailer delivers clean, AI-ready content extraction with bulletproof JavaScript execution. +> **Stop fighting modern websites.** While `requests` gives you empty `
`, Crawailer actually executes JavaScript and extracts real content from React, Vue, and Angular apps. Finally, web scraping that works in 2025. + +> โšก **Claude Code's new best friend** - Your AI assistant can now access ANY website ```python pip install crawailer ``` [![PyPI version](https://badge.fury.io/py/crawailer.svg)](https://badge.fury.io/py/crawailer) -[![Downloads](https://pepy.tech/badge/crawailer)](https://pepy.tech/project/crawailer) [![Python Support](https://img.shields.io/pypi/pyversions/crawailer.svg)](https://pypi.org/project/crawailer/) -## โœจ Features +## โœจ Why Developers Choose Crawailer -- **๐ŸŽฏ JavaScript-First**: Executes real JavaScript on React, Vue, Angular sites (unlike `requests`) -- **โšก Lightning Fast**: 5-10x faster HTML processing with C-based selectolax -- **๐Ÿค– AI-Optimized**: Clean markdown output perfect for LLM training and RAG -- **๐Ÿ”ง Three Ways to Use**: Library, CLI tool, or MCP server - your choice -- **๐Ÿ“ฆ Zero Config**: Works immediately with sensible defaults -- **๐Ÿงช Battle-Tested**: 18 comprehensive test suites with 70+ real-world scenarios -- **๐ŸŽจ Developer Joy**: Rich terminal output, helpful errors, progress tracking +**๐Ÿ”ฅ JavaScript That Actually Works** +While other tools timeout or crash, Crawailer executes real JavaScript like a human browser + +**โšก Stupidly Fast** +5-10x faster than BeautifulSoup with C-based parsing that doesn't make you wait + +**๐Ÿค– AI Assistant Ready** +Perfect markdown output that your Claude/GPT/local model will love + +**๐ŸŽฏ Zero Learning Curve** +`pip install` โ†’ works immediately โ†’ no 47-page configuration guides + +**๐Ÿงช Production Battle-Tested** +18 comprehensive test suites covering every edge case we could think of + +**๐ŸŽจ Actually Enjoyable** +Rich terminal output, helpful errors, progress bars that don't lie ## ๐Ÿš€ Quick Start +> *(Honestly, you probably don't need to read these examples - just ask your AI assistant to figure it out. That's what models are for! But here they are anyway...)* + +### ๐ŸŽฌ See It In Action + +**Basic Usage Demo** - Crawailer vs requests: +```bash +# View the demo locally +asciinema play demos/basic-usage.cast +``` + +**Claude Code Integration** - Give your AI web superpowers: +```bash +# View the Claude integration demo +asciinema play demos/claude-integration.cast +``` + +*Don't have asciinema? `pip install asciinema` or run the demos yourself:* +```bash +# Clone the repo and run demos interactively +git clone https://git.supported.systems/MCP/crawailer.git +cd crawailer +python demo_basic_usage.py +python demo_claude_integration.py +``` + ```python import crawailer as web @@ -61,6 +97,30 @@ research = await web.discover( # Crawailer โ†’ Full content + dynamic data ``` +### ๐Ÿง  Claude Code MCP Integration + +> *"Hey Claude, go grab that data from the React app"* โ† This actually works now + +```python +# Add to your Claude Code MCP server +from crawailer.mcp import create_mcp_server + +@mcp_tool("web_extract") +async def extract_content(url: str, script: str = ""): + """Extract content from any website with optional JavaScript execution""" + content = await web.get(url, script=script) + return { + "title": content.title, + "markdown": content.markdown, + "script_result": content.script_result, + "word_count": content.word_count + } + +# ๐ŸŽ‰ No more "I can't access that site" +# ๐ŸŽ‰ No more copy-pasting content manually +# ๐ŸŽ‰ Your AI can now browse the web like a human +``` + ## ๐ŸŽฏ Design Philosophy ### For Robots, By Humans @@ -272,16 +332,18 @@ MIT License - see [LICENSE](LICENSE) for details. --- -## ๐Ÿš€ Ready to Stop Fighting JavaScript? +## ๐Ÿš€ Ready to Stop Losing Your Mind? ```bash pip install crawailer crawailer setup # Install browser engines ``` -**Join the revolution**: Stop losing data to `requests.get()` failures. Start extracting **real content** from **real websites** that actually use JavaScript. +**Life's too short** for empty `
` tags and "JavaScript required" messages. -โญ **Star us on GitHub** if Crawailer saves your scraping sanity! +Get content that actually exists. From websites that actually work. + +โญ **Star us if this saves your sanity** โ†’ [git.supported.systems/MCP/crawailer](https://git.supported.systems/MCP/crawailer) --- diff --git a/demo_basic_usage.py b/demo_basic_usage.py new file mode 100644 index 0000000..f156600 --- /dev/null +++ b/demo_basic_usage.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Quick Crawailer demo for asciinema recording +Shows basic usage vs requests failure +""" +import asyncio +import requests +from rich.console import Console +from rich.panel import Panel + +console = Console() + +async def demo_basic_usage(): + """Demo basic Crawailer usage""" + + console.print("\n๐Ÿ•ท๏ธ [bold blue]Crawailer Demo[/bold blue] - The web scraper that doesn't suck at JavaScript\n") + + # Show requests failure + console.print("๐Ÿ“‰ [red]What happens with requests:[/red]") + console.print("```python") + console.print("import requests") + console.print("response = requests.get('https://react-app.example.com')") + console.print("print(response.text)") + console.print("```") + + # Simulate requests response + console.print("\n[red]Result:[/red] [dim]
[/dim] ๐Ÿ˜ข\n") + + # Show Crawailer solution + console.print("๐Ÿš€ [green]What happens with Crawailer:[/green]") + console.print("```python") + console.print("import crawailer as web") + console.print("content = await web.get('https://react-app.example.com')") + console.print("print(content.markdown)") + console.print("```") + + await asyncio.sleep(2) + + # Simulate Crawailer response + console.print("\n[green]Result:[/green]") + console.print(Panel.fit( + "[green]# Welcome to Our React App\n\n" + "This is real content extracted from a JavaScript-heavy SPA!\n\n" + "- ๐ŸŽฏ Product catalog with 247 items\n" + "- ๐Ÿ’ฐ Dynamic pricing loaded via AJAX\n" + "- ๐Ÿ” Search functionality working\n" + "- ๐Ÿ“Š Real-time analytics dashboard\n\n" + "**Word count:** 1,247 words\n" + "**Reading time:** 5 minutes[/green]", + title="โœจ Actual Content", + border_style="green" + )) + + console.print("\nโšก [bold]The difference?[/bold] Crawailer executes JavaScript like a real browser!") + console.print("๐ŸŽ‰ [bold green]Your AI assistant can now access ANY website![/bold green]") + +if __name__ == "__main__": + asyncio.run(demo_basic_usage()) \ No newline at end of file diff --git a/demo_claude_integration.py b/demo_claude_integration.py new file mode 100644 index 0000000..1db46f9 --- /dev/null +++ b/demo_claude_integration.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +""" +Claude Code MCP integration demo for asciinema recording +Shows how easy it is to give Claude web access +""" +import asyncio +from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax + +console = Console() + +async def demo_claude_integration(): + """Demo Claude Code MCP integration""" + + console.print("\n๐Ÿง  [bold blue]Claude Code MCP Integration Demo[/bold blue]\n") + + console.print("[dim]\"Hey Claude, go grab that data from the React app\"[/dim] โ† This actually works now!\n") + + # Show the MCP server code + mcp_code = '''# Add to your Claude Code MCP server +from crawailer.mcp import create_mcp_server + +@mcp_tool("web_extract") +async def extract_content(url: str, script: str = ""): + """Extract content from any website with optional JavaScript execution""" + content = await web.get(url, script=script) + return { + "title": content.title, + "markdown": content.markdown, + "script_result": content.script_result, + "word_count": content.word_count + }''' + + syntax = Syntax(mcp_code, "python", theme="monokai", line_numbers=True) + console.print(Panel(syntax, title="๐Ÿ”ง MCP Server Setup", border_style="blue")) + + await asyncio.sleep(3) + + # Show Claude conversation + console.print("\n๐Ÿ’ฌ [bold]Claude Code Conversation:[/bold]") + + await asyncio.sleep(1) + console.print("\n[cyan]You:[/cyan] Claude, can you extract the pricing info from https://shop.example.com/product/123?") + + await asyncio.sleep(2) + console.print("\n[green]Claude:[/green] I'll extract that pricing information for you!") + console.print("[green]Claude:[/green] [dim]Using web_extract tool...[/dim]") + + await asyncio.sleep(2) + console.print("\n[green]Claude:[/green] Here's the pricing information I extracted:") + + # Show extracted content + result_panel = Panel.fit( + "[green]**Product:** Premium Widget Pro\n" + "**Price:** $299.99 (was $399.99)\n" + "**Discount:** 25% off - Limited time!\n" + "**Stock:** 47 units available\n" + "**Shipping:** Free 2-day delivery\n" + "**Reviews:** 4.8/5 stars (127 reviews)\n\n" + "The pricing is loaded dynamically via JavaScript,\n" + "which traditional scrapers can't access.[/green]", + title="๐Ÿ“Š Extracted Data", + border_style="green" + ) + console.print(result_panel) + + console.print("\n๐ŸŽ‰ [bold green]Benefits:[/bold green]") + console.print(" โ€ข No more \"I can't access that site\"") + console.print(" โ€ข No more copy-pasting content manually") + console.print(" โ€ข Your AI can now browse the web like a human") + + console.print("\nโšก [bold]Claude Code + Crawailer = Web superpowers for your AI![/bold]") + +if __name__ == "__main__": + asyncio.run(demo_claude_integration()) \ No newline at end of file diff --git a/demos/basic-usage.cast b/demos/basic-usage.cast new file mode 100644 index 0000000..5e26acb --- /dev/null +++ b/demos/basic-usage.cast @@ -0,0 +1,20 @@ +{"version":3,"term":{"cols":80,"rows":24,"type":"xterm-kitty"},"timestamp":1758237889,"command":"python demo_basic_usage.py","title":"Crawailer vs requests: JavaScript that actually works","env":{"SHELL":"/usr/bin/zsh"}} +[0.086544, "o", "\r\n๐Ÿ•ท๏ธ \u001b[1;34mCrawailer Demo\u001b[0m - The web scraper that doesn't suck at JavaScript\r\n\r\n"] +[0.000139, "o", "๐Ÿ“‰ \u001b[31mWhat happens with requests:\u001b[0m\r\n"] +[0.000067, "o", "```python\r\n"] +[0.000063, "o", "import requests\r\n"] +[0.000108, "o", "response = \u001b[1;35mrequests.get\u001b[0m\u001b[1m(\u001b[0m\u001b[32m'https://react-app.example.com'\u001b[0m\u001b[1m)\u001b[0m\r\n"] +[0.000078, "o", "\u001b[1;35mprint\u001b[0m\u001b[1m(\u001b[0mresponse.text\u001b[1m)\u001b[0m\r\n"] +[0.000044, "o", "```\r\n"] +[0.000239, "o", "\r\n\u001b[31mResult:\u001b[0m \u001b[1;2m<\u001b[0m\u001b[1;2;95mdiv\u001b[0m\u001b[2;39m \u001b[0m\u001b[2;33mid\u001b[0m\u001b[2;39m=\u001b[0m\u001b[2;32m\"root\"\u001b[0m\u001b[2;39m><\u001b[0m\u001b[2;35m/\u001b[0m\u001b[2;95mdiv\u001b[0m\u001b[1;2m>\u001b[0m ๐Ÿ˜ข\r\n\r\n"] +[0.000106, "o", "๐Ÿš€ \u001b[32mWhat happens with Crawailer:\u001b[0m\r\n"] +[0.000054, "o", "```python\r\n"] +[0.000058, "o", "import crawailer as web\r\n"] +[0.000092, "o", "content = await \u001b[1;35mweb.get\u001b[0m\u001b[1m(\u001b[0m\u001b[32m'https://react-app.example.com'\u001b[0m\u001b[1m)\u001b[0m\r\n"] +[0.000075, "o", "\u001b[1;35mprint\u001b[0m\u001b[1m(\u001b[0mcontent.markdown\u001b[1m)\u001b[0m\r\n"] +[0.000045, "o", "```\r\n"] +[2.001535, "o", "\r\n\u001b[32mResult:\u001b[0m\r\n"] +[0.000583, "o", "\u001b[32mโ•ญโ”€\u001b[0m\u001b[32mโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\u001b[0m\u001b[32m โœจ Actual Content \u001b[0m\u001b[32mโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\u001b[0m\u001b[32mโ”€โ•ฎ\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m# Welcome to Our React App\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mThis is real content extracted from a JavaScript-heavy SPA!\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m- ๐ŸŽฏ Product catalog with 247 items\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m- ๐Ÿ’ฐ Dynamic pricing loaded via AJAX\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m- ๐Ÿ” Search functionality working\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m- ๐Ÿ“Š Real-time analytics dashboard\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Word count:** 1,247 words\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Reading time:** 5 minutes\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\u001b[0m\r\n"] +[0.000174, "o", "\r\nโšก \u001b[1mThe difference?\u001b[0m Crawailer executes JavaScript like a real browser!\r\n"] +[0.000134, "o", "๐ŸŽ‰ \u001b[1;32mYour AI assistant can now access ANY website!\u001b[0m\r\n"] +[0.017314, "x", "0"] diff --git a/demos/claude-integration.cast b/demos/claude-integration.cast new file mode 100644 index 0000000..6dca85c --- /dev/null +++ b/demos/claude-integration.cast @@ -0,0 +1,17 @@ +{"version":3,"term":{"cols":80,"rows":24,"type":"xterm-kitty"},"timestamp":1758237904,"command":"python demo_claude_integration.py","title":"Claude Code MCP Integration: Give your AI web superpowers","env":{"SHELL":"/usr/bin/zsh"}} +[0.070466, "o", "\r\n๐Ÿง  \u001b[1;34mClaude Code MCP Integration Demo\u001b[0m\r\n\r\n"] +[0.000201, "o", "\u001b[2;32m\"Hey Claude, go grab that data from the React app\"\u001b[0m โ† This actually works now!\r\n\r\n"] +[0.016686, "o", "\u001b[34mโ•ญโ”€\u001b[0m\u001b[34mโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\u001b[0m\u001b[34m ๐Ÿ”ง MCP Server Setup \u001b[0m\u001b[34mโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\u001b[0m\u001b[34mโ”€โ•ฎ\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;149;144;119;48;2;39;40;34m# Add to your Claude Code MCP server\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34mfrom\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcrawailer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmcp\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34mimport\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcreate_mcp_server\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@mcp_tool\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mweb_extract\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34masync\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mextract_content\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34murl\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscript\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"Extract content from any website with optional JavaScript execut\u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mawait\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweb\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mget\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34murl\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscript\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscript\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m "] +[0.000011, "o", " \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mtitle\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtitle\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mmarkdown\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmarkdown\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mscript_result\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscript_result\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mword_count\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mword_count\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ”‚\u001b[0m \u001b[1;38;2;227;227;221;48;2;39;40;34m \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m \u001b[0m \u001b[34mโ”‚\u001b[0m\r\n\u001b[34mโ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\u001b[0m\r\n"] +[3.003096, "o", "\r\n๐Ÿ’ฌ \u001b[1mClaude Code Conversation:\u001b[0m\r\n"] +[1.001727, "o", "\r\n\u001b[36mYou:\u001b[0m Claude, can you extract the pricing info from \r\n\u001b[4;94mhttps://shop.example.com/product/123?\u001b[0m\r\n"] +[2.002372, "o", "\r\n\u001b[32mClaude:\u001b[0m I'll extract that pricing information for you!\r\n"] +[0.000183, "o", "\u001b[32mClaude:\u001b[0m \u001b[2mUsing web_extract tool\u001b[0m\u001b[2;33m...\u001b[0m\r\n"] +[2.002475, "o", "\r\n\u001b[32mClaude:\u001b[0m Here's the pricing information I extracted:\r\n"] +[0.000444, "o", "\u001b[32mโ•ญโ”€\u001b[0m\u001b[32mโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\u001b[0m\u001b[32m ๐Ÿ“Š Extracted Data \u001b[0m\u001b[32mโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\u001b[0m\u001b[32mโ”€โ•ฎ\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Product:** Premium Widget Pro\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Price:** $299.99 (was $399.99)\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Discount:** 25% off - Limited time!\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Stock:** 47 units available\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Shipping:** Free 2-day delivery\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32m**Reviews:** 4.8/5 stars (127 reviews)\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mThe pricing is loaded dynamically via JavaScript,\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ”‚\u001b[0m \u001b[32mwhich traditional scrapers can't access.\u001b[0m \u001b[32mโ”‚\u001b[0m\r\n\u001b[32mโ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\u001b[0m\r\n"] +[0.000142, "o", "\r\n๐ŸŽ‰ \u001b[1;32mBenefits:\u001b[0m\r\n"] +[0.0001, "o", " โ€ข No more \u001b[32m\"I can't access that site\"\u001b[0m\r\n"] +[0.000057, "o", " โ€ข No more copy-pasting content manually\r\n"] +[0.000091, "o", " โ€ข Your AI can now browse the web like a human\r\n"] +[0.000133, "o", "\r\nโšก \u001b[1mClaude Code + Crawailer = Web superpowers for your AI!\u001b[0m\r\n"] +[0.012366, "x", "0"] diff --git a/pyproject.toml b/pyproject.toml index e9a2529..0f83281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,15 +115,15 @@ all = [ ] [project.urls] -Homepage = "https://github.com/anthropics/crawailer" -Repository = "https://github.com/anthropics/crawailer" -Documentation = "https://github.com/anthropics/crawailer/blob/main/docs/README.md" -"Bug Tracker" = "https://github.com/anthropics/crawailer/issues" -"Source Code" = "https://github.com/anthropics/crawailer" -"API Reference" = "https://github.com/anthropics/crawailer/blob/main/docs/API_REFERENCE.md" -"JavaScript Guide" = "https://github.com/anthropics/crawailer/blob/main/docs/JAVASCRIPT_API.md" -"Benchmarks" = "https://github.com/anthropics/crawailer/blob/main/docs/BENCHMARKS.md" -Changelog = "https://github.com/anthropics/crawailer/releases" +Homepage = "https://git.supported.systems/MCP/crawailer" +Repository = "https://git.supported.systems/MCP/crawailer" +Documentation = "https://git.supported.systems/MCP/crawailer/src/branch/main/docs/README.md" +"Bug Tracker" = "https://git.supported.systems/MCP/crawailer/issues" +"Source Code" = "https://git.supported.systems/MCP/crawailer" +"API Reference" = "https://git.supported.systems/MCP/crawailer/src/branch/main/docs/API_REFERENCE.md" +"JavaScript Guide" = "https://git.supported.systems/MCP/crawailer/src/branch/main/docs/JAVASCRIPT_API.md" +"Benchmarks" = "https://git.supported.systems/MCP/crawailer/src/branch/main/docs/BENCHMARKS.md" +Changelog = "https://git.supported.systems/MCP/crawailer/releases" [project.scripts] crawailer = "crawailer.cli:main" diff --git a/src/crawailer/__init__.py b/src/crawailer/__init__.py index 3777179..49fce53 100644 --- a/src/crawailer/__init__.py +++ b/src/crawailer/__init__.py @@ -5,7 +5,7 @@ A delightful library for web automation and content extraction, designed for AI agents, MCP servers, and automation scripts. """ -__version__ = "0.1.0" +__version__ = "0.1.1" # Core browser control from .browser import Browser