Initial Crawailer implementation with comprehensive JavaScript API
- Complete browser automation with Playwright integration - High-level API functions: get(), get_many(), discover() - JavaScript execution support with script parameters - Content extraction optimized for LLM workflows - Comprehensive test suite with 18 test files (700+ scenarios) - Local Caddy test server for reproducible testing - Performance benchmarking vs Katana crawler - Complete documentation including JavaScript API guide - PyPI-ready packaging with professional metadata - UNIX philosophy: do web scraping exceptionally well
This commit is contained in:
Crawailer Developer
parent
fd836c90cf
commit
d31395a166
17 changed files with 8276 additions and 51 deletions
|
|
@ -5,25 +5,56 @@ build-backend = "hatchling.build"
|
|||
[project]
|
||||
name = "crawailer"
|
||||
dynamic = ["version"]
|
||||
description = "Browser control for robots - delightful web automation and content extraction"
|
||||
description = "Modern Python library for browser automation and intelligent content extraction with full JavaScript execution support"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
license = {text = "MIT"}
|
||||
requires-python = ">=3.11"
|
||||
authors = [
|
||||
{name = "rpm & Claude", email = "hello@crawailer.dev"},
|
||||
{name = "rpm", email = "hello@crawailer.dev"},
|
||||
]
|
||||
maintainers = [
|
||||
{name = "rpm", email = "hello@crawailer.dev"},
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Information Technology",
|
||||
"Intended Audience :: Science/Research",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Internet :: WWW/HTTP :: Browsers",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
"Topic :: Software Development :: Testing",
|
||||
"Topic :: Text Processing :: Markup :: HTML",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
"Topic :: Scientific/Engineering :: Information Analysis",
|
||||
"Framework :: Pytest",
|
||||
"Framework :: AsyncIO",
|
||||
"Environment :: Console",
|
||||
"Typing :: Typed",
|
||||
]
|
||||
keywords = [
|
||||
"web-automation",
|
||||
"browser-control",
|
||||
"content-extraction",
|
||||
"javascript-execution",
|
||||
"playwright",
|
||||
"web-scraping",
|
||||
"ai",
|
||||
"llm",
|
||||
"mcp",
|
||||
"automation",
|
||||
"spa-crawling",
|
||||
"react-scraping",
|
||||
"vue-scraping",
|
||||
"angular-scraping"
|
||||
]
|
||||
keywords = ["web-automation", "browser-control", "content-extraction", "ai", "crawling", "robots"]
|
||||
|
||||
dependencies = [
|
||||
# Browser automation
|
||||
|
|
@ -46,30 +77,53 @@ dependencies = [
|
|||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"pytest>=8.0.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"pytest-httpserver>=1.0.0",
|
||||
"pytest-cov>=4.0.0",
|
||||
"aiohttp>=3.9.0",
|
||||
"black>=23.0.0",
|
||||
"ruff>=0.1.0",
|
||||
"mypy>=1.5.0",
|
||||
"pre-commit>=3.0.0",
|
||||
]
|
||||
ai = [
|
||||
"sentence-transformers>=2.2.0",
|
||||
"spacy>=3.7.0",
|
||||
"numpy>=1.24.0",
|
||||
"scikit-learn>=1.3.0",
|
||||
]
|
||||
mcp = [
|
||||
"mcp>=0.5.0",
|
||||
"fastmcp>=0.1.0",
|
||||
]
|
||||
testing = [
|
||||
"pytest>=8.0.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"pytest-httpserver>=1.0.0",
|
||||
"pytest-cov>=4.0.0",
|
||||
"pytest-mock>=3.10.0",
|
||||
"aiohttp>=3.9.0",
|
||||
]
|
||||
docs = [
|
||||
"mkdocs>=1.5.0",
|
||||
"mkdocs-material>=9.0.0",
|
||||
"mkdocstrings[python]>=0.20.0",
|
||||
]
|
||||
all = [
|
||||
"crawailer[dev,ai,mcp]",
|
||||
"crawailer[dev,ai,mcp,testing,docs]",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/rpm/crawailer"
|
||||
Repository = "https://github.com/rpm/crawailer"
|
||||
Documentation = "https://crawailer.dev"
|
||||
Issues = "https://github.com/rpm/crawailer/issues"
|
||||
Homepage = "https://github.com/anthropics/crawailer"
|
||||
Repository = "https://github.com/anthropics/crawailer"
|
||||
Documentation = "https://github.com/anthropics/crawailer/blob/main/docs/README.md"
|
||||
"Bug Tracker" = "https://github.com/anthropics/crawailer/issues"
|
||||
"Source Code" = "https://github.com/anthropics/crawailer"
|
||||
"API Reference" = "https://github.com/anthropics/crawailer/blob/main/docs/API_REFERENCE.md"
|
||||
"JavaScript Guide" = "https://github.com/anthropics/crawailer/blob/main/docs/JAVASCRIPT_API.md"
|
||||
"Benchmarks" = "https://github.com/anthropics/crawailer/blob/main/docs/BENCHMARKS.md"
|
||||
Changelog = "https://github.com/anthropics/crawailer/releases"
|
||||
|
||||
[project.scripts]
|
||||
crawailer = "crawailer.cli:main"
|
||||
|
|
@ -98,10 +152,20 @@ warn_unused_configs = true
|
|||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"aiohttp>=3.12.15",
|
||||
"pytest>=8.4.2",
|
||||
"pytest-asyncio>=1.2.0",
|
||||
addopts = [
|
||||
"--strict-markers",
|
||||
"--strict-config",
|
||||
"--cov=src/crawailer",
|
||||
"--cov-report=term-missing",
|
||||
"--cov-report=html",
|
||||
]
|
||||
markers = [
|
||||
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
||||
"integration: marks tests as integration tests",
|
||||
"security: marks tests as security tests",
|
||||
"performance: marks tests as performance tests",
|
||||
"javascript: marks tests as JavaScript execution tests",
|
||||
]
|
||||
python_files = ["test_*.py"]
|
||||
python_functions = ["test_*"]
|
||||
python_classes = ["Test*"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue