Initial commit: JavaScript API enhancement preparation
- Comprehensive test suite (700+ lines) for JS execution in high-level API - Test coverage analysis and validation infrastructure - Enhancement proposal and implementation strategy - Mock HTTP server with realistic JavaScript scenarios - Parallel implementation strategy using expert agents and git worktrees Ready for test-driven implementation of JavaScript enhancements.
This commit is contained in:
Crawailer Developer
commit
7634f9fc32
26 changed files with 9261 additions and 0 deletions
173
test_runner.py
Normal file
173
test_runner.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Simple test runner to validate our JavaScript API tests without external dependencies."""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
||||
|
||||
# Mock playwright before any imports
|
||||
sys.modules['playwright'] = MagicMock()
|
||||
sys.modules['playwright.async_api'] = MagicMock()
|
||||
|
||||
async def test_mock_server():
|
||||
"""Test our mock HTTP server functionality."""
|
||||
print("🧪 Testing Mock HTTP Server...")
|
||||
|
||||
# Import our mock server
|
||||
from tests.test_javascript_api import MockHTTPServer
|
||||
|
||||
server = MockHTTPServer()
|
||||
base_url = await server.start()
|
||||
print(f"✅ Mock server started at {base_url}")
|
||||
|
||||
# Test the server endpoints
|
||||
import aiohttp
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Test index page
|
||||
async with session.get(f"{base_url}/") as resp:
|
||||
text = await resp.text()
|
||||
assert "Test Page" in text
|
||||
print("✅ Index page works")
|
||||
|
||||
# Test dynamic price page
|
||||
async with session.get(f"{base_url}/dynamic-price") as resp:
|
||||
text = await resp.text()
|
||||
assert "Amazing Product" in text
|
||||
assert "price-container" in text
|
||||
print("✅ Dynamic price page works")
|
||||
|
||||
# Test infinite scroll page
|
||||
async with session.get(f"{base_url}/infinite-scroll") as resp:
|
||||
text = await resp.text()
|
||||
assert "Infinite Content" in text
|
||||
assert "loadMore" in text
|
||||
print("✅ Infinite scroll page works")
|
||||
|
||||
await server.stop()
|
||||
print("✅ Mock server stopped cleanly")
|
||||
|
||||
def test_webcontent_enhancements():
|
||||
"""Test WebContent with JavaScript fields."""
|
||||
print("🧪 Testing WebContent JavaScript enhancements...")
|
||||
|
||||
# We need to mock the WebContent class since we can't import it
|
||||
# But we can test the concept
|
||||
|
||||
class MockWebContent:
|
||||
def __init__(self, url, title, text, markdown, html, script_result=None, script_error=None):
|
||||
self.url = url
|
||||
self.title = title
|
||||
self.text = text
|
||||
self.markdown = markdown
|
||||
self.html = html
|
||||
self.script_result = script_result
|
||||
self.script_error = script_error
|
||||
|
||||
# Test with script result
|
||||
content = MockWebContent(
|
||||
url="https://example.com",
|
||||
title="Test",
|
||||
text="Content",
|
||||
markdown="# Test",
|
||||
html="<html></html>",
|
||||
script_result={"data": "value"}
|
||||
)
|
||||
|
||||
assert content.script_result == {"data": "value"}
|
||||
assert content.script_error is None
|
||||
print("✅ WebContent with script_result works")
|
||||
|
||||
# Test with script error
|
||||
content_error = MockWebContent(
|
||||
url="https://example.com",
|
||||
title="Test",
|
||||
text="Content",
|
||||
markdown="# Test",
|
||||
html="<html></html>",
|
||||
script_error="ReferenceError: x is not defined"
|
||||
)
|
||||
|
||||
assert content_error.script_result is None
|
||||
assert "ReferenceError" in content_error.script_error
|
||||
print("✅ WebContent with script_error works")
|
||||
|
||||
def test_api_signatures():
|
||||
"""Test that our proposed API signatures make sense."""
|
||||
print("🧪 Testing proposed API signatures...")
|
||||
|
||||
# Test function signature compatibility
|
||||
def mock_get(url, *, wait_for=None, script=None, script_before=None,
|
||||
script_after=None, timeout=30, clean=True,
|
||||
extract_links=True, extract_metadata=True):
|
||||
return {
|
||||
'url': url,
|
||||
'wait_for': wait_for,
|
||||
'script': script,
|
||||
'script_before': script_before,
|
||||
'script_after': script_after,
|
||||
'timeout': timeout
|
||||
}
|
||||
|
||||
# Test basic call
|
||||
result = mock_get("https://example.com")
|
||||
assert result['url'] == "https://example.com"
|
||||
assert result['script'] is None
|
||||
print("✅ Basic get() signature works")
|
||||
|
||||
# Test with script
|
||||
result = mock_get("https://example.com", script="return document.title")
|
||||
assert result['script'] == "return document.title"
|
||||
print("✅ get() with script parameter works")
|
||||
|
||||
# Test with script_before/after
|
||||
result = mock_get("https://example.com",
|
||||
script_before="window.scrollTo(0, document.body.scrollHeight)",
|
||||
script_after="return window.scrollY")
|
||||
assert result['script_before'] is not None
|
||||
assert result['script_after'] is not None
|
||||
print("✅ get() with script_before/script_after works")
|
||||
|
||||
async def main():
|
||||
"""Run all our validation tests."""
|
||||
print("🚀 Starting JavaScript API Enhancement Tests\n")
|
||||
|
||||
try:
|
||||
# Test mock server
|
||||
await test_mock_server()
|
||||
print()
|
||||
|
||||
# Test WebContent enhancements
|
||||
test_webcontent_enhancements()
|
||||
print()
|
||||
|
||||
# Test API signatures
|
||||
test_api_signatures()
|
||||
print()
|
||||
|
||||
print("🎉 All validation tests passed!")
|
||||
print("\n📋 Test Summary:")
|
||||
print(" ✅ Mock HTTP server with JavaScript scenarios")
|
||||
print(" ✅ WebContent enhancements for script results")
|
||||
print(" ✅ Proposed API signatures are valid")
|
||||
print(" ✅ Error handling patterns work")
|
||||
|
||||
print("\n🔍 Next Steps:")
|
||||
print(" 1. Install Playwright browsers: crawailer setup")
|
||||
print(" 2. Implement JavaScript execution in api.py")
|
||||
print(" 3. Update Browser.fetch_page() for script execution")
|
||||
print(" 4. Add script_result/script_error to WebContent")
|
||||
print(" 5. Run full test suite: pytest tests/test_javascript_api.py")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Test failed: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
Loading…
Add table
Add a link
Reference in a new issue