Complete Phase 2: Browser JavaScript integration with script_before/script_after support
This commit is contained in:
Crawailer Developer
parent
05df964ce1
commit
e544086e6b
5 changed files with 247 additions and 9 deletions
|
|
@ -869,6 +869,179 @@ class TestBrowserJavaScriptExecution:
|
|||
timeout=1
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_execute_script_basic(self):
|
||||
"""Test basic script execution (alias for compatibility)."""
|
||||
await self.test_execute_script_basic()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_execute_script_error(self):
|
||||
"""Test script execution error handling (alias for compatibility)."""
|
||||
await self.test_execute_script_error()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_script_timeout(self):
|
||||
"""Test script execution timeout (alias for compatibility)."""
|
||||
await self.test_execute_script_timeout()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_fetch_page_with_scripts(self):
|
||||
"""Test fetch_page with script_before and script_after parameters."""
|
||||
browser = Browser(BrowserConfig())
|
||||
|
||||
# Mock Playwright components
|
||||
mock_page = AsyncMock()
|
||||
mock_page.goto = AsyncMock()
|
||||
mock_page.set_viewport_size = AsyncMock()
|
||||
mock_page.content.return_value = "<html><body><h1>Test</h1></body></html>"
|
||||
mock_page.title.return_value = "Test Page"
|
||||
mock_page.close = AsyncMock()
|
||||
|
||||
# Mock script execution results
|
||||
script_calls = []
|
||||
def mock_evaluate(script):
|
||||
script_calls.append(script)
|
||||
if "before" in script:
|
||||
return {"before_result": "success"}
|
||||
elif "after" in script:
|
||||
return {"after_result": "complete"}
|
||||
return None
|
||||
|
||||
mock_page.evaluate.side_effect = mock_evaluate
|
||||
|
||||
mock_browser = AsyncMock()
|
||||
mock_browser.new_page.return_value = mock_page
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_page.goto.return_value = mock_response
|
||||
|
||||
browser._browser = mock_browser
|
||||
browser._is_started = True
|
||||
|
||||
# Test with both script_before and script_after
|
||||
result = await browser.fetch_page(
|
||||
"https://example.com",
|
||||
script_before="return {before: true}",
|
||||
script_after="return {after: true}"
|
||||
)
|
||||
|
||||
# Verify the result structure
|
||||
assert result["url"] == "https://example.com"
|
||||
assert result["status"] == 200
|
||||
assert result["html"] == "<html><body><h1>Test</h1></body></html>"
|
||||
assert result["title"] == "Test Page"
|
||||
assert "script_result" in result
|
||||
assert "script_error" in result
|
||||
|
||||
# Script result should contain both before and after results
|
||||
assert result["script_result"] == {
|
||||
"script_before": {"before_result": "success"},
|
||||
"script_after": {"after_result": "complete"}
|
||||
}
|
||||
assert result["script_error"] is None
|
||||
|
||||
# Verify script execution order (before content extraction, after content extraction)
|
||||
assert len(script_calls) == 2
|
||||
mock_page.evaluate.assert_any_call("return {before: true}")
|
||||
mock_page.evaluate.assert_any_call("return {after: true}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_fetch_page_script_before_only(self):
|
||||
"""Test fetch_page with only script_before parameter."""
|
||||
browser = Browser(BrowserConfig())
|
||||
|
||||
# Mock setup
|
||||
mock_page = AsyncMock()
|
||||
mock_page.goto = AsyncMock()
|
||||
mock_page.set_viewport_size = AsyncMock()
|
||||
mock_page.content.return_value = "<html><body><h1>Test</h1></body></html>"
|
||||
mock_page.title.return_value = "Test Page"
|
||||
mock_page.evaluate.return_value = {"data": "extracted"}
|
||||
mock_page.close = AsyncMock()
|
||||
|
||||
mock_browser = AsyncMock()
|
||||
mock_browser.new_page.return_value = mock_page
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_page.goto.return_value = mock_response
|
||||
|
||||
browser._browser = mock_browser
|
||||
browser._is_started = True
|
||||
|
||||
result = await browser.fetch_page(
|
||||
"https://example.com",
|
||||
script_before="return document.querySelector('h1').innerText"
|
||||
)
|
||||
|
||||
assert result["script_result"] == {"data": "extracted"}
|
||||
assert result["script_error"] is None
|
||||
mock_page.evaluate.assert_called_once_with("return document.querySelector('h1').innerText")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_fetch_page_script_error_handling(self):
|
||||
"""Test fetch_page script error handling."""
|
||||
browser = Browser(BrowserConfig())
|
||||
|
||||
# Mock setup
|
||||
mock_page = AsyncMock()
|
||||
mock_page.goto = AsyncMock()
|
||||
mock_page.set_viewport_size = AsyncMock()
|
||||
mock_page.content.return_value = "<html><body><h1>Test</h1></body></html>"
|
||||
mock_page.title.return_value = "Test Page"
|
||||
mock_page.evaluate.side_effect = Exception("Script syntax error")
|
||||
mock_page.close = AsyncMock()
|
||||
|
||||
mock_browser = AsyncMock()
|
||||
mock_browser.new_page.return_value = mock_page
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_page.goto.return_value = mock_response
|
||||
|
||||
browser._browser = mock_browser
|
||||
browser._is_started = True
|
||||
|
||||
result = await browser.fetch_page(
|
||||
"https://example.com",
|
||||
script_before="invalid javascript syntax %@#$"
|
||||
)
|
||||
|
||||
assert result["script_result"] is None
|
||||
assert "Script execution error: Script syntax error" in result["script_error"]
|
||||
# Page should still load successfully
|
||||
assert result["status"] == 200
|
||||
assert result["html"] == "<html><body><h1>Test</h1></body></html>"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_fetch_page_page_load_error_with_scripts(self):
|
||||
"""Test fetch_page when page load fails but scripts were requested."""
|
||||
browser = Browser(BrowserConfig())
|
||||
|
||||
# Mock setup
|
||||
mock_page = AsyncMock()
|
||||
mock_page.goto.side_effect = Exception("Network error")
|
||||
mock_page.set_viewport_size = AsyncMock()
|
||||
mock_page.close = AsyncMock()
|
||||
|
||||
mock_browser = AsyncMock()
|
||||
mock_browser.new_page.return_value = mock_page
|
||||
|
||||
browser._browser = mock_browser
|
||||
browser._is_started = True
|
||||
|
||||
result = await browser.fetch_page(
|
||||
"https://unreachable-site.com",
|
||||
script_before="return true"
|
||||
)
|
||||
|
||||
# Should handle the error gracefully
|
||||
assert result["status"] == 0
|
||||
assert result["error"] == "Network error"
|
||||
assert result["script_result"] is None
|
||||
assert "Page load failed, scripts not executed: Network error" in result["script_error"]
|
||||
|
||||
|
||||
# Test utilities and integration
|
||||
class TestJavaScriptIntegration:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue