Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

increase timeout settings for interactions in browser service to accommodate slow SPAs and heavy pages, ensuring more reliable page loading and interaction handling
Build and Push Docker Images / build-and-push (push) Successful in 1m12s
Details

Browse Source
This commit is contained in:
Bram
2026-05-04 15:02:55 +02:00
parent e182f87cac
commit d96ea7ce29
1 changed files with 57 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+57 -20
View File
@@ -9,6 +9,10 @@ from app.config import CUSTOM_USER_AGENT
active_pages = set() active_pages = set()
page_creation_times = {} page_creation_times = {}
# Interactions endpoint: allow slow SPAs / heavy pages (Playwright + overall asyncio cap).
INTERACTIONS_TIMEOUT_MS = 120_000
INTERACTIONS_OPERATION_TIMEOUT_SECONDS = 120.0
async def visit_url_service(decoded_url): async def visit_url_service(decoded_url):
"""Service function to visit a URL and get its content""" """Service function to visit a URL and get its content"""
print(f"Visiting URL: {decoded_url}") print(f"Visiting URL: {decoded_url}")
@@ -241,11 +245,15 @@ async def interactions_service(
print(f"Error setting forced viewport width: {e}") print(f"Error setting forced viewport width: {e}")
# Navigate and wait for initial DOM. # Navigate and wait for initial DOM.
await page.goto(decoded_url, wait_until="domcontentloaded", timeout=30000) await page.goto(
await page.wait_for_load_state("load", timeout=30000) decoded_url, wait_until="domcontentloaded", timeout=INTERACTIONS_TIMEOUT_MS
)
await page.wait_for_load_state("load", timeout=INTERACTIONS_TIMEOUT_MS)
# Ensure the page is fully loaded (helps with JS-driven UIs). # Ensure the page is fully loaded (helps with JS-driven UIs).
await page.wait_for_function("document.readyState === 'complete'", timeout=30000) await page.wait_for_function(
"document.readyState === 'complete'", timeout=INTERACTIONS_TIMEOUT_MS
)
# Best-effort cookie/banner handling (helps avoid click interception). # Best-effort cookie/banner handling (helps avoid click interception).
cookie_buttons = [ cookie_buttons = [
@@ -307,7 +315,9 @@ async def interactions_service(
if not action: if not action:
raise ValueError("Interaction must contain an action") raise ValueError("Interaction must contain an action")
async def wait_for_xpath(xp: str, timeout_ms: int = 30000) -> None: async def wait_for_xpath(
xp: str, timeout_ms: int = INTERACTIONS_TIMEOUT_MS
) -> None:
# Accept both raw XPath ("//div") and Playwright-style ("xpath=//div"). # Accept both raw XPath ("//div") and Playwright-style ("xpath=//div").
normalized = xp[len("xpath=") :] if xp.startswith("xpath=") else xp normalized = xp[len("xpath=") :] if xp.startswith("xpath=") else xp
await page.wait_for_function( await page.wait_for_function(
@@ -332,7 +342,9 @@ async def interactions_service(
if selector or xpath_selector: if selector or xpath_selector:
if action == "click": if action == "click":
if selector: if selector:
await page.wait_for_selector(selector, timeout=30000) await page.wait_for_selector(
selector, timeout=INTERACTIONS_TIMEOUT_MS
)
# Avoid Playwright auto-scrolling: click via DOM. # Avoid Playwright auto-scrolling: click via DOM.
await page.evaluate( await page.evaluate(
"""(sel) => { """(sel) => {
@@ -345,18 +357,22 @@ async def interactions_service(
else: else:
if not xpath_selector: if not xpath_selector:
raise ValueError("xpath_selector must be a non-empty string") raise ValueError("xpath_selector must be a non-empty string")
await wait_for_xpath(xpath_selector, timeout_ms=30000) await wait_for_xpath(
xpath_selector, timeout_ms=INTERACTIONS_TIMEOUT_MS
)
normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector
# Use Playwright's click to generate real pointer/mouse events. # Use Playwright's click to generate real pointer/mouse events.
locator = page.locator(f"xpath={normalized_xpath}").first locator = page.locator(f"xpath={normalized_xpath}").first
await locator.wait_for(timeout=30000) await locator.wait_for(timeout=INTERACTIONS_TIMEOUT_MS)
await locator.click(force=True, timeout=30000) await locator.click(force=True, timeout=INTERACTIONS_TIMEOUT_MS)
elif action == "type": elif action == "type":
if text is None: if text is None:
raise ValueError("Type interaction must contain text") raise ValueError("Type interaction must contain text")
if selector: if selector:
await page.wait_for_selector(selector, timeout=30000) await page.wait_for_selector(
selector, timeout=INTERACTIONS_TIMEOUT_MS
)
# Avoid Playwright auto-scrolling: focus/fill via DOM. # Avoid Playwright auto-scrolling: focus/fill via DOM.
await page.evaluate( await page.evaluate(
"""({ sel, value }) => { """({ sel, value }) => {
@@ -374,13 +390,15 @@ async def interactions_service(
else: else:
if not xpath_selector: if not xpath_selector:
raise ValueError("xpath_selector must be a non-empty string") raise ValueError("xpath_selector must be a non-empty string")
await wait_for_xpath(xpath_selector, timeout_ms=30000) await wait_for_xpath(
xpath_selector, timeout_ms=INTERACTIONS_TIMEOUT_MS
)
normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector
locator = page.locator(f"xpath={normalized_xpath}").first locator = page.locator(f"xpath={normalized_xpath}").first
await locator.wait_for(timeout=30000) await locator.wait_for(timeout=INTERACTIONS_TIMEOUT_MS)
# Prefer fill/type so frameworks see real input events. # Prefer fill/type so frameworks see real input events.
await locator.click(force=True, timeout=30000) await locator.click(force=True, timeout=INTERACTIONS_TIMEOUT_MS)
await locator.fill(text, timeout=30000) await locator.fill(text, timeout=INTERACTIONS_TIMEOUT_MS)
else: else:
raise ValueError(f"Unknown interaction action: {action}") raise ValueError(f"Unknown interaction action: {action}")
@@ -393,7 +411,9 @@ async def interactions_service(
vx, vy = await resolve_viewport_point(x, y) vx, vy = await resolve_viewport_point(x, y)
element, handle = await element_from_point_retry(vx, vy) element, handle = await element_from_point_retry(vx, vy)
try: try:
await element.click(force=True, timeout=30000) await element.click(
force=True, timeout=INTERACTIONS_TIMEOUT_MS
)
finally: finally:
try: try:
await handle.dispose() await handle.dispose()
@@ -409,10 +429,12 @@ async def interactions_service(
vx, vy = await resolve_viewport_point(x, y) vx, vy = await resolve_viewport_point(x, y)
element, handle = await element_from_point_retry(vx, vy) element, handle = await element_from_point_retry(vx, vy)
try: try:
await element.click(force=True, timeout=30000) await element.click(
force=True, timeout=INTERACTIONS_TIMEOUT_MS
)
# Ensure the click actually put focus on the intended control. # Ensure the click actually put focus on the intended control.
try: try:
await element.focus(timeout=30000) await element.focus(timeout=INTERACTIONS_TIMEOUT_MS)
except Exception: except Exception:
pass pass
await page.wait_for_timeout(100) await page.wait_for_timeout(100)
@@ -455,7 +477,12 @@ async def interactions_service(
print(f"Error during interactions capture: {e}") print(f"Error during interactions capture: {e}")
return {"status": "error", "url": decoded_url, "error": str(e)} return {"status": "error", "url": decoded_url, "error": str(e)}
return await safe_browser_operation(decoded_url, interactions_operation) return await safe_browser_operation(
decoded_url,
interactions_operation,
page_timeout_ms=INTERACTIONS_TIMEOUT_MS,
operation_timeout_seconds=INTERACTIONS_OPERATION_TIMEOUT_SECONDS,
)
async def extract_seo_service(decoded_url): async def extract_seo_service(decoded_url):
"""Service function to extract SEO information from a website""" """Service function to extract SEO information from a website"""
@@ -740,7 +767,13 @@ async def get_resulting_url_service(decoded_url):
# Perform the operation # Perform the operation
return await safe_browser_operation(decoded_url, resulting_url_operation) return await safe_browser_operation(decoded_url, resulting_url_operation)
async def safe_browser_operation(url, operation_func): async def safe_browser_operation(
url,
operation_func,
*,
page_timeout_ms: int = 30_000,
operation_timeout_seconds: float = 60.0,
):
"""Safely perform browser operations with proper cleanup and timeouts""" """Safely perform browser operations with proper cleanup and timeouts"""
browser = None browser = None
context = None context = None
@@ -767,14 +800,18 @@ async def safe_browser_operation(url, operation_func):
) )
page = await asyncio.wait_for(context.new_page(), timeout=10.0) page = await asyncio.wait_for(context.new_page(), timeout=10.0)
page.set_default_timeout(30000) # 30 second timeout page.set_default_timeout(page_timeout_ms)
page.set_default_navigation_timeout(page_timeout_ms)
# Track page creation time for force cleanup # Track page creation time for force cleanup
page_creation_times[page] = time.time() page_creation_times[page] = time.time()
active_pages.add(page) active_pages.add(page)
# Call the operation function that uses the page with timeout # Call the operation function that uses the page with timeout
result = await asyncio.wait_for(operation_func(page), timeout=60.0) result = await asyncio.wait_for(
operation_func(page),
timeout=operation_timeout_seconds,
)
return result return result