From d96ea7ce29173d706c68b21284fa405cecc419e0 Mon Sep 17 00:00:00 2001 From: Bram Date: Mon, 4 May 2026 15:02:55 +0200 Subject: [PATCH] increase timeout settings for interactions in browser service to accommodate slow SPAs and heavy pages, ensuring more reliable page loading and interaction handling --- Dockers/puppeteer-api/app/services/browser.py | 77 ++++++++++++++----- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index af1583e..3ad5a12 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -9,6 +9,10 @@ from app.config import CUSTOM_USER_AGENT active_pages = set() page_creation_times = {} +# Interactions endpoint: allow slow SPAs / heavy pages (Playwright + overall asyncio cap). +INTERACTIONS_TIMEOUT_MS = 120_000 +INTERACTIONS_OPERATION_TIMEOUT_SECONDS = 120.0 + async def visit_url_service(decoded_url): """Service function to visit a URL and get its content""" print(f"Visiting URL: {decoded_url}") @@ -241,11 +245,15 @@ async def interactions_service( print(f"Error setting forced viewport width: {e}") # Navigate and wait for initial DOM. - await page.goto(decoded_url, wait_until="domcontentloaded", timeout=30000) - await page.wait_for_load_state("load", timeout=30000) + await page.goto( + decoded_url, wait_until="domcontentloaded", timeout=INTERACTIONS_TIMEOUT_MS + ) + await page.wait_for_load_state("load", timeout=INTERACTIONS_TIMEOUT_MS) # Ensure the page is fully loaded (helps with JS-driven UIs). - await page.wait_for_function("document.readyState === 'complete'", timeout=30000) + await page.wait_for_function( + "document.readyState === 'complete'", timeout=INTERACTIONS_TIMEOUT_MS + ) # Best-effort cookie/banner handling (helps avoid click interception). cookie_buttons = [ @@ -307,7 +315,9 @@ async def interactions_service( if not action: raise ValueError("Interaction must contain an action") - async def wait_for_xpath(xp: str, timeout_ms: int = 30000) -> None: + async def wait_for_xpath( + xp: str, timeout_ms: int = INTERACTIONS_TIMEOUT_MS + ) -> None: # Accept both raw XPath ("//div") and Playwright-style ("xpath=//div"). normalized = xp[len("xpath=") :] if xp.startswith("xpath=") else xp await page.wait_for_function( @@ -332,7 +342,9 @@ async def interactions_service( if selector or xpath_selector: if action == "click": if selector: - await page.wait_for_selector(selector, timeout=30000) + await page.wait_for_selector( + selector, timeout=INTERACTIONS_TIMEOUT_MS + ) # Avoid Playwright auto-scrolling: click via DOM. await page.evaluate( """(sel) => { @@ -345,18 +357,22 @@ async def interactions_service( else: if not xpath_selector: raise ValueError("xpath_selector must be a non-empty string") - await wait_for_xpath(xpath_selector, timeout_ms=30000) + await wait_for_xpath( + xpath_selector, timeout_ms=INTERACTIONS_TIMEOUT_MS + ) normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector # Use Playwright's click to generate real pointer/mouse events. locator = page.locator(f"xpath={normalized_xpath}").first - await locator.wait_for(timeout=30000) - await locator.click(force=True, timeout=30000) + await locator.wait_for(timeout=INTERACTIONS_TIMEOUT_MS) + await locator.click(force=True, timeout=INTERACTIONS_TIMEOUT_MS) elif action == "type": if text is None: raise ValueError("Type interaction must contain text") if selector: - await page.wait_for_selector(selector, timeout=30000) + await page.wait_for_selector( + selector, timeout=INTERACTIONS_TIMEOUT_MS + ) # Avoid Playwright auto-scrolling: focus/fill via DOM. await page.evaluate( """({ sel, value }) => { @@ -374,13 +390,15 @@ async def interactions_service( else: if not xpath_selector: raise ValueError("xpath_selector must be a non-empty string") - await wait_for_xpath(xpath_selector, timeout_ms=30000) + await wait_for_xpath( + xpath_selector, timeout_ms=INTERACTIONS_TIMEOUT_MS + ) normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector locator = page.locator(f"xpath={normalized_xpath}").first - await locator.wait_for(timeout=30000) + await locator.wait_for(timeout=INTERACTIONS_TIMEOUT_MS) # Prefer fill/type so frameworks see real input events. - await locator.click(force=True, timeout=30000) - await locator.fill(text, timeout=30000) + await locator.click(force=True, timeout=INTERACTIONS_TIMEOUT_MS) + await locator.fill(text, timeout=INTERACTIONS_TIMEOUT_MS) else: raise ValueError(f"Unknown interaction action: {action}") @@ -393,7 +411,9 @@ async def interactions_service( vx, vy = await resolve_viewport_point(x, y) element, handle = await element_from_point_retry(vx, vy) try: - await element.click(force=True, timeout=30000) + await element.click( + force=True, timeout=INTERACTIONS_TIMEOUT_MS + ) finally: try: await handle.dispose() @@ -409,10 +429,12 @@ async def interactions_service( vx, vy = await resolve_viewport_point(x, y) element, handle = await element_from_point_retry(vx, vy) try: - await element.click(force=True, timeout=30000) + await element.click( + force=True, timeout=INTERACTIONS_TIMEOUT_MS + ) # Ensure the click actually put focus on the intended control. try: - await element.focus(timeout=30000) + await element.focus(timeout=INTERACTIONS_TIMEOUT_MS) except Exception: pass await page.wait_for_timeout(100) @@ -455,7 +477,12 @@ async def interactions_service( print(f"Error during interactions capture: {e}") return {"status": "error", "url": decoded_url, "error": str(e)} - return await safe_browser_operation(decoded_url, interactions_operation) + return await safe_browser_operation( + decoded_url, + interactions_operation, + page_timeout_ms=INTERACTIONS_TIMEOUT_MS, + operation_timeout_seconds=INTERACTIONS_OPERATION_TIMEOUT_SECONDS, + ) async def extract_seo_service(decoded_url): """Service function to extract SEO information from a website""" @@ -740,7 +767,13 @@ async def get_resulting_url_service(decoded_url): # Perform the operation return await safe_browser_operation(decoded_url, resulting_url_operation) -async def safe_browser_operation(url, operation_func): +async def safe_browser_operation( + url, + operation_func, + *, + page_timeout_ms: int = 30_000, + operation_timeout_seconds: float = 60.0, +): """Safely perform browser operations with proper cleanup and timeouts""" browser = None context = None @@ -767,14 +800,18 @@ async def safe_browser_operation(url, operation_func): ) page = await asyncio.wait_for(context.new_page(), timeout=10.0) - page.set_default_timeout(30000) # 30 second timeout + page.set_default_timeout(page_timeout_ms) + page.set_default_navigation_timeout(page_timeout_ms) # Track page creation time for force cleanup page_creation_times[page] = time.time() active_pages.add(page) # Call the operation function that uses the page with timeout - result = await asyncio.wait_for(operation_func(page), timeout=60.0) + result = await asyncio.wait_for( + operation_func(page), + timeout=operation_timeout_seconds, + ) return result