increase timeout settings for interactions in browser service to accommodate slow SPAs and heavy pages, ensuring more reliable page loading and interaction handling
Build and Push Docker Images / build-and-push (push) Successful in 1m12s
Build and Push Docker Images / build-and-push (push) Successful in 1m12s
This commit is contained in:
@@ -9,6 +9,10 @@ from app.config import CUSTOM_USER_AGENT
|
||||
active_pages = set()
|
||||
page_creation_times = {}
|
||||
|
||||
# Interactions endpoint: allow slow SPAs / heavy pages (Playwright + overall asyncio cap).
|
||||
INTERACTIONS_TIMEOUT_MS = 120_000
|
||||
INTERACTIONS_OPERATION_TIMEOUT_SECONDS = 120.0
|
||||
|
||||
async def visit_url_service(decoded_url):
|
||||
"""Service function to visit a URL and get its content"""
|
||||
print(f"Visiting URL: {decoded_url}")
|
||||
@@ -241,11 +245,15 @@ async def interactions_service(
|
||||
print(f"Error setting forced viewport width: {e}")
|
||||
|
||||
# Navigate and wait for initial DOM.
|
||||
await page.goto(decoded_url, wait_until="domcontentloaded", timeout=30000)
|
||||
await page.wait_for_load_state("load", timeout=30000)
|
||||
await page.goto(
|
||||
decoded_url, wait_until="domcontentloaded", timeout=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
await page.wait_for_load_state("load", timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
|
||||
# Ensure the page is fully loaded (helps with JS-driven UIs).
|
||||
await page.wait_for_function("document.readyState === 'complete'", timeout=30000)
|
||||
await page.wait_for_function(
|
||||
"document.readyState === 'complete'", timeout=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
|
||||
# Best-effort cookie/banner handling (helps avoid click interception).
|
||||
cookie_buttons = [
|
||||
@@ -307,7 +315,9 @@ async def interactions_service(
|
||||
if not action:
|
||||
raise ValueError("Interaction must contain an action")
|
||||
|
||||
async def wait_for_xpath(xp: str, timeout_ms: int = 30000) -> None:
|
||||
async def wait_for_xpath(
|
||||
xp: str, timeout_ms: int = INTERACTIONS_TIMEOUT_MS
|
||||
) -> None:
|
||||
# Accept both raw XPath ("//div") and Playwright-style ("xpath=//div").
|
||||
normalized = xp[len("xpath=") :] if xp.startswith("xpath=") else xp
|
||||
await page.wait_for_function(
|
||||
@@ -332,7 +342,9 @@ async def interactions_service(
|
||||
if selector or xpath_selector:
|
||||
if action == "click":
|
||||
if selector:
|
||||
await page.wait_for_selector(selector, timeout=30000)
|
||||
await page.wait_for_selector(
|
||||
selector, timeout=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
# Avoid Playwright auto-scrolling: click via DOM.
|
||||
await page.evaluate(
|
||||
"""(sel) => {
|
||||
@@ -345,18 +357,22 @@ async def interactions_service(
|
||||
else:
|
||||
if not xpath_selector:
|
||||
raise ValueError("xpath_selector must be a non-empty string")
|
||||
await wait_for_xpath(xpath_selector, timeout_ms=30000)
|
||||
await wait_for_xpath(
|
||||
xpath_selector, timeout_ms=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector
|
||||
# Use Playwright's click to generate real pointer/mouse events.
|
||||
locator = page.locator(f"xpath={normalized_xpath}").first
|
||||
await locator.wait_for(timeout=30000)
|
||||
await locator.click(force=True, timeout=30000)
|
||||
await locator.wait_for(timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
await locator.click(force=True, timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
|
||||
elif action == "type":
|
||||
if text is None:
|
||||
raise ValueError("Type interaction must contain text")
|
||||
if selector:
|
||||
await page.wait_for_selector(selector, timeout=30000)
|
||||
await page.wait_for_selector(
|
||||
selector, timeout=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
# Avoid Playwright auto-scrolling: focus/fill via DOM.
|
||||
await page.evaluate(
|
||||
"""({ sel, value }) => {
|
||||
@@ -374,13 +390,15 @@ async def interactions_service(
|
||||
else:
|
||||
if not xpath_selector:
|
||||
raise ValueError("xpath_selector must be a non-empty string")
|
||||
await wait_for_xpath(xpath_selector, timeout_ms=30000)
|
||||
await wait_for_xpath(
|
||||
xpath_selector, timeout_ms=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
normalized_xpath = xpath_selector[len("xpath=") :] if xpath_selector.startswith("xpath=") else xpath_selector
|
||||
locator = page.locator(f"xpath={normalized_xpath}").first
|
||||
await locator.wait_for(timeout=30000)
|
||||
await locator.wait_for(timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
# Prefer fill/type so frameworks see real input events.
|
||||
await locator.click(force=True, timeout=30000)
|
||||
await locator.fill(text, timeout=30000)
|
||||
await locator.click(force=True, timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
await locator.fill(text, timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown interaction action: {action}")
|
||||
@@ -393,7 +411,9 @@ async def interactions_service(
|
||||
vx, vy = await resolve_viewport_point(x, y)
|
||||
element, handle = await element_from_point_retry(vx, vy)
|
||||
try:
|
||||
await element.click(force=True, timeout=30000)
|
||||
await element.click(
|
||||
force=True, timeout=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
await handle.dispose()
|
||||
@@ -409,10 +429,12 @@ async def interactions_service(
|
||||
vx, vy = await resolve_viewport_point(x, y)
|
||||
element, handle = await element_from_point_retry(vx, vy)
|
||||
try:
|
||||
await element.click(force=True, timeout=30000)
|
||||
await element.click(
|
||||
force=True, timeout=INTERACTIONS_TIMEOUT_MS
|
||||
)
|
||||
# Ensure the click actually put focus on the intended control.
|
||||
try:
|
||||
await element.focus(timeout=30000)
|
||||
await element.focus(timeout=INTERACTIONS_TIMEOUT_MS)
|
||||
except Exception:
|
||||
pass
|
||||
await page.wait_for_timeout(100)
|
||||
@@ -455,7 +477,12 @@ async def interactions_service(
|
||||
print(f"Error during interactions capture: {e}")
|
||||
return {"status": "error", "url": decoded_url, "error": str(e)}
|
||||
|
||||
return await safe_browser_operation(decoded_url, interactions_operation)
|
||||
return await safe_browser_operation(
|
||||
decoded_url,
|
||||
interactions_operation,
|
||||
page_timeout_ms=INTERACTIONS_TIMEOUT_MS,
|
||||
operation_timeout_seconds=INTERACTIONS_OPERATION_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
async def extract_seo_service(decoded_url):
|
||||
"""Service function to extract SEO information from a website"""
|
||||
@@ -740,7 +767,13 @@ async def get_resulting_url_service(decoded_url):
|
||||
# Perform the operation
|
||||
return await safe_browser_operation(decoded_url, resulting_url_operation)
|
||||
|
||||
async def safe_browser_operation(url, operation_func):
|
||||
async def safe_browser_operation(
|
||||
url,
|
||||
operation_func,
|
||||
*,
|
||||
page_timeout_ms: int = 30_000,
|
||||
operation_timeout_seconds: float = 60.0,
|
||||
):
|
||||
"""Safely perform browser operations with proper cleanup and timeouts"""
|
||||
browser = None
|
||||
context = None
|
||||
@@ -767,14 +800,18 @@ async def safe_browser_operation(url, operation_func):
|
||||
)
|
||||
|
||||
page = await asyncio.wait_for(context.new_page(), timeout=10.0)
|
||||
page.set_default_timeout(30000) # 30 second timeout
|
||||
page.set_default_timeout(page_timeout_ms)
|
||||
page.set_default_navigation_timeout(page_timeout_ms)
|
||||
|
||||
# Track page creation time for force cleanup
|
||||
page_creation_times[page] = time.time()
|
||||
active_pages.add(page)
|
||||
|
||||
# Call the operation function that uses the page with timeout
|
||||
result = await asyncio.wait_for(operation_func(page), timeout=60.0)
|
||||
result = await asyncio.wait_for(
|
||||
operation_func(page),
|
||||
timeout=operation_timeout_seconds,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
Reference in New Issue
Block a user