from playwright.async_api import async_playwright from app.config import CUSTOM_USER_AGENT, BROWSER_INSTANCE_TIMEOUT_MINUTES import asyncio import time # Global tracking for browser instances created by this module page_creation_times = {} active_pages = set() async def wait_for_network_idle(page): """Wait until no network requests are in flight""" await page.wait_for_load_state('networkidle') async def safe_browser_operation(url, operation_func): """Safely perform browser operations with proper cleanup and timeouts""" browser = None context = None page = None playwright = None try: # Get or create playwright instance with timeout playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0) browser = await asyncio.wait_for(playwright.chromium.launch( headless=True, args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'], ), timeout=30.0) # Create context and page with timeout context = await asyncio.wait_for(browser.new_context( user_agent=CUSTOM_USER_AGENT, viewport={'width': 1920, 'height': 1080}, ignore_https_errors=True, ), timeout=10.0) page = await asyncio.wait_for(context.new_page(), timeout=10.0) page.set_default_timeout(30000) # 30 second timeout # Track page creation time for force cleanup page_creation_times[page] = time.time() active_pages.add(page) # Call the operation function that uses the page with timeout result = await asyncio.wait_for(operation_func(page), timeout=60.0) return result except asyncio.TimeoutError as e: print(f"Browser operation timeout for {url}: {e}") return { "status": "error", "error": f"Operation timeout: {str(e)}", "url": url } except Exception as e: print(f"Error during browser operation for {url}: {e}") # Return error result instead of re-raising to allow graceful handling return { "status": "error", "error": str(e), "url": url } finally: # Cleanup page tracking with timeout if page: try: # Remove from tracking if page in active_pages: active_pages.remove(page) if page in page_creation_times: del page_creation_times[page] await asyncio.wait_for(page.close(), timeout=5.0) except (asyncio.TimeoutError, Exception) as e: print(f"Error closing page: {e}") try: await page.close(force=True) except: pass # Ensure context is closed properly with timeout if context: try: await asyncio.wait_for(context.close(), timeout=5.0) except (asyncio.TimeoutError, Exception) as e: print(f"Error closing context: {e}") # Ensure browser is closed properly with timeout if browser: try: await asyncio.wait_for(browser.close(), timeout=10.0) except asyncio.TimeoutError: try: await browser.close(force=True) except: pass except Exception as e: print(f"Error closing browser: {e}") # Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX if playwright: try: await asyncio.wait_for(playwright.stop(), timeout=10.0) except (asyncio.TimeoutError, Exception) as e: print(f"Error closing playwright: {e}") async def force_cleanup_old_pages(): """Force cleanup old page instances created by this module""" print(f"Checking for old page instances in browser_utils (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...") current_time = time.time() timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60 cleaned_pages = 0 # Clean up old pages pages_to_cleanup = [] for page in list(active_pages): creation_time = page_creation_times.get(page, 0) if current_time - creation_time > timeout_seconds: pages_to_cleanup.append(page) print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)") for page in pages_to_cleanup: try: if page in active_pages: active_pages.remove(page) if page in page_creation_times: del page_creation_times[page] await page.close() cleaned_pages += 1 except Exception as e: print(f"Error cleaning up old page: {e}") print(f"Force cleanup completed: {cleaned_pages} pages cleaned")