from playwright.async_api import async_playwright from app.config import CUSTOM_USER_AGENT, BROWSER_INSTANCE_TIMEOUT_MINUTES import asyncio import time # Global tracking for browser instances created by this module page_creation_times = {} active_pages = set() async def wait_for_network_idle(page): """Wait until no network requests are in flight""" await page.wait_for_load_state('networkidle') async def safe_browser_operation(url, operation_func): """Safely perform browser operations with proper cleanup""" browser = None context = None page = None playwright = None try: # Get or create playwright instance playwright = await async_playwright().start() browser = await playwright.chromium.launch( headless=True, args=['--no-sandbox', '--disable-setuid-sandbox'], ) # Create context and page context = await browser.new_context( user_agent=CUSTOM_USER_AGENT, viewport={'width': 1920, 'height': 1080}, ignore_https_errors=True, ) page = await context.new_page() page.set_default_timeout(30000) # Track page creation time for force cleanup page_creation_times[page] = time.time() active_pages.add(page) # Call the operation function that uses the page result = await operation_func(page) return result except Exception as e: print(f"Error during browser operation: {e}") # Return error result instead of re-raising to allow graceful handling return { "status": "error", "error": str(e), "url": url } finally: # Cleanup page tracking if page: try: # Remove from tracking if page in active_pages: active_pages.remove(page) if page in page_creation_times: del page_creation_times[page] await page.close() except Exception as e: print(f"Error closing page: {e}") # Ensure context is closed properly if context: try: await context.close() except Exception as e: print(f"Error closing context: {e}") # Ensure browser is closed properly if browser: try: await browser.close() except Exception as e: print(f"Error closing browser: {e}") # Ensure playwright is closed properly - THIS IS THE KEY FIX if playwright: try: await playwright.stop() except Exception as e: print(f"Error closing playwright: {e}") async def force_cleanup_old_pages(): """Force cleanup old page instances created by this module""" print(f"Checking for old page instances in browser_utils (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...") current_time = time.time() timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60 cleaned_pages = 0 # Clean up old pages pages_to_cleanup = [] for page in list(active_pages): creation_time = page_creation_times.get(page, 0) if current_time - creation_time > timeout_seconds: pages_to_cleanup.append(page) print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)") for page in pages_to_cleanup: try: if page in active_pages: active_pages.remove(page) if page in page_creation_times: del page_creation_times[page] await page.close() cleaned_pages += 1 except Exception as e: print(f"Error cleaning up old page: {e}") print(f"Force cleanup completed: {cleaned_pages} pages cleaned")