from pyppeteer import launch from app.config import CUSTOM_USER_AGENT import asyncio async def wait_for_network_idle(page): """Wait until no network requests are in flight""" await page.waitForNetworkIdle(idleTime=500, timeout=30000) async def safe_browser_operation(url, operation_func): """Safely perform browser operations with proper cleanup""" browser = None page = None try: browser = await launch( headless=True, executablePath='/usr/bin/google-chrome', args=['--no-sandbox', '--disable-setuid-sandbox'], handleSIGINT=False, handleSIGTERM=False, handleSIGHUP=False ) # Create new page with timeout page = await browser.newPage() page.setDefaultNavigationTimeout(30000) # Set custom user agent await page.setUserAgent(CUSTOM_USER_AGENT) # Call the operation function that uses the page result = await operation_func(page) return result except Exception as e: print(f"Error during browser operation: {e}") # Return error result instead of re-raising to allow graceful handling return { "status": "error", "error": str(e), "url": url } finally: # Ensure page is closed properly if page: try: await page.close() except Exception as e: print(f"Error closing page: {e}") # Ensure browser is closed properly if browser: try: await browser.close() except Exception as e: print(f"Error closing browser: {e}") # We don't re-raise here to avoid masking the original error