57 lines
1.7 KiB
Python
57 lines
1.7 KiB
Python
from pyppeteer import launch
|
|
from app.config import CUSTOM_USER_AGENT
|
|
import asyncio
|
|
|
|
async def wait_for_network_idle(page):
|
|
"""Wait until no network requests are in flight"""
|
|
await page.waitForNetworkIdle(idleTime=500, timeout=30000)
|
|
|
|
async def safe_browser_operation(url, operation_func):
|
|
"""Safely perform browser operations with proper cleanup"""
|
|
browser = None
|
|
page = None
|
|
try:
|
|
browser = await launch(
|
|
headless=True,
|
|
executablePath='/usr/bin/google-chrome',
|
|
args=['--no-sandbox', '--disable-setuid-sandbox'],
|
|
handleSIGINT=False,
|
|
handleSIGTERM=False,
|
|
handleSIGHUP=False
|
|
)
|
|
|
|
# Create new page with timeout
|
|
page = await browser.newPage()
|
|
page.setDefaultNavigationTimeout(30000)
|
|
|
|
# Set custom user agent
|
|
await page.setUserAgent(CUSTOM_USER_AGENT)
|
|
|
|
# Call the operation function that uses the page
|
|
result = await operation_func(page)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during browser operation: {e}")
|
|
# Return error result instead of re-raising to allow graceful handling
|
|
return {
|
|
"status": "error",
|
|
"error": str(e),
|
|
"url": url
|
|
}
|
|
finally:
|
|
# Ensure page is closed properly
|
|
if page:
|
|
try:
|
|
await page.close()
|
|
except Exception as e:
|
|
print(f"Error closing page: {e}")
|
|
|
|
# Ensure browser is closed properly
|
|
if browser:
|
|
try:
|
|
await browser.close()
|
|
except Exception as e:
|
|
print(f"Error closing browser: {e}")
|
|
# We don't re-raise here to avoid masking the original error |