134 lines
4.8 KiB
Python
134 lines
4.8 KiB
Python
from playwright.async_api import async_playwright
|
|
from app.config import CUSTOM_USER_AGENT, BROWSER_INSTANCE_TIMEOUT_MINUTES
|
|
import asyncio
|
|
import time
|
|
|
|
# Global tracking for browser instances created by this module
|
|
page_creation_times = {}
|
|
active_pages = set()
|
|
|
|
async def wait_for_network_idle(page):
|
|
"""Wait until no network requests are in flight"""
|
|
await page.wait_for_load_state('networkidle')
|
|
|
|
async def safe_browser_operation(url, operation_func):
|
|
"""Safely perform browser operations with proper cleanup and timeouts"""
|
|
browser = None
|
|
context = None
|
|
page = None
|
|
playwright = None
|
|
|
|
try:
|
|
# Get or create playwright instance with timeout
|
|
playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0)
|
|
|
|
browser = await asyncio.wait_for(playwright.chromium.launch(
|
|
headless=True,
|
|
args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'],
|
|
), timeout=30.0)
|
|
|
|
# Create context and page with timeout
|
|
context = await asyncio.wait_for(browser.new_context(
|
|
user_agent=CUSTOM_USER_AGENT,
|
|
viewport={'width': 1920, 'height': 1080},
|
|
ignore_https_errors=True,
|
|
), timeout=10.0)
|
|
|
|
page = await asyncio.wait_for(context.new_page(), timeout=10.0)
|
|
page.set_default_timeout(30000) # 30 second timeout
|
|
|
|
# Track page creation time for force cleanup
|
|
page_creation_times[page] = time.time()
|
|
active_pages.add(page)
|
|
|
|
# Call the operation function that uses the page with timeout
|
|
result = await asyncio.wait_for(operation_func(page), timeout=60.0)
|
|
|
|
return result
|
|
|
|
except asyncio.TimeoutError as e:
|
|
print(f"Browser operation timeout for {url}: {e}")
|
|
return {
|
|
"status": "error",
|
|
"error": f"Operation timeout: {str(e)}",
|
|
"url": url
|
|
}
|
|
except Exception as e:
|
|
print(f"Error during browser operation for {url}: {e}")
|
|
# Return error result instead of re-raising to allow graceful handling
|
|
return {
|
|
"status": "error",
|
|
"error": str(e),
|
|
"url": url
|
|
}
|
|
finally:
|
|
# Cleanup page tracking with timeout
|
|
if page:
|
|
try:
|
|
# Remove from tracking
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await asyncio.wait_for(page.close(), timeout=5.0)
|
|
except (asyncio.TimeoutError, Exception) as e:
|
|
print(f"Error closing page: {e}")
|
|
try:
|
|
await page.close(force=True)
|
|
except:
|
|
pass
|
|
|
|
# Ensure context is closed properly with timeout
|
|
if context:
|
|
try:
|
|
await asyncio.wait_for(context.close(), timeout=5.0)
|
|
except (asyncio.TimeoutError, Exception) as e:
|
|
print(f"Error closing context: {e}")
|
|
|
|
# Ensure browser is closed properly with timeout
|
|
if browser:
|
|
try:
|
|
await asyncio.wait_for(browser.close(), timeout=10.0)
|
|
except asyncio.TimeoutError:
|
|
try:
|
|
await browser.close(force=True)
|
|
except:
|
|
pass
|
|
except Exception as e:
|
|
print(f"Error closing browser: {e}")
|
|
|
|
# Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX
|
|
if playwright:
|
|
try:
|
|
await asyncio.wait_for(playwright.stop(), timeout=10.0)
|
|
except (asyncio.TimeoutError, Exception) as e:
|
|
print(f"Error closing playwright: {e}")
|
|
|
|
async def force_cleanup_old_pages():
|
|
"""Force cleanup old page instances created by this module"""
|
|
print(f"Checking for old page instances in browser_utils (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...")
|
|
|
|
current_time = time.time()
|
|
timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60
|
|
cleaned_pages = 0
|
|
|
|
# Clean up old pages
|
|
pages_to_cleanup = []
|
|
for page in list(active_pages):
|
|
creation_time = page_creation_times.get(page, 0)
|
|
if current_time - creation_time > timeout_seconds:
|
|
pages_to_cleanup.append(page)
|
|
print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)")
|
|
|
|
for page in pages_to_cleanup:
|
|
try:
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await page.close()
|
|
cleaned_pages += 1
|
|
except Exception as e:
|
|
print(f"Error cleaning up old page: {e}")
|
|
|
|
print(f"Force cleanup completed: {cleaned_pages} pages cleaned") |