118 lines
3.8 KiB
Python
118 lines
3.8 KiB
Python
from playwright.async_api import async_playwright
|
|
from app.config import CUSTOM_USER_AGENT, BROWSER_INSTANCE_TIMEOUT_MINUTES
|
|
import asyncio
|
|
import time
|
|
|
|
# Global tracking for browser instances created by this module
|
|
page_creation_times = {}
|
|
active_pages = set()
|
|
|
|
async def wait_for_network_idle(page):
|
|
"""Wait until no network requests are in flight"""
|
|
await page.wait_for_load_state('networkidle')
|
|
|
|
async def safe_browser_operation(url, operation_func):
|
|
"""Safely perform browser operations with proper cleanup"""
|
|
browser = None
|
|
context = None
|
|
page = None
|
|
playwright = None
|
|
|
|
try:
|
|
# Get or create playwright instance
|
|
playwright = await async_playwright().start()
|
|
|
|
browser = await playwright.chromium.launch(
|
|
headless=True,
|
|
args=['--no-sandbox', '--disable-setuid-sandbox'],
|
|
)
|
|
|
|
# Create context and page
|
|
context = await browser.new_context(
|
|
user_agent=CUSTOM_USER_AGENT,
|
|
viewport={'width': 1920, 'height': 1080},
|
|
ignore_https_errors=True,
|
|
)
|
|
|
|
page = await context.new_page()
|
|
page.set_default_timeout(30000)
|
|
|
|
# Track page creation time for force cleanup
|
|
page_creation_times[page] = time.time()
|
|
active_pages.add(page)
|
|
|
|
# Call the operation function that uses the page
|
|
result = await operation_func(page)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during browser operation: {e}")
|
|
# Return error result instead of re-raising to allow graceful handling
|
|
return {
|
|
"status": "error",
|
|
"error": str(e),
|
|
"url": url
|
|
}
|
|
finally:
|
|
# Cleanup page tracking
|
|
if page:
|
|
try:
|
|
# Remove from tracking
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await page.close()
|
|
except Exception as e:
|
|
print(f"Error closing page: {e}")
|
|
|
|
# Ensure context is closed properly
|
|
if context:
|
|
try:
|
|
await context.close()
|
|
except Exception as e:
|
|
print(f"Error closing context: {e}")
|
|
|
|
# Ensure browser is closed properly
|
|
if browser:
|
|
try:
|
|
await browser.close()
|
|
except Exception as e:
|
|
print(f"Error closing browser: {e}")
|
|
|
|
# Ensure playwright is closed properly - THIS IS THE KEY FIX
|
|
if playwright:
|
|
try:
|
|
await playwright.stop()
|
|
except Exception as e:
|
|
print(f"Error closing playwright: {e}")
|
|
|
|
async def force_cleanup_old_pages():
|
|
"""Force cleanup old page instances created by this module"""
|
|
print(f"Checking for old page instances in browser_utils (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...")
|
|
|
|
current_time = time.time()
|
|
timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60
|
|
cleaned_pages = 0
|
|
|
|
# Clean up old pages
|
|
pages_to_cleanup = []
|
|
for page in list(active_pages):
|
|
creation_time = page_creation_times.get(page, 0)
|
|
if current_time - creation_time > timeout_seconds:
|
|
pages_to_cleanup.append(page)
|
|
print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)")
|
|
|
|
for page in pages_to_cleanup:
|
|
try:
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await page.close()
|
|
cleaned_pages += 1
|
|
except Exception as e:
|
|
print(f"Error cleaning up old page: {e}")
|
|
|
|
print(f"Force cleanup completed: {cleaned_pages} pages cleaned") |