From e819d0ddcb34aab62b605c544374b53c578c593c Mon Sep 17 00:00:00 2001 From: Bram Date: Fri, 8 Aug 2025 13:43:07 +0200 Subject: [PATCH] cleanup old browser instances job --- Dockers/puppeteer-api/app/main.py | 10 ++ Dockers/puppeteer-api/app/services/browser.py | 131 ++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/Dockers/puppeteer-api/app/main.py b/Dockers/puppeteer-api/app/main.py index 51827c7..948b897 100644 --- a/Dockers/puppeteer-api/app/main.py +++ b/Dockers/puppeteer-api/app/main.py @@ -4,6 +4,7 @@ from apscheduler.triggers.cron import CronTrigger from app.database import init_db, cleanup_old_cache_entries from app.config import CLEANUP_CRON, CACHE_EXPIRY_HOURS from app.routes import health, browser, cache +from app.services.browser import force_cleanup_old_pages # Create FastAPI app app = FastAPI() @@ -22,6 +23,14 @@ scheduler.add_job( replace_existing=True ) +# Add browser cleanup job +scheduler.add_job( + force_cleanup_old_pages, + CronTrigger.from_crontab(CLEANUP_CRON), + id='browser_cleanup_job', + replace_existing=True +) + # Initialize database on startup init_db() @@ -31,6 +40,7 @@ def start_scheduler(): scheduler.start() print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}") print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours") + print("Browser cleanup scheduler started") # Shutdown the scheduler when the application stops @app.on_event("shutdown") diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 02c2b16..22b0635 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -1,6 +1,13 @@ from app.utils.browser_utils import safe_browser_operation import asyncio from urllib.parse import unquote +import time +from playwright.async_api import async_playwright +from app.config import CUSTOM_USER_AGENT + +# Global tracking for active pages and their creation times +active_pages = set() +page_creation_times = {} async def visit_url_service(decoded_url): """Service function to visit a URL and get its content""" @@ -330,3 +337,127 @@ async def get_resulting_url_service(decoded_url): # Perform the operation return await safe_browser_operation(decoded_url, resulting_url_operation) + +async def safe_browser_operation(url, operation_func): + """Safely perform browser operations with proper cleanup and timeouts""" + browser = None + context = None + page = None + playwright = None + + try: + # Get or create playwright instance with timeout + playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0) + + browser = await asyncio.wait_for(playwright.chromium.launch( + headless=True, + args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'], + ), timeout=30.0) + + # Create context and page with timeout + context = await asyncio.wait_for(browser.new_context( + user_agent=CUSTOM_USER_AGENT, + viewport={'width': 1920, 'height': 1080}, + ignore_https_errors=True, + ), timeout=10.0) + + page = await asyncio.wait_for(context.new_page(), timeout=10.0) + page.set_default_timeout(30000) # 30 second timeout + + # Track page creation time for force cleanup + page_creation_times[page] = time.time() + active_pages.add(page) + + # Call the operation function that uses the page with timeout + result = await asyncio.wait_for(operation_func(page), timeout=60.0) + + return result + + except asyncio.TimeoutError as e: + print(f"Browser operation timeout for {url}: {e}") + return { + "status": "error", + "error": f"Operation timeout: {str(e)}", + "url": url + } + except Exception as e: + print(f"Error during browser operation for {url}: {e}") + # Return error result instead of re-raising to allow graceful handling + return { + "status": "error", + "error": str(e), + "url": url + } + finally: + # Cleanup page tracking with timeout + if page: + try: + # Remove from tracking + if page in active_pages: + active_pages.remove(page) + if page in page_creation_times: + del page_creation_times[page] + await asyncio.wait_for(page.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: + print(f"Error closing page: {e}") + try: + await page.close(force=True) + except: + pass + + # Ensure context is closed properly with timeout + if context: + try: + await asyncio.wait_for(context.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: + print(f"Error closing context: {e}") + + # Ensure browser is closed properly with timeout + if browser: + try: + await asyncio.wait_for(browser.close(), timeout=10.0) + except asyncio.TimeoutError: + try: + await browser.close(force=True) + except: + pass + except Exception as e: + print(f"Error closing browser: {e}") + + # Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX + if playwright: + try: + await asyncio.wait_for(playwright.stop(), timeout=10.0) + except (asyncio.TimeoutError, Exception) as e: + print(f"Error closing playwright: {e}") + +async def force_cleanup_old_pages(): + """Force cleanup old page instances created by this module""" + from app.config import BROWSER_INSTANCE_TIMEOUT_MINUTES + + print(f"Checking for old page instances in browser service (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...") + + current_time = time.time() + timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60 + cleaned_pages = 0 + + # Clean up old pages + pages_to_cleanup = [] + for page in list(active_pages): + creation_time = page_creation_times.get(page, 0) + if current_time - creation_time > timeout_seconds: + pages_to_cleanup.append(page) + print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)") + + for page in pages_to_cleanup: + try: + if page in active_pages: + active_pages.remove(page) + if page in page_creation_times: + del page_creation_times[page] + await page.close() + cleaned_pages += 1 + except Exception as e: + print(f"Error cleaning up old page: {e}") + + print(f"Force cleanup completed: {cleaned_pages} pages cleaned")