cleanup old browser instances job

Dockers/puppeteer-api/app/main.py

@@ -4,6 +4,7 @@ from apscheduler.triggers.cron import CronTrigger
 from app.database import init_db, cleanup_old_cache_entries
 from app.config import CLEANUP_CRON, CACHE_EXPIRY_HOURS
 from app.routes import health, browser, cache
+from app.services.browser import force_cleanup_old_pages
 
 # Create FastAPI app
 app = FastAPI()
@@ -22,6 +23,14 @@ scheduler.add_job(
     replace_existing=True
 )
 
+# Add browser cleanup job
+scheduler.add_job(
+    force_cleanup_old_pages,
+    CronTrigger.from_crontab(CLEANUP_CRON),
+    id='browser_cleanup_job',
+    replace_existing=True
+)
+
 # Initialize database on startup
 init_db()
 
@@ -31,6 +40,7 @@ def start_scheduler():
     scheduler.start()
     print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
     print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours")
+    print("Browser cleanup scheduler started")
 
 # Shutdown the scheduler when the application stops
 @app.on_event("shutdown")

Dockers/puppeteer-api/app/services/browser.py

@@ -1,6 +1,13 @@
 from app.utils.browser_utils import safe_browser_operation
 import asyncio
 from urllib.parse import unquote
+import time
+from playwright.async_api import async_playwright
+from app.config import CUSTOM_USER_AGENT
+
+# Global tracking for active pages and their creation times
+active_pages = set()
+page_creation_times = {}
 
 async def visit_url_service(decoded_url):
     """Service function to visit a URL and get its content"""
@@ -330,3 +337,127 @@ async def get_resulting_url_service(decoded_url):
 
     # Perform the operation
     return await safe_browser_operation(decoded_url, resulting_url_operation)
+
+async def safe_browser_operation(url, operation_func):
+    """Safely perform browser operations with proper cleanup and timeouts"""
+    browser = None
+    context = None
+    page = None
+    playwright = None
+
+    try:
+        # Get or create playwright instance with timeout
+        playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0)
+
+        browser = await asyncio.wait_for(playwright.chromium.launch(
+            headless=True,
+            args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'],
+        ), timeout=30.0)
+
+        # Create context and page with timeout
+        context = await asyncio.wait_for(browser.new_context(
+            user_agent=CUSTOM_USER_AGENT,
+            viewport={'width': 1920, 'height': 1080},
+            ignore_https_errors=True,
+        ), timeout=10.0)
+
+        page = await asyncio.wait_for(context.new_page(), timeout=10.0)
+        page.set_default_timeout(30000)  # 30 second timeout
+
+        # Track page creation time for force cleanup
+        page_creation_times[page] = time.time()
+        active_pages.add(page)
+
+        # Call the operation function that uses the page with timeout
+        result = await asyncio.wait_for(operation_func(page), timeout=60.0)
+
+        return result
+
+    except asyncio.TimeoutError as e:
+        print(f"Browser operation timeout for {url}: {e}")
+        return {
+            "status": "error",
+            "error": f"Operation timeout: {str(e)}",
+            "url": url
+        }
+    except Exception as e:
+        print(f"Error during browser operation for {url}: {e}")
+        # Return error result instead of re-raising to allow graceful handling
+        return {
+            "status": "error",
+            "error": str(e),
+            "url": url
+        }
+    finally:
+        # Cleanup page tracking with timeout
+        if page:
+            try:
+                # Remove from tracking
+                if page in active_pages:
+                    active_pages.remove(page)
+                if page in page_creation_times:
+                    del page_creation_times[page]
+                await asyncio.wait_for(page.close(), timeout=5.0)
+            except (asyncio.TimeoutError, Exception) as e:
+                print(f"Error closing page: {e}")
+                try:
+                    await page.close(force=True)
+                except:
+                    pass
+
+        # Ensure context is closed properly with timeout
+        if context:
+            try:
+                await asyncio.wait_for(context.close(), timeout=5.0)
+            except (asyncio.TimeoutError, Exception) as e:
+                print(f"Error closing context: {e}")
+
+        # Ensure browser is closed properly with timeout
+        if browser:
+            try:
+                await asyncio.wait_for(browser.close(), timeout=10.0)
+            except asyncio.TimeoutError:
+                try:
+                    await browser.close(force=True)
+                except:
+                    pass
+            except Exception as e:
+                print(f"Error closing browser: {e}")
+
+        # Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX
+        if playwright:
+            try:
+                await asyncio.wait_for(playwright.stop(), timeout=10.0)
+            except (asyncio.TimeoutError, Exception) as e:
+                print(f"Error closing playwright: {e}")
+
+async def force_cleanup_old_pages():
+    """Force cleanup old page instances created by this module"""
+    from app.config import BROWSER_INSTANCE_TIMEOUT_MINUTES
+
+    print(f"Checking for old page instances in browser service (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...")
+
+    current_time = time.time()
+    timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60
+    cleaned_pages = 0
+
+    # Clean up old pages
+    pages_to_cleanup = []
+    for page in list(active_pages):
+        creation_time = page_creation_times.get(page, 0)
+        if current_time - creation_time > timeout_seconds:
+            pages_to_cleanup.append(page)
+            print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)")
+
+    for page in pages_to_cleanup:
+        try:
+            if page in active_pages:
+                active_pages.remove(page)
+            if page in page_creation_times:
+                del page_creation_times[page]
+            await page.close()
+            cleaned_pages += 1
+        except Exception as e:
+            print(f"Error cleaning up old page: {e}")
+
+    print(f"Force cleanup completed: {cleaned_pages} pages cleaned")