potential performance upgrade

Dockers/puppeteer-api/app/utils/browser_utils.py

@@ -12,42 +12,49 @@ async def wait_for_network_idle(page):
     await page.wait_for_load_state('networkidle')
 
 async def safe_browser_operation(url, operation_func):
-    """Safely perform browser operations with proper cleanup"""
+    """Safely perform browser operations with proper cleanup and timeouts"""
     browser = None
     context = None
     page = None
     playwright = None
 
     try:
-        # Get or create playwright instance
-        playwright = await async_playwright().start()
+        # Get or create playwright instance with timeout
+        playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0)
 
-        browser = await playwright.chromium.launch(
+        browser = await asyncio.wait_for(playwright.chromium.launch(
             headless=True,
-            args=['--no-sandbox', '--disable-setuid-sandbox'],
-        )
+            args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'],
+        ), timeout=30.0)
 
-        # Create context and page
-        context = await browser.new_context(
+        # Create context and page with timeout
+        context = await asyncio.wait_for(browser.new_context(
             user_agent=CUSTOM_USER_AGENT,
             viewport={'width': 1920, 'height': 1080},
             ignore_https_errors=True,
-        )
+        ), timeout=10.0)
 
-        page = await context.new_page()
-        page.set_default_timeout(30000)
+        page = await asyncio.wait_for(context.new_page(), timeout=10.0)
+        page.set_default_timeout(30000)  # 30 second timeout
 
         # Track page creation time for force cleanup
         page_creation_times[page] = time.time()
         active_pages.add(page)
 
-        # Call the operation function that uses the page
-        result = await operation_func(page)
+        # Call the operation function that uses the page with timeout
+        result = await asyncio.wait_for(operation_func(page), timeout=60.0)
 
         return result
 
+    except asyncio.TimeoutError as e:
+        print(f"Browser operation timeout for {url}: {e}")
+        return {
+            "status": "error",
+            "error": f"Operation timeout: {str(e)}",
+            "url": url
+        }
     except Exception as e:
-        print(f"Error during browser operation: {e}")
+        print(f"Error during browser operation for {url}: {e}")
         # Return error result instead of re-raising to allow graceful handling
         return {
             "status": "error",
@@ -55,7 +62,7 @@ async def safe_browser_operation(url, operation_func):
             "url": url
         }
     finally:
-        # Cleanup page tracking
+        # Cleanup page tracking with timeout
         if page:
             try:
                 # Remove from tracking
@@ -63,29 +70,38 @@ async def safe_browser_operation(url, operation_func):
                     active_pages.remove(page)
                 if page in page_creation_times:
                     del page_creation_times[page]
-                await page.close()
-            except Exception as e:
+                await asyncio.wait_for(page.close(), timeout=5.0)
+            except (asyncio.TimeoutError, Exception) as e:
                 print(f"Error closing page: {e}")
+                try:
+                    await page.close(force=True)
+                except:
+                    pass
 
-        # Ensure context is closed properly
+        # Ensure context is closed properly with timeout
         if context:
             try:
-                await context.close()
-            except Exception as e:
+                await asyncio.wait_for(context.close(), timeout=5.0)
+            except (asyncio.TimeoutError, Exception) as e:
                 print(f"Error closing context: {e}")
 
-        # Ensure browser is closed properly
+        # Ensure browser is closed properly with timeout
         if browser:
             try:
-                await browser.close()
+                await asyncio.wait_for(browser.close(), timeout=10.0)
+            except asyncio.TimeoutError:
+                try:
+                    await browser.close(force=True)
+                except:
+                    pass
             except Exception as e:
                 print(f"Error closing browser: {e}")
 
-        # Ensure playwright is closed properly - THIS IS THE KEY FIX
+        # Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX
         if playwright:
             try:
-                await playwright.stop()
-            except Exception as e:
+                await asyncio.wait_for(playwright.stop(), timeout=10.0)
+            except (asyncio.TimeoutError, Exception) as e:
                 print(f"Error closing playwright: {e}")
 
 async def force_cleanup_old_pages():

Dockers/puppeteer-api/app/utils/cloudflare_bypass.py

@@ -273,73 +273,101 @@ class CloudflareBypass:
         """)
 
     @staticmethod
-    async def handle_cloudflare_challenge(page: Page, max_retries: int = 3) -> bool:
-        """Handle Cloudflare challenges and wait for them to complete"""
+    async def handle_cloudflare_challenge(page: Page, max_retries: int = 2) -> bool:
+        """Handle Cloudflare challenge with improved efficiency"""
         for attempt in range(max_retries):
             try:
-                # Check for Cloudflare challenge
-                cloudflare_selectors = [
-                    '#challenge-form',
-                    '#cf-please-wait',
-                    '.cf-browser-verification',
-                    '#cf-wrapper',
-                    'iframe[src*="cloudflare"]'
-                ]
+                print(f"Handling Cloudflare challenge (attempt {attempt + 1}/{max_retries})")
 
-                for selector in cloudflare_selectors:
-                    try:
-                        element = await page.wait_for_selector(selector, timeout=5000)
-                        if element:
-                            print(f"Cloudflare challenge detected on attempt {attempt + 1}")
-                            # Wait for challenge to complete
-                            await asyncio.sleep(5 + random.uniform(2, 8))
-                            break
-                    except:
-                        continue
-
-                # Wait for page to load completely
-                await page.wait_for_load_state('networkidle', timeout=30000)
-
-                # Check if we're past the challenge
-                title = await page.title()
-                if 'Cloudflare' not in title and 'challenge' not in title.lower():
-                    print("Successfully bypassed Cloudflare challenge")
+                # Wait for challenge form with shorter timeout
+                try:
+                    await page.wait_for_selector('#challenge-form', timeout=10000)  # 10 seconds
+                except:
+                    # If no challenge form, might already be bypassed
                     return True
 
-                # Add some random delay
-                await asyncio.sleep(random.uniform(3, 8))
+                # Wait for the challenge to complete with shorter timeout
+                try:
+                    await page.wait_for_function('''
+                        () => {
+                            return !document.querySelector('#challenge-form') &&
+                                   !document.querySelector('#cf-wrapper') &&
+                                   !document.querySelector('#cf-please-wait') &&
+                                   !document.querySelector('.cf-browser-verification');
+                        }
+                    ''', timeout=15000)  # 15 seconds
+
+                    print("Cloudflare challenge completed")
+                    return True
+
+                except Exception as e:
+                    print(f"Challenge completion timeout (attempt {attempt + 1}): {e}")
+
+                    # Try to click any "Verify" buttons
+                    try:
+                        verify_buttons = await page.query_selector_all('button[type="submit"], input[type="submit"]')
+                        for button in verify_buttons:
+                            try:
+                                await button.click(timeout=5000)
+                                await asyncio.sleep(2)
+                            except:
+                                continue
+                    except:
+                        pass
+
+                # Short delay before retry
+                if attempt < max_retries - 1:
+                    await asyncio.sleep(2)
 
             except Exception as e:
                 print(f"Error handling Cloudflare challenge (attempt {attempt + 1}): {e}")
                 if attempt < max_retries - 1:
-                    await asyncio.sleep(random.uniform(5, 15))
+                    await asyncio.sleep(2)
 
+        print("Failed to bypass Cloudflare challenge after all attempts")
         return False
 
     @staticmethod
     async def navigate_with_stealth(page: Page, url: str) -> bool:
-        """Navigate to URL with stealth measures and handle Cloudflare"""
+        """Navigate to URL with stealth measures and improved performance"""
         try:
-            # Add random delay before navigation
-            await asyncio.sleep(random.uniform(1, 3))
+            # Set shorter timeouts for better performance
+            page.set_default_timeout(30000)  # 30 seconds
+            page.set_default_navigation_timeout(30000)  # 30 seconds
 
-            # Navigate to the URL
-            await page.goto(url, wait_until='domcontentloaded')
+            # Navigate with stealth measures
+            response = await page.goto(url, wait_until='domcontentloaded')  # Changed from 'networkidle' to 'domcontentloaded' for speed
 
-            # Handle Cloudflare challenge
-            success = await CloudflareBypass.handle_cloudflare_challenge(page)
-
-            if not success:
-                print("Failed to bypass Cloudflare challenge")
+            if not response:
                 return False
 
-            # Add human-like scrolling
-            await CloudflareBypass._simulate_human_scrolling(page)
+            # Quick check for Cloudflare challenge
+            if response.status == 403 or response.status == 503:
+                # Handle Cloudflare challenge
+                return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2)  # Reduced retries
+
+            # Check for Cloudflare indicators in the page
+            cloudflare_detected = await page.evaluate('''() => {
+                return document.title.toLowerCase().includes('cloudflare') ||
+                       !!document.querySelector('#challenge-form') ||
+                       !!document.querySelector('#cf-wrapper') ||
+                       !!document.querySelector('#cf-please-wait') ||
+                       !!document.querySelector('.cf-browser-verification');
+            }''')
+
+            if cloudflare_detected:
+                return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2)  # Reduced retries
+
+            # Wait for page to be ready (shorter wait)
+            try:
+                await page.wait_for_load_state('domcontentloaded', timeout=10000)  # 10 second timeout
+            except:
+                pass  # Continue even if load state timeout
 
             return True
 
         except Exception as e:
-            print(f"Error during stealth navigation: {e}")
+            print(f"Error during stealth navigation to {url}: {e}")
             return False
 
     @staticmethod

Dockers/puppeteer-api/main.py

@@ -22,6 +22,9 @@ from contextlib import asynccontextmanager
 from app.utils.browser_utils import force_cleanup_old_pages
 from app.utils.cloudflare_bypass import CloudflareBypass
 
+# Add memory monitoring
+import gc
+
 app = FastAPI()
 
 # Get API key from environment variable
@@ -39,9 +42,11 @@ CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
 CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
 
 # Browser pool configuration - Reduced for better resource management
-MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '3'))  # Reduced from 5 to 3
-BROWSER_TTL = int(os.getenv('BROWSER_TTL', '1800'))  # Reduced from 3600 to 1800 seconds (30 minutes)
-MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '5'))  # New: limit concurrent operations
+MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '2'))  # Reduced from 3 to 2 for better resource management
+BROWSER_TTL = int(os.getenv('BROWSER_TTL', '900'))  # Reduced from 1800 to 900 seconds (15 minutes)
+MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '3'))  # Reduced from 5 to 3
+BROWSER_OPERATION_TIMEOUT = int(os.getenv('BROWSER_OPERATION_TIMEOUT', '60'))  # New: timeout for browser operations
+PAGE_TIMEOUT = int(os.getenv('PAGE_TIMEOUT', '30'))  # New: timeout for page operations
 
 # Browser pool management
 browser_pool = Queue(maxsize=MAX_BROWSERS)  # Add maxsize to prevent unbounded growth
@@ -52,6 +57,7 @@ active_browsers = set()  # Track active browsers
 active_pages = set()  # Track active pages for force cleanup
 operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS)  # Limit concurrent operations
 playwright_instance = None  # Global playwright instance
+browser_health_check_task = None  # Track health check task
 
 # Rate limiting configuration
 RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60'))  # requests per minute
@@ -111,9 +117,48 @@ async def create_browser():
         proxy_url = os.getenv('PROXY_URL')
         proxy_config = CloudflareBypass.get_proxy_config(proxy_url)
 
+        # Add memory and performance optimizations
+        browser_args = CloudflareBypass.get_stealth_args() + [
+            '--memory-pressure-off',
+            '--max_old_space_size=256',  # Reduced memory usage
+            '--disable-dev-shm-usage',
+            '--disable-gpu-sandbox',
+            '--disable-software-rasterizer',
+            '--disable-background-timer-throttling',
+            '--disable-backgrounding-occluded-windows',
+            '--disable-renderer-backgrounding',
+            '--disable-features=TranslateUI',
+            '--disable-ipc-flooding-protection',
+            '--disable-hang-monitor',
+            '--disable-prompt-on-repost',
+            '--disable-sync',
+            '--no-first-run',
+            '--no-default-browser-check',
+            '--no-pings',
+            '--no-zygote',
+            '--single-process',
+            '--disable-background-networking',
+            '--disable-background-timer-throttling',
+            '--disable-backgrounding-occluded-windows',
+            '--disable-breakpad',
+            '--disable-component-extensions-with-background-pages',
+            '--disable-dev-shm-usage',
+            '--disable-features=TranslateUI',
+            '--disable-ipc-flooding-protection',
+            '--disable-renderer-backgrounding',
+            '--disable-sync',
+            '--force-color-profile=srgb',
+            '--metrics-recording-only',
+            '--no-first-run',
+            '--safebrowsing-disable-auto-update',
+            '--enable-automation',
+            '--password-store=basic',
+            '--use-mock-keychain',
+        ]
+
         browser = await playwright_instance.chromium.launch(
             headless=True,
-            args=CloudflareBypass.get_stealth_args(),
+            args=browser_args,
             ignore_default_args=['--enable-automation'],
             **proxy_config
         )
@@ -124,10 +169,17 @@ async def create_browser():
         return browser
     except Exception as e:
         print(f"Error creating browser: {e}")
+        # Clean up playwright instance if browser creation fails
+        if playwright_instance:
+            try:
+                await playwright_instance.stop()
+                playwright_instance = None
+            except:
+                pass
         raise
 
 async def cleanup_browser(browser):
-    """Safely cleanup a browser instance"""
+    """Safely cleanup a browser instance with improved error handling"""
     try:
         if browser in active_browsers:
             active_browsers.remove(browser)
@@ -136,82 +188,117 @@ async def cleanup_browser(browser):
             del browser_creation_times[browser]
 
         # Close all pages first and clean up page tracking
-        pages = browser.contexts[0].pages if browser.contexts else []
-        for page in pages:
+        try:
+            contexts = browser.contexts
+            for context in contexts:
+                pages = context.pages
+                for page in pages:
+                    try:
+                        if page in active_pages:
+                            active_pages.remove(page)
+                        if page in page_creation_times:
+                            del page_creation_times[page]
+                        await asyncio.wait_for(page.close(), timeout=5.0)
+                    except (asyncio.TimeoutError, Exception) as e:
+                        print(f"Error closing page: {e}")
+                        # Force close if normal close fails
+                        try:
+                            await page.close(force=True)
+                        except:
+                            pass
+
+                # Close context
+                try:
+                    await asyncio.wait_for(context.close(), timeout=5.0)
+                except (asyncio.TimeoutError, Exception) as e:
+                    print(f"Error closing context: {e}")
+        except Exception as e:
+            print(f"Error during context/page cleanup: {e}")
+
+        # Close browser with timeout
+        try:
+            await asyncio.wait_for(browser.close(), timeout=10.0)
+        except asyncio.TimeoutError:
+            print("Browser close timeout, forcing close")
             try:
-                if page in active_pages:
-                    active_pages.remove(page)
-                if page in page_creation_times:
-                    del page_creation_times[page]
-                await page.close()
-            except Exception as e:
-                print(f"Error closing page: {e}")
+                await browser.close(force=True)
+            except:
+                pass
+        except Exception as e:
+            print(f"Error during browser cleanup: {e}")
 
-        # Close browser
-        await browser.close()
         print(f"Browser cleaned up. Total active browsers: {len(active_browsers)}")
     except Exception as e:
         print(f"Error during browser cleanup: {e}")
 
 async def check_browser_health():
-    """Check browser health and recycle if needed"""
+    """Check browser health and recycle if needed with improved efficiency"""
+    global browser_health_check_task
+
     while True:
         try:
-            # Sleep for 2 minutes between checks (reduced from 5 minutes)
-            await asyncio.sleep(120)
+            # Sleep for 1 minute between checks (reduced from 2 minutes)
+            await asyncio.sleep(60)
+
+            # Monitor memory usage
+            await monitor_memory_usage()
 
             # First, force cleanup old instances
             await force_cleanup_old_instances()
 
-            async with browser_lock:
-                # Get all browsers from the pool
-                browsers = []
-                while not browser_pool.empty():
-                    try:
-                        browsers.append(await browser_pool.get_nowait())
-                    except asyncio.QueueEmpty:
-                        break
-
-                # Check each browser
-                for browser in browsers:
-                    try:
-                        # Check if browser is too old
-                        if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
-                            print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
-                            await cleanup_browser(browser)
-                            browser = await create_browser()
-                        else:
-                            # Quick health check
-                            contexts = browser.contexts
-                            if contexts:
-                                pages = contexts[0].pages
-
-                        # Put back in pool if healthy
-                        if not browser_pool.full():
-                            await browser_pool.put(browser)
-                        else:
-                            # Pool is full, cleanup this browser
-                            await cleanup_browser(browser)
-                    except Exception as e:
-                        print(f"Error checking browser health: {e}")
-                        # Cleanup the problematic browser
+            # Use a timeout for the entire health check operation
+            async with asyncio.timeout(30):  # 30 second timeout for health check
+                async with browser_lock:
+                    # Get all browsers from the pool
+                    browsers = []
+                    while not browser_pool.empty():
                         try:
-                            await cleanup_browser(browser)
-                        except:
-                            pass
+                            browsers.append(await browser_pool.get_nowait())
+                        except asyncio.QueueEmpty:
+                            break
 
-                # Create new browsers if pool is empty
-                while browser_pool.qsize() < MAX_BROWSERS:
-                    try:
-                        browser = await create_browser()
-                        await browser_pool.put(browser)
-                    except Exception as e:
-                        print(f"Error creating browser for pool: {e}")
-                        break
+                    # Check each browser
+                    for browser in browsers:
+                        try:
+                            # Check if browser is too old
+                            if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
+                                print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
+                                await cleanup_browser(browser)
+                                browser = await create_browser()
+                            else:
+                                # Quick health check - just verify browser is still responsive
+                                try:
+                                    contexts = browser.contexts
+                                    if not contexts:
+                                        # Browser has no contexts, might be dead
+                                        print("Browser has no contexts, recycling")
+                                        await cleanup_browser(browser)
+                                        browser = await create_browser()
+                                except Exception as e:
+                                    print(f"Browser health check failed: {e}")
+                                    await cleanup_browser(browser)
+                                    browser = await create_browser()
 
+                            # Put back in pool if healthy
+                            if not browser_pool.full():
+                                await browser_pool.put(browser)
+                            else:
+                                # Pool is full, cleanup this browser
+                                await cleanup_browser(browser)
+                        except Exception as e:
+                            print(f"Error checking browser health: {e}")
+                            # Cleanup the problematic browser
+                            try:
+                                await cleanup_browser(browser)
+                            except:
+                                pass
+
+        except asyncio.TimeoutError:
+            print("Browser health check timed out, continuing...")
         except Exception as e:
             print(f"Error in browser health check: {e}")
-            await asyncio.sleep(60)  # Wait before retrying
+            # Continue the loop even if there's an error
+            await asyncio.sleep(30)  # Wait a bit before retrying
 
 async def force_cleanup_old_instances():
     """Force cleanup old browser and page instances based on timeout"""
@@ -266,58 +353,95 @@ async def force_cleanup_old_instances():
     print(f"Force cleanup completed: {cleaned_browsers} browsers, {cleaned_pages} pages cleaned")
 
 async def force_cleanup_all_browsers():
-    """Force cleanup all browsers - useful for emergency situations"""
+    """Force cleanup all browsers - useful for emergency situations with improved efficiency"""
     print("Force cleaning up all browsers...")
 
-    async with browser_lock:
-        # Clean up browsers in pool
-        while not browser_pool.empty():
-            try:
-                browser = await browser_pool.get_nowait()
-                await cleanup_browser(browser)
-            except asyncio.QueueEmpty:
-                break
+    try:
+        async with asyncio.timeout(20):  # 20 second timeout for force cleanup
+            async with browser_lock:
+                # Clean up browsers in pool
+                while not browser_pool.empty():
+                    try:
+                        browser = await asyncio.wait_for(browser_pool.get_nowait(), timeout=1.0)
+                        await asyncio.wait_for(cleanup_browser(browser), timeout=5.0)
+                    except (asyncio.TimeoutError, asyncio.QueueEmpty):
+                        break
+                    except Exception as e:
+                        print(f"Error cleaning up browser from pool: {e}")
 
-        # Clean up any remaining active browsers
-        for browser in list(active_browsers):
-            try:
-                await cleanup_browser(browser)
-            except Exception as e:
-                print(f"Error force cleaning browser: {e}")
+                # Clean up any remaining active browsers
+                for browser in list(active_browsers):
+                    try:
+                        await asyncio.wait_for(cleanup_browser(browser), timeout=5.0)
+                    except (asyncio.TimeoutError, Exception) as e:
+                        print(f"Error force cleaning browser: {e}")
 
-    print("Force cleanup completed")
+        print("Force cleanup completed")
+    except asyncio.TimeoutError:
+        print("Force cleanup timed out")
+    except Exception as e:
+        print(f"Error during force cleanup: {e}")
 
 @app.on_event("startup")
 async def init_browser_pool():
-    """Initialize the browser pool on startup"""
+    """Initialize the browser pool on startup with improved error handling"""
     print("Initializing browser pool...")
 
-    # Start browser health check task
-    asyncio.create_task(check_browser_health())
+    try:
+        # Start browser health check task
+        global browser_health_check_task
+        browser_health_check_task = asyncio.create_task(check_browser_health())
 
-    # Pre-populate pool with initial browsers
-    for _ in range(min(2, MAX_BROWSERS)):
-        try:
-            browser = await create_browser()
-            await browser_pool.put(browser)
-        except Exception as e:
-            print(f"Error creating initial browser: {e}")
+        # Pre-populate pool with initial browsers (reduced from 2 to 1)
+        for _ in range(min(1, MAX_BROWSERS)):
+            try:
+                browser = await asyncio.wait_for(create_browser(), timeout=30.0)
+                await asyncio.wait_for(browser_pool.put(browser), timeout=5.0)
+                print(f"Created initial browser {_ + 1}")
+            except (asyncio.TimeoutError, Exception) as e:
+                print(f"Error creating initial browser {_ + 1}: {e}")
+                break
 
-    print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
+        print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
+    except Exception as e:
+        print(f"Error during browser pool initialization: {e}")
+        # Don't fail startup, but log the error
 
 @app.on_event("shutdown")
 async def cleanup_browser_pool():
-    """Cleanup browser pool on shutdown"""
+    """Cleanup browser pool on shutdown with improved error handling"""
     print("Cleaning up browser pool...")
-    await force_cleanup_all_browsers()
 
-    # Stop playwright instance
-    global playwright_instance
-    if playwright_instance:
-        await playwright_instance.stop()
-        playwright_instance = None
+    try:
+        # Cancel health check task
+        global browser_health_check_task
+        if browser_health_check_task and not browser_health_check_task.done():
+            browser_health_check_task.cancel()
+            try:
+                await browser_health_check_task
+            except asyncio.CancelledError:
+                pass
 
-    print("Browser pool cleanup completed")
+        # Force cleanup all browsers with timeout
+        await asyncio.wait_for(force_cleanup_all_browsers(), timeout=30.0)
+
+        # Stop playwright instance
+        global playwright_instance
+        if playwright_instance:
+            try:
+                await asyncio.wait_for(playwright_instance.stop(), timeout=10.0)
+            except asyncio.TimeoutError:
+                print("Playwright stop timeout")
+            except Exception as e:
+                print(f"Error stopping playwright: {e}")
+            finally:
+                playwright_instance = None
+
+        print("Browser pool cleanup completed")
+    except asyncio.TimeoutError:
+        print("Browser pool cleanup timed out")
+    except Exception as e:
+        print(f"Error during browser pool cleanup: {e}")
 
 def signal_handler(signum, frame):
     """Handle shutdown signals"""
@@ -329,6 +453,21 @@ def signal_handler(signum, frame):
 signal.signal(signal.SIGINT, signal_handler)
 signal.signal(signal.SIGTERM, signal_handler)
 
+# Add memory monitoring
+async def monitor_memory_usage():
+    """Monitor memory usage and trigger cleanup if needed"""
+    try:
+        memory = psutil.virtual_memory()
+        if memory.percent > 85:  # If memory usage is above 85%
+            print(f"High memory usage detected: {memory.percent}%")
+            # Force garbage collection
+            gc.collect()
+            # Force cleanup old instances
+            await force_cleanup_old_instances()
+            print("Memory cleanup completed")
+    except Exception as e:
+        print(f"Error monitoring memory: {e}")
+
 def init_db():
     """Initialize the SQLite database"""
     conn = sqlite3.connect('/db/cache.db')
@@ -363,9 +502,9 @@ def init_db():
     print("Database initialized")
 
 def get_cached_data(url, route):
-    """Get cached data for a URL and route"""
+    """Get cached data for a URL and route with improved error handling"""
     try:
-        conn = sqlite3.connect('/db/cache.db')
+        conn = sqlite3.connect('/db/cache.db', timeout=10.0)  # Add timeout
         cursor = conn.cursor()
 
         cursor.execute('''
@@ -384,24 +523,36 @@ def get_cached_data(url, route):
             if datetime.now() - created_time < timedelta(hours=CACHE_EXPIRY_HOURS):
                 return json.loads(data)
 
+        return None
+    except sqlite3.OperationalError as e:
+        print(f"Database operational error getting cached data: {e}")
         return None
     except Exception as e:
         print(f"Error getting cached data: {e}")
         return None
 
 def save_to_cache(url, route, data):
-    """Save data to cache"""
+    """Save data to cache with improved error handling"""
     try:
-        conn = sqlite3.connect('/db/cache.db')
+        conn = sqlite3.connect('/db/cache.db', timeout=10.0)  # Add timeout
         cursor = conn.cursor()
 
+        # Limit data size to prevent memory issues
+        data_str = json.dumps(data)
+        if len(data_str) > 10 * 1024 * 1024:  # 10MB limit
+            print(f"Data too large for cache: {len(data_str)} bytes")
+            conn.close()
+            return
+
         cursor.execute('''
             INSERT OR REPLACE INTO cache (url, route, data, created_at)
             VALUES (?, ?, ?, CURRENT_TIMESTAMP)
-        ''', (url, route, json.dumps(data)))
+        ''', (url, route, data_str))
 
         conn.commit()
         conn.close()
+    except sqlite3.OperationalError as e:
+        print(f"Database operational error saving to cache: {e}")
     except Exception as e:
         print(f"Error saving to cache: {e}")
 
@@ -458,46 +609,60 @@ async def health_check():
     return {"status": "healthy"}
 
 async def safe_browser_operation(url, operation_func):
-    """Safely perform a browser operation with proper resource management"""
+    """Safely perform a browser operation with proper resource management and timeouts"""
     async with operation_semaphore:
         browser = None
         context = None
         page = None
 
         try:
-            # Get browser from pool or create new one
+            # Get browser from pool or create new one with timeout
             try:
-                browser = await asyncio.wait_for(browser_pool.get(), timeout=10.0)
+                browser = await asyncio.wait_for(browser_pool.get(), timeout=5.0)
             except asyncio.TimeoutError:
                 print("Timeout getting browser from pool, creating new one")
-                browser = await create_browser()
+                browser = await asyncio.wait_for(create_browser(), timeout=30.0)
 
             # Create stealth context with Cloudflare bypass
-            context = await CloudflareBypass.setup_stealth_context(browser)
+            context = await asyncio.wait_for(
+                CloudflareBypass.setup_stealth_context(browser),
+                timeout=10.0
+            )
 
-            page = await context.new_page()
+            page = await asyncio.wait_for(context.new_page(), timeout=10.0)
+            page.set_default_timeout(PAGE_TIMEOUT * 1000)  # Convert to milliseconds
 
             # Track page creation time for force cleanup
             page_creation_times[page] = time.time()
             active_pages.add(page)
 
             # Setup stealth page with additional measures
-            await CloudflareBypass.setup_stealth_page(page)
+            await asyncio.wait_for(
+                CloudflareBypass.setup_stealth_page(page),
+                timeout=5.0
+            )
 
-            # Set up request interception for better performance (but allow essential resources)
+            # Set up optimized request interception for better performance
             await page.route("**/*", lambda route: route.abort()
-                           if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
+                           if route.request.resource_type in ['image', 'stylesheet', 'font', 'media', 'script']
+                           and not route.request.url.startswith('data:')
                            else route.continue_())
 
-            # Perform the operation
-            result = await operation_func(page)
+            # Perform the operation with timeout
+            result = await asyncio.wait_for(
+                operation_func(page),
+                timeout=BROWSER_OPERATION_TIMEOUT
+            )
             return result
 
+        except asyncio.TimeoutError as e:
+            print(f"Browser operation timeout for {url}: {e}")
+            raise HTTPException(status_code=408, detail=f"Operation timeout: {str(e)}")
         except Exception as e:
-            print(f"Error in browser operation: {e}")
+            print(f"Error in browser operation for {url}: {e}")
             raise
         finally:
-            # Cleanup
+            # Cleanup with timeouts
             if page:
                 try:
                     # Remove from tracking
@@ -505,23 +670,38 @@ async def safe_browser_operation(url, operation_func):
                         active_pages.remove(page)
                     if page in page_creation_times:
                         del page_creation_times[page]
-                    await page.close()
-                except:
-                    pass
+                    await asyncio.wait_for(page.close(), timeout=5.0)
+                except (asyncio.TimeoutError, Exception):
+                    try:
+                        await page.close(force=True)
+                    except:
+                        pass
             if context:
                 try:
-                    await context.close()
-                except:
+                    await asyncio.wait_for(context.close(), timeout=5.0)
+                except (asyncio.TimeoutError, Exception):
                     pass
             if browser:
                 try:
-                    # Return browser to pool if it's still healthy
+                    # Return browser to pool if it's still healthy and pool isn't full
                     if not browser_pool.full():
-                        await browser_pool.put(browser)
+                        # Quick health check before returning to pool
+                        try:
+                            contexts = browser.contexts
+                            if contexts:
+                                await asyncio.wait_for(browser_pool.put(browser), timeout=2.0)
+                            else:
+                                await cleanup_browser(browser)
+                        except (asyncio.TimeoutError, Exception):
+                            await cleanup_browser(browser)
                     else:
                         await cleanup_browser(browser)
-                except:
-                    pass
+                except Exception as e:
+                    print(f"Error returning browser to pool: {e}")
+                    try:
+                        await cleanup_browser(browser)
+                    except:
+                        pass
 
 @app.get("/")
 async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
@@ -832,8 +1012,13 @@ async def system_status(x_api_key: Optional[str] = Header(None)):
                 "cpu_percent": cpu_percent,
                 "memory_percent": memory.percent,
                 "memory_available_gb": round(memory.available / (1024**3), 2),
+                "memory_used_gb": round(memory.used / (1024**3), 2),
+                "memory_free_gb": round(memory.free / (1024**3), 2),
+                "memory_total_gb": round(memory.total / (1024**3), 2),
                 "disk_percent": disk.percent,
-                "disk_free_gb": round(disk.free / (1024**3), 2)
+                "disk_free_gb": round(disk.free / (1024**3), 2),
+                "disk_used_gb": round(disk.used / (1024**3), 2),
+                "disk_total_gb": round(disk.total / (1024**3), 2)
             },
             "browser_pool": {
                 "pool_size": pool_size,
@@ -841,7 +1026,11 @@ async def system_status(x_api_key: Optional[str] = Header(None)):
                 "active_browsers": active_browser_count,
                 "active_pages": active_page_count,
                 "browser_ttl_seconds": BROWSER_TTL,
-                "instance_timeout_minutes": BROWSER_INSTANCE_TIMEOUT_MINUTES
+                "instance_timeout_minutes": BROWSER_INSTANCE_TIMEOUT_MINUTES,
+                "max_concurrent_operations": MAX_CONCURRENT_OPERATIONS,
+                "browser_operation_timeout_seconds": BROWSER_OPERATION_TIMEOUT,
+                "page_timeout_seconds": PAGE_TIMEOUT,
+                "pool_health": "healthy" if pool_size > 0 and active_browser_count <= MAX_BROWSERS else "warning"
             },
             "cache": {
                 "total_entries": cache_count,
@@ -850,6 +1039,10 @@ async def system_status(x_api_key: Optional[str] = Header(None)):
             "rate_limiting": {
                 "requests_per_minute": RATE_LIMIT_MINUTE,
                 "window_seconds": RATE_LIMIT_WINDOW
+            },
+            "performance": {
+                "memory_pressure": "high" if memory.percent > 85 else "normal" if memory.percent > 70 else "low",
+                "cpu_pressure": "high" if cpu_percent > 80 else "normal" if cpu_percent > 50 else "low"
             }
         }
     except Exception as e: