diff --git a/Dockers/puppeteer-api/app/utils/browser_utils.py b/Dockers/puppeteer-api/app/utils/browser_utils.py index 618bb6a..8c3faab 100644 --- a/Dockers/puppeteer-api/app/utils/browser_utils.py +++ b/Dockers/puppeteer-api/app/utils/browser_utils.py @@ -12,42 +12,49 @@ async def wait_for_network_idle(page): await page.wait_for_load_state('networkidle') async def safe_browser_operation(url, operation_func): - """Safely perform browser operations with proper cleanup""" + """Safely perform browser operations with proper cleanup and timeouts""" browser = None context = None page = None playwright = None try: - # Get or create playwright instance - playwright = await async_playwright().start() + # Get or create playwright instance with timeout + playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0) - browser = await playwright.chromium.launch( + browser = await asyncio.wait_for(playwright.chromium.launch( headless=True, - args=['--no-sandbox', '--disable-setuid-sandbox'], - ) + args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'], + ), timeout=30.0) - # Create context and page - context = await browser.new_context( + # Create context and page with timeout + context = await asyncio.wait_for(browser.new_context( user_agent=CUSTOM_USER_AGENT, viewport={'width': 1920, 'height': 1080}, ignore_https_errors=True, - ) + ), timeout=10.0) - page = await context.new_page() - page.set_default_timeout(30000) + page = await asyncio.wait_for(context.new_page(), timeout=10.0) + page.set_default_timeout(30000) # 30 second timeout # Track page creation time for force cleanup page_creation_times[page] = time.time() active_pages.add(page) - # Call the operation function that uses the page - result = await operation_func(page) + # Call the operation function that uses the page with timeout + result = await asyncio.wait_for(operation_func(page), timeout=60.0) return result + except asyncio.TimeoutError as e: + print(f"Browser operation timeout for {url}: {e}") + return { + "status": "error", + "error": f"Operation timeout: {str(e)}", + "url": url + } except Exception as e: - print(f"Error during browser operation: {e}") + print(f"Error during browser operation for {url}: {e}") # Return error result instead of re-raising to allow graceful handling return { "status": "error", @@ -55,7 +62,7 @@ async def safe_browser_operation(url, operation_func): "url": url } finally: - # Cleanup page tracking + # Cleanup page tracking with timeout if page: try: # Remove from tracking @@ -63,29 +70,38 @@ async def safe_browser_operation(url, operation_func): active_pages.remove(page) if page in page_creation_times: del page_creation_times[page] - await page.close() - except Exception as e: + await asyncio.wait_for(page.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: print(f"Error closing page: {e}") + try: + await page.close(force=True) + except: + pass - # Ensure context is closed properly + # Ensure context is closed properly with timeout if context: try: - await context.close() - except Exception as e: + await asyncio.wait_for(context.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: print(f"Error closing context: {e}") - # Ensure browser is closed properly + # Ensure browser is closed properly with timeout if browser: try: - await browser.close() + await asyncio.wait_for(browser.close(), timeout=10.0) + except asyncio.TimeoutError: + try: + await browser.close(force=True) + except: + pass except Exception as e: print(f"Error closing browser: {e}") - # Ensure playwright is closed properly - THIS IS THE KEY FIX + # Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX if playwright: try: - await playwright.stop() - except Exception as e: + await asyncio.wait_for(playwright.stop(), timeout=10.0) + except (asyncio.TimeoutError, Exception) as e: print(f"Error closing playwright: {e}") async def force_cleanup_old_pages(): diff --git a/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py b/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py index b8452ee..af7df20 100644 --- a/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py +++ b/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py @@ -273,73 +273,101 @@ class CloudflareBypass: """) @staticmethod - async def handle_cloudflare_challenge(page: Page, max_retries: int = 3) -> bool: - """Handle Cloudflare challenges and wait for them to complete""" + async def handle_cloudflare_challenge(page: Page, max_retries: int = 2) -> bool: + """Handle Cloudflare challenge with improved efficiency""" for attempt in range(max_retries): try: - # Check for Cloudflare challenge - cloudflare_selectors = [ - '#challenge-form', - '#cf-please-wait', - '.cf-browser-verification', - '#cf-wrapper', - 'iframe[src*="cloudflare"]' - ] + print(f"Handling Cloudflare challenge (attempt {attempt + 1}/{max_retries})") - for selector in cloudflare_selectors: - try: - element = await page.wait_for_selector(selector, timeout=5000) - if element: - print(f"Cloudflare challenge detected on attempt {attempt + 1}") - # Wait for challenge to complete - await asyncio.sleep(5 + random.uniform(2, 8)) - break - except: - continue - - # Wait for page to load completely - await page.wait_for_load_state('networkidle', timeout=30000) - - # Check if we're past the challenge - title = await page.title() - if 'Cloudflare' not in title and 'challenge' not in title.lower(): - print("Successfully bypassed Cloudflare challenge") + # Wait for challenge form with shorter timeout + try: + await page.wait_for_selector('#challenge-form', timeout=10000) # 10 seconds + except: + # If no challenge form, might already be bypassed return True - # Add some random delay - await asyncio.sleep(random.uniform(3, 8)) + # Wait for the challenge to complete with shorter timeout + try: + await page.wait_for_function(''' + () => { + return !document.querySelector('#challenge-form') && + !document.querySelector('#cf-wrapper') && + !document.querySelector('#cf-please-wait') && + !document.querySelector('.cf-browser-verification'); + } + ''', timeout=15000) # 15 seconds + + print("Cloudflare challenge completed") + return True + + except Exception as e: + print(f"Challenge completion timeout (attempt {attempt + 1}): {e}") + + # Try to click any "Verify" buttons + try: + verify_buttons = await page.query_selector_all('button[type="submit"], input[type="submit"]') + for button in verify_buttons: + try: + await button.click(timeout=5000) + await asyncio.sleep(2) + except: + continue + except: + pass + + # Short delay before retry + if attempt < max_retries - 1: + await asyncio.sleep(2) except Exception as e: print(f"Error handling Cloudflare challenge (attempt {attempt + 1}): {e}") if attempt < max_retries - 1: - await asyncio.sleep(random.uniform(5, 15)) + await asyncio.sleep(2) + print("Failed to bypass Cloudflare challenge after all attempts") return False @staticmethod async def navigate_with_stealth(page: Page, url: str) -> bool: - """Navigate to URL with stealth measures and handle Cloudflare""" + """Navigate to URL with stealth measures and improved performance""" try: - # Add random delay before navigation - await asyncio.sleep(random.uniform(1, 3)) + # Set shorter timeouts for better performance + page.set_default_timeout(30000) # 30 seconds + page.set_default_navigation_timeout(30000) # 30 seconds - # Navigate to the URL - await page.goto(url, wait_until='domcontentloaded') + # Navigate with stealth measures + response = await page.goto(url, wait_until='domcontentloaded') # Changed from 'networkidle' to 'domcontentloaded' for speed - # Handle Cloudflare challenge - success = await CloudflareBypass.handle_cloudflare_challenge(page) - - if not success: - print("Failed to bypass Cloudflare challenge") + if not response: return False - # Add human-like scrolling - await CloudflareBypass._simulate_human_scrolling(page) + # Quick check for Cloudflare challenge + if response.status == 403 or response.status == 503: + # Handle Cloudflare challenge + return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2) # Reduced retries + + # Check for Cloudflare indicators in the page + cloudflare_detected = await page.evaluate('''() => { + return document.title.toLowerCase().includes('cloudflare') || + !!document.querySelector('#challenge-form') || + !!document.querySelector('#cf-wrapper') || + !!document.querySelector('#cf-please-wait') || + !!document.querySelector('.cf-browser-verification'); + }''') + + if cloudflare_detected: + return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2) # Reduced retries + + # Wait for page to be ready (shorter wait) + try: + await page.wait_for_load_state('domcontentloaded', timeout=10000) # 10 second timeout + except: + pass # Continue even if load state timeout return True except Exception as e: - print(f"Error during stealth navigation: {e}") + print(f"Error during stealth navigation to {url}: {e}") return False @staticmethod diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index 14cc56a..6545822 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -22,6 +22,9 @@ from contextlib import asynccontextmanager from app.utils.browser_utils import force_cleanup_old_pages from app.utils.cloudflare_bypass import CloudflareBypass +# Add memory monitoring +import gc + app = FastAPI() # Get API key from environment variable @@ -39,9 +42,11 @@ CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *') CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36' # Browser pool configuration - Reduced for better resource management -MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '3')) # Reduced from 5 to 3 -BROWSER_TTL = int(os.getenv('BROWSER_TTL', '1800')) # Reduced from 3600 to 1800 seconds (30 minutes) -MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '5')) # New: limit concurrent operations +MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '2')) # Reduced from 3 to 2 for better resource management +BROWSER_TTL = int(os.getenv('BROWSER_TTL', '900')) # Reduced from 1800 to 900 seconds (15 minutes) +MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '3')) # Reduced from 5 to 3 +BROWSER_OPERATION_TIMEOUT = int(os.getenv('BROWSER_OPERATION_TIMEOUT', '60')) # New: timeout for browser operations +PAGE_TIMEOUT = int(os.getenv('PAGE_TIMEOUT', '30')) # New: timeout for page operations # Browser pool management browser_pool = Queue(maxsize=MAX_BROWSERS) # Add maxsize to prevent unbounded growth @@ -52,6 +57,7 @@ active_browsers = set() # Track active browsers active_pages = set() # Track active pages for force cleanup operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS) # Limit concurrent operations playwright_instance = None # Global playwright instance +browser_health_check_task = None # Track health check task # Rate limiting configuration RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60')) # requests per minute @@ -111,9 +117,48 @@ async def create_browser(): proxy_url = os.getenv('PROXY_URL') proxy_config = CloudflareBypass.get_proxy_config(proxy_url) + # Add memory and performance optimizations + browser_args = CloudflareBypass.get_stealth_args() + [ + '--memory-pressure-off', + '--max_old_space_size=256', # Reduced memory usage + '--disable-dev-shm-usage', + '--disable-gpu-sandbox', + '--disable-software-rasterizer', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-renderer-backgrounding', + '--disable-features=TranslateUI', + '--disable-ipc-flooding-protection', + '--disable-hang-monitor', + '--disable-prompt-on-repost', + '--disable-sync', + '--no-first-run', + '--no-default-browser-check', + '--no-pings', + '--no-zygote', + '--single-process', + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-component-extensions-with-background-pages', + '--disable-dev-shm-usage', + '--disable-features=TranslateUI', + '--disable-ipc-flooding-protection', + '--disable-renderer-backgrounding', + '--disable-sync', + '--force-color-profile=srgb', + '--metrics-recording-only', + '--no-first-run', + '--safebrowsing-disable-auto-update', + '--enable-automation', + '--password-store=basic', + '--use-mock-keychain', + ] + browser = await playwright_instance.chromium.launch( headless=True, - args=CloudflareBypass.get_stealth_args(), + args=browser_args, ignore_default_args=['--enable-automation'], **proxy_config ) @@ -124,10 +169,17 @@ async def create_browser(): return browser except Exception as e: print(f"Error creating browser: {e}") + # Clean up playwright instance if browser creation fails + if playwright_instance: + try: + await playwright_instance.stop() + playwright_instance = None + except: + pass raise async def cleanup_browser(browser): - """Safely cleanup a browser instance""" + """Safely cleanup a browser instance with improved error handling""" try: if browser in active_browsers: active_browsers.remove(browser) @@ -136,82 +188,117 @@ async def cleanup_browser(browser): del browser_creation_times[browser] # Close all pages first and clean up page tracking - pages = browser.contexts[0].pages if browser.contexts else [] - for page in pages: + try: + contexts = browser.contexts + for context in contexts: + pages = context.pages + for page in pages: + try: + if page in active_pages: + active_pages.remove(page) + if page in page_creation_times: + del page_creation_times[page] + await asyncio.wait_for(page.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: + print(f"Error closing page: {e}") + # Force close if normal close fails + try: + await page.close(force=True) + except: + pass + + # Close context + try: + await asyncio.wait_for(context.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: + print(f"Error closing context: {e}") + except Exception as e: + print(f"Error during context/page cleanup: {e}") + + # Close browser with timeout + try: + await asyncio.wait_for(browser.close(), timeout=10.0) + except asyncio.TimeoutError: + print("Browser close timeout, forcing close") try: - if page in active_pages: - active_pages.remove(page) - if page in page_creation_times: - del page_creation_times[page] - await page.close() - except Exception as e: - print(f"Error closing page: {e}") + await browser.close(force=True) + except: + pass + except Exception as e: + print(f"Error during browser cleanup: {e}") - # Close browser - await browser.close() print(f"Browser cleaned up. Total active browsers: {len(active_browsers)}") except Exception as e: print(f"Error during browser cleanup: {e}") async def check_browser_health(): - """Check browser health and recycle if needed""" + """Check browser health and recycle if needed with improved efficiency""" + global browser_health_check_task + while True: try: - # Sleep for 2 minutes between checks (reduced from 5 minutes) - await asyncio.sleep(120) + # Sleep for 1 minute between checks (reduced from 2 minutes) + await asyncio.sleep(60) + + # Monitor memory usage + await monitor_memory_usage() # First, force cleanup old instances await force_cleanup_old_instances() - async with browser_lock: - # Get all browsers from the pool - browsers = [] - while not browser_pool.empty(): - try: - browsers.append(await browser_pool.get_nowait()) - except asyncio.QueueEmpty: - break - - # Check each browser - for browser in browsers: - try: - # Check if browser is too old - if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL: - print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)") - await cleanup_browser(browser) - browser = await create_browser() - else: - # Quick health check - contexts = browser.contexts - if contexts: - pages = contexts[0].pages - - # Put back in pool if healthy - if not browser_pool.full(): - await browser_pool.put(browser) - else: - # Pool is full, cleanup this browser - await cleanup_browser(browser) - except Exception as e: - print(f"Error checking browser health: {e}") - # Cleanup the problematic browser + # Use a timeout for the entire health check operation + async with asyncio.timeout(30): # 30 second timeout for health check + async with browser_lock: + # Get all browsers from the pool + browsers = [] + while not browser_pool.empty(): try: - await cleanup_browser(browser) - except: - pass + browsers.append(await browser_pool.get_nowait()) + except asyncio.QueueEmpty: + break - # Create new browsers if pool is empty - while browser_pool.qsize() < MAX_BROWSERS: - try: - browser = await create_browser() - await browser_pool.put(browser) - except Exception as e: - print(f"Error creating browser for pool: {e}") - break + # Check each browser + for browser in browsers: + try: + # Check if browser is too old + if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL: + print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)") + await cleanup_browser(browser) + browser = await create_browser() + else: + # Quick health check - just verify browser is still responsive + try: + contexts = browser.contexts + if not contexts: + # Browser has no contexts, might be dead + print("Browser has no contexts, recycling") + await cleanup_browser(browser) + browser = await create_browser() + except Exception as e: + print(f"Browser health check failed: {e}") + await cleanup_browser(browser) + browser = await create_browser() + # Put back in pool if healthy + if not browser_pool.full(): + await browser_pool.put(browser) + else: + # Pool is full, cleanup this browser + await cleanup_browser(browser) + except Exception as e: + print(f"Error checking browser health: {e}") + # Cleanup the problematic browser + try: + await cleanup_browser(browser) + except: + pass + + except asyncio.TimeoutError: + print("Browser health check timed out, continuing...") except Exception as e: print(f"Error in browser health check: {e}") - await asyncio.sleep(60) # Wait before retrying + # Continue the loop even if there's an error + await asyncio.sleep(30) # Wait a bit before retrying async def force_cleanup_old_instances(): """Force cleanup old browser and page instances based on timeout""" @@ -266,58 +353,95 @@ async def force_cleanup_old_instances(): print(f"Force cleanup completed: {cleaned_browsers} browsers, {cleaned_pages} pages cleaned") async def force_cleanup_all_browsers(): - """Force cleanup all browsers - useful for emergency situations""" + """Force cleanup all browsers - useful for emergency situations with improved efficiency""" print("Force cleaning up all browsers...") - async with browser_lock: - # Clean up browsers in pool - while not browser_pool.empty(): - try: - browser = await browser_pool.get_nowait() - await cleanup_browser(browser) - except asyncio.QueueEmpty: - break + try: + async with asyncio.timeout(20): # 20 second timeout for force cleanup + async with browser_lock: + # Clean up browsers in pool + while not browser_pool.empty(): + try: + browser = await asyncio.wait_for(browser_pool.get_nowait(), timeout=1.0) + await asyncio.wait_for(cleanup_browser(browser), timeout=5.0) + except (asyncio.TimeoutError, asyncio.QueueEmpty): + break + except Exception as e: + print(f"Error cleaning up browser from pool: {e}") - # Clean up any remaining active browsers - for browser in list(active_browsers): - try: - await cleanup_browser(browser) - except Exception as e: - print(f"Error force cleaning browser: {e}") + # Clean up any remaining active browsers + for browser in list(active_browsers): + try: + await asyncio.wait_for(cleanup_browser(browser), timeout=5.0) + except (asyncio.TimeoutError, Exception) as e: + print(f"Error force cleaning browser: {e}") - print("Force cleanup completed") + print("Force cleanup completed") + except asyncio.TimeoutError: + print("Force cleanup timed out") + except Exception as e: + print(f"Error during force cleanup: {e}") @app.on_event("startup") async def init_browser_pool(): - """Initialize the browser pool on startup""" + """Initialize the browser pool on startup with improved error handling""" print("Initializing browser pool...") - # Start browser health check task - asyncio.create_task(check_browser_health()) + try: + # Start browser health check task + global browser_health_check_task + browser_health_check_task = asyncio.create_task(check_browser_health()) - # Pre-populate pool with initial browsers - for _ in range(min(2, MAX_BROWSERS)): - try: - browser = await create_browser() - await browser_pool.put(browser) - except Exception as e: - print(f"Error creating initial browser: {e}") + # Pre-populate pool with initial browsers (reduced from 2 to 1) + for _ in range(min(1, MAX_BROWSERS)): + try: + browser = await asyncio.wait_for(create_browser(), timeout=30.0) + await asyncio.wait_for(browser_pool.put(browser), timeout=5.0) + print(f"Created initial browser {_ + 1}") + except (asyncio.TimeoutError, Exception) as e: + print(f"Error creating initial browser {_ + 1}: {e}") + break - print(f"Browser pool initialized with {browser_pool.qsize()} browsers") + print(f"Browser pool initialized with {browser_pool.qsize()} browsers") + except Exception as e: + print(f"Error during browser pool initialization: {e}") + # Don't fail startup, but log the error @app.on_event("shutdown") async def cleanup_browser_pool(): - """Cleanup browser pool on shutdown""" + """Cleanup browser pool on shutdown with improved error handling""" print("Cleaning up browser pool...") - await force_cleanup_all_browsers() - # Stop playwright instance - global playwright_instance - if playwright_instance: - await playwright_instance.stop() - playwright_instance = None + try: + # Cancel health check task + global browser_health_check_task + if browser_health_check_task and not browser_health_check_task.done(): + browser_health_check_task.cancel() + try: + await browser_health_check_task + except asyncio.CancelledError: + pass - print("Browser pool cleanup completed") + # Force cleanup all browsers with timeout + await asyncio.wait_for(force_cleanup_all_browsers(), timeout=30.0) + + # Stop playwright instance + global playwright_instance + if playwright_instance: + try: + await asyncio.wait_for(playwright_instance.stop(), timeout=10.0) + except asyncio.TimeoutError: + print("Playwright stop timeout") + except Exception as e: + print(f"Error stopping playwright: {e}") + finally: + playwright_instance = None + + print("Browser pool cleanup completed") + except asyncio.TimeoutError: + print("Browser pool cleanup timed out") + except Exception as e: + print(f"Error during browser pool cleanup: {e}") def signal_handler(signum, frame): """Handle shutdown signals""" @@ -329,6 +453,21 @@ def signal_handler(signum, frame): signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) +# Add memory monitoring +async def monitor_memory_usage(): + """Monitor memory usage and trigger cleanup if needed""" + try: + memory = psutil.virtual_memory() + if memory.percent > 85: # If memory usage is above 85% + print(f"High memory usage detected: {memory.percent}%") + # Force garbage collection + gc.collect() + # Force cleanup old instances + await force_cleanup_old_instances() + print("Memory cleanup completed") + except Exception as e: + print(f"Error monitoring memory: {e}") + def init_db(): """Initialize the SQLite database""" conn = sqlite3.connect('/db/cache.db') @@ -363,9 +502,9 @@ def init_db(): print("Database initialized") def get_cached_data(url, route): - """Get cached data for a URL and route""" + """Get cached data for a URL and route with improved error handling""" try: - conn = sqlite3.connect('/db/cache.db') + conn = sqlite3.connect('/db/cache.db', timeout=10.0) # Add timeout cursor = conn.cursor() cursor.execute(''' @@ -384,24 +523,36 @@ def get_cached_data(url, route): if datetime.now() - created_time < timedelta(hours=CACHE_EXPIRY_HOURS): return json.loads(data) + return None + except sqlite3.OperationalError as e: + print(f"Database operational error getting cached data: {e}") return None except Exception as e: print(f"Error getting cached data: {e}") return None def save_to_cache(url, route, data): - """Save data to cache""" + """Save data to cache with improved error handling""" try: - conn = sqlite3.connect('/db/cache.db') + conn = sqlite3.connect('/db/cache.db', timeout=10.0) # Add timeout cursor = conn.cursor() + # Limit data size to prevent memory issues + data_str = json.dumps(data) + if len(data_str) > 10 * 1024 * 1024: # 10MB limit + print(f"Data too large for cache: {len(data_str)} bytes") + conn.close() + return + cursor.execute(''' INSERT OR REPLACE INTO cache (url, route, data, created_at) VALUES (?, ?, ?, CURRENT_TIMESTAMP) - ''', (url, route, json.dumps(data))) + ''', (url, route, data_str)) conn.commit() conn.close() + except sqlite3.OperationalError as e: + print(f"Database operational error saving to cache: {e}") except Exception as e: print(f"Error saving to cache: {e}") @@ -458,46 +609,60 @@ async def health_check(): return {"status": "healthy"} async def safe_browser_operation(url, operation_func): - """Safely perform a browser operation with proper resource management""" + """Safely perform a browser operation with proper resource management and timeouts""" async with operation_semaphore: browser = None context = None page = None try: - # Get browser from pool or create new one + # Get browser from pool or create new one with timeout try: - browser = await asyncio.wait_for(browser_pool.get(), timeout=10.0) + browser = await asyncio.wait_for(browser_pool.get(), timeout=5.0) except asyncio.TimeoutError: print("Timeout getting browser from pool, creating new one") - browser = await create_browser() + browser = await asyncio.wait_for(create_browser(), timeout=30.0) # Create stealth context with Cloudflare bypass - context = await CloudflareBypass.setup_stealth_context(browser) + context = await asyncio.wait_for( + CloudflareBypass.setup_stealth_context(browser), + timeout=10.0 + ) - page = await context.new_page() + page = await asyncio.wait_for(context.new_page(), timeout=10.0) + page.set_default_timeout(PAGE_TIMEOUT * 1000) # Convert to milliseconds # Track page creation time for force cleanup page_creation_times[page] = time.time() active_pages.add(page) # Setup stealth page with additional measures - await CloudflareBypass.setup_stealth_page(page) + await asyncio.wait_for( + CloudflareBypass.setup_stealth_page(page), + timeout=5.0 + ) - # Set up request interception for better performance (but allow essential resources) + # Set up optimized request interception for better performance await page.route("**/*", lambda route: route.abort() - if route.request.resource_type in ['image', 'stylesheet', 'font', 'media'] + if route.request.resource_type in ['image', 'stylesheet', 'font', 'media', 'script'] + and not route.request.url.startswith('data:') else route.continue_()) - # Perform the operation - result = await operation_func(page) + # Perform the operation with timeout + result = await asyncio.wait_for( + operation_func(page), + timeout=BROWSER_OPERATION_TIMEOUT + ) return result + except asyncio.TimeoutError as e: + print(f"Browser operation timeout for {url}: {e}") + raise HTTPException(status_code=408, detail=f"Operation timeout: {str(e)}") except Exception as e: - print(f"Error in browser operation: {e}") + print(f"Error in browser operation for {url}: {e}") raise finally: - # Cleanup + # Cleanup with timeouts if page: try: # Remove from tracking @@ -505,23 +670,38 @@ async def safe_browser_operation(url, operation_func): active_pages.remove(page) if page in page_creation_times: del page_creation_times[page] - await page.close() - except: - pass + await asyncio.wait_for(page.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception): + try: + await page.close(force=True) + except: + pass if context: try: - await context.close() - except: + await asyncio.wait_for(context.close(), timeout=5.0) + except (asyncio.TimeoutError, Exception): pass if browser: try: - # Return browser to pool if it's still healthy + # Return browser to pool if it's still healthy and pool isn't full if not browser_pool.full(): - await browser_pool.put(browser) + # Quick health check before returning to pool + try: + contexts = browser.contexts + if contexts: + await asyncio.wait_for(browser_pool.put(browser), timeout=2.0) + else: + await cleanup_browser(browser) + except (asyncio.TimeoutError, Exception): + await cleanup_browser(browser) else: await cleanup_browser(browser) - except: - pass + except Exception as e: + print(f"Error returning browser to pool: {e}") + try: + await cleanup_browser(browser) + except: + pass @app.get("/") async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): @@ -832,8 +1012,13 @@ async def system_status(x_api_key: Optional[str] = Header(None)): "cpu_percent": cpu_percent, "memory_percent": memory.percent, "memory_available_gb": round(memory.available / (1024**3), 2), + "memory_used_gb": round(memory.used / (1024**3), 2), + "memory_free_gb": round(memory.free / (1024**3), 2), + "memory_total_gb": round(memory.total / (1024**3), 2), "disk_percent": disk.percent, - "disk_free_gb": round(disk.free / (1024**3), 2) + "disk_free_gb": round(disk.free / (1024**3), 2), + "disk_used_gb": round(disk.used / (1024**3), 2), + "disk_total_gb": round(disk.total / (1024**3), 2) }, "browser_pool": { "pool_size": pool_size, @@ -841,7 +1026,11 @@ async def system_status(x_api_key: Optional[str] = Header(None)): "active_browsers": active_browser_count, "active_pages": active_page_count, "browser_ttl_seconds": BROWSER_TTL, - "instance_timeout_minutes": BROWSER_INSTANCE_TIMEOUT_MINUTES + "instance_timeout_minutes": BROWSER_INSTANCE_TIMEOUT_MINUTES, + "max_concurrent_operations": MAX_CONCURRENT_OPERATIONS, + "browser_operation_timeout_seconds": BROWSER_OPERATION_TIMEOUT, + "page_timeout_seconds": PAGE_TIMEOUT, + "pool_health": "healthy" if pool_size > 0 and active_browser_count <= MAX_BROWSERS else "warning" }, "cache": { "total_entries": cache_count, @@ -850,6 +1039,10 @@ async def system_status(x_api_key: Optional[str] = Header(None)): "rate_limiting": { "requests_per_minute": RATE_LIMIT_MINUTE, "window_seconds": RATE_LIMIT_WINDOW + }, + "performance": { + "memory_pressure": "high" if memory.percent > 85 else "normal" if memory.percent > 70 else "low", + "cpu_pressure": "high" if cpu_percent > 80 else "normal" if cpu_percent > 50 else "low" } } except Exception as e: