potential performance upgrade

Dockers/puppeteer-api/app/utils/browser_utils.py

@@ -12,42 +12,49 @@ async def wait_for_network_idle(page):
     await page.wait_for_load_state('networkidle')
 
 async def safe_browser_operation(url, operation_func):
-    """Safely perform browser operations with proper cleanup"""
+    """Safely perform browser operations with proper cleanup and timeouts"""
     browser = None
     context = None
     page = None
     playwright = None
 
     try:
-        # Get or create playwright instance
-        playwright = await async_playwright().start()
+        # Get or create playwright instance with timeout
+        playwright = await asyncio.wait_for(async_playwright().start(), timeout=30.0)
 
-        browser = await playwright.chromium.launch(
+        browser = await asyncio.wait_for(playwright.chromium.launch(
             headless=True,
-            args=['--no-sandbox', '--disable-setuid-sandbox'],
-        )
+            args=['--no-sandbox', '--disable-setuid-sandbox', '--max_old_space_size=256'],
+        ), timeout=30.0)
 
-        # Create context and page
-        context = await browser.new_context(
+        # Create context and page with timeout
+        context = await asyncio.wait_for(browser.new_context(
             user_agent=CUSTOM_USER_AGENT,
             viewport={'width': 1920, 'height': 1080},
             ignore_https_errors=True,
-        )
+        ), timeout=10.0)
 
-        page = await context.new_page()
-        page.set_default_timeout(30000)
+        page = await asyncio.wait_for(context.new_page(), timeout=10.0)
+        page.set_default_timeout(30000)  # 30 second timeout
 
         # Track page creation time for force cleanup
         page_creation_times[page] = time.time()
         active_pages.add(page)
 
-        # Call the operation function that uses the page
-        result = await operation_func(page)
+        # Call the operation function that uses the page with timeout
+        result = await asyncio.wait_for(operation_func(page), timeout=60.0)
 
         return result
 
+    except asyncio.TimeoutError as e:
+        print(f"Browser operation timeout for {url}: {e}")
+        return {
+            "status": "error",
+            "error": f"Operation timeout: {str(e)}",
+            "url": url
+        }
     except Exception as e:
-        print(f"Error during browser operation: {e}")
+        print(f"Error during browser operation for {url}: {e}")
         # Return error result instead of re-raising to allow graceful handling
         return {
             "status": "error",
@@ -55,7 +62,7 @@ async def safe_browser_operation(url, operation_func):
             "url": url
         }
     finally:
-        # Cleanup page tracking
+        # Cleanup page tracking with timeout
         if page:
             try:
                 # Remove from tracking
@@ -63,29 +70,38 @@ async def safe_browser_operation(url, operation_func):
                     active_pages.remove(page)
                 if page in page_creation_times:
                     del page_creation_times[page]
-                await page.close()
-            except Exception as e:
+                await asyncio.wait_for(page.close(), timeout=5.0)
+            except (asyncio.TimeoutError, Exception) as e:
                 print(f"Error closing page: {e}")
+                try:
+                    await page.close(force=True)
+                except:
+                    pass
 
-        # Ensure context is closed properly
+        # Ensure context is closed properly with timeout
         if context:
             try:
-                await context.close()
-            except Exception as e:
+                await asyncio.wait_for(context.close(), timeout=5.0)
+            except (asyncio.TimeoutError, Exception) as e:
                 print(f"Error closing context: {e}")
 
-        # Ensure browser is closed properly
+        # Ensure browser is closed properly with timeout
         if browser:
             try:
-                await browser.close()
+                await asyncio.wait_for(browser.close(), timeout=10.0)
+            except asyncio.TimeoutError:
+                try:
+                    await browser.close(force=True)
+                except:
+                    pass
             except Exception as e:
                 print(f"Error closing browser: {e}")
 
-        # Ensure playwright is closed properly - THIS IS THE KEY FIX
+        # Ensure playwright is closed properly with timeout - THIS IS THE KEY FIX
         if playwright:
             try:
-                await playwright.stop()
-            except Exception as e:
+                await asyncio.wait_for(playwright.stop(), timeout=10.0)
+            except (asyncio.TimeoutError, Exception) as e:
                 print(f"Error closing playwright: {e}")
 
 async def force_cleanup_old_pages():

Dockers/puppeteer-api/app/utils/cloudflare_bypass.py

@@ -273,73 +273,101 @@ class CloudflareBypass:
         """)
 
     @staticmethod
-    async def handle_cloudflare_challenge(page: Page, max_retries: int = 3) -> bool:
-        """Handle Cloudflare challenges and wait for them to complete"""
+    async def handle_cloudflare_challenge(page: Page, max_retries: int = 2) -> bool:
+        """Handle Cloudflare challenge with improved efficiency"""
         for attempt in range(max_retries):
             try:
-                # Check for Cloudflare challenge
-                cloudflare_selectors = [
-                    '#challenge-form',
-                    '#cf-please-wait',
-                    '.cf-browser-verification',
-                    '#cf-wrapper',
-                    'iframe[src*="cloudflare"]'
-                ]
+                print(f"Handling Cloudflare challenge (attempt {attempt + 1}/{max_retries})")
 
-                for selector in cloudflare_selectors:
-                    try:
-                        element = await page.wait_for_selector(selector, timeout=5000)
-                        if element:
-                            print(f"Cloudflare challenge detected on attempt {attempt + 1}")
-                            # Wait for challenge to complete
-                            await asyncio.sleep(5 + random.uniform(2, 8))
-                            break
-                    except:
-                        continue
-
-                # Wait for page to load completely
-                await page.wait_for_load_state('networkidle', timeout=30000)
-
-                # Check if we're past the challenge
-                title = await page.title()
-                if 'Cloudflare' not in title and 'challenge' not in title.lower():
-                    print("Successfully bypassed Cloudflare challenge")
+                # Wait for challenge form with shorter timeout
+                try:
+                    await page.wait_for_selector('#challenge-form', timeout=10000)  # 10 seconds
+                except:
+                    # If no challenge form, might already be bypassed
                     return True
 
-                # Add some random delay
-                await asyncio.sleep(random.uniform(3, 8))
+                # Wait for the challenge to complete with shorter timeout
+                try:
+                    await page.wait_for_function('''
+                        () => {
+                            return !document.querySelector('#challenge-form') &&
+                                   !document.querySelector('#cf-wrapper') &&
+                                   !document.querySelector('#cf-please-wait') &&
+                                   !document.querySelector('.cf-browser-verification');
+                        }
+                    ''', timeout=15000)  # 15 seconds
+
+                    print("Cloudflare challenge completed")
+                    return True
+
+                except Exception as e:
+                    print(f"Challenge completion timeout (attempt {attempt + 1}): {e}")
+
+                    # Try to click any "Verify" buttons
+                    try:
+                        verify_buttons = await page.query_selector_all('button[type="submit"], input[type="submit"]')
+                        for button in verify_buttons:
+                            try:
+                                await button.click(timeout=5000)
+                                await asyncio.sleep(2)
+                            except:
+                                continue
+                    except:
+                        pass
+
+                # Short delay before retry
+                if attempt < max_retries - 1:
+                    await asyncio.sleep(2)
 
             except Exception as e:
                 print(f"Error handling Cloudflare challenge (attempt {attempt + 1}): {e}")
                 if attempt < max_retries - 1:
-                    await asyncio.sleep(random.uniform(5, 15))
+                    await asyncio.sleep(2)
 
+        print("Failed to bypass Cloudflare challenge after all attempts")
         return False
 
     @staticmethod
     async def navigate_with_stealth(page: Page, url: str) -> bool:
-        """Navigate to URL with stealth measures and handle Cloudflare"""
+        """Navigate to URL with stealth measures and improved performance"""
         try:
-            # Add random delay before navigation
-            await asyncio.sleep(random.uniform(1, 3))
+            # Set shorter timeouts for better performance
+            page.set_default_timeout(30000)  # 30 seconds
+            page.set_default_navigation_timeout(30000)  # 30 seconds
 
-            # Navigate to the URL
-            await page.goto(url, wait_until='domcontentloaded')
+            # Navigate with stealth measures
+            response = await page.goto(url, wait_until='domcontentloaded')  # Changed from 'networkidle' to 'domcontentloaded' for speed
 
-            # Handle Cloudflare challenge
-            success = await CloudflareBypass.handle_cloudflare_challenge(page)
-
-            if not success:
-                print("Failed to bypass Cloudflare challenge")
+            if not response:
                 return False
 
-            # Add human-like scrolling
-            await CloudflareBypass._simulate_human_scrolling(page)
+            # Quick check for Cloudflare challenge
+            if response.status == 403 or response.status == 503:
+                # Handle Cloudflare challenge
+                return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2)  # Reduced retries
+
+            # Check for Cloudflare indicators in the page
+            cloudflare_detected = await page.evaluate('''() => {
+                return document.title.toLowerCase().includes('cloudflare') ||
+                       !!document.querySelector('#challenge-form') ||
+                       !!document.querySelector('#cf-wrapper') ||
+                       !!document.querySelector('#cf-please-wait') ||
+                       !!document.querySelector('.cf-browser-verification');
+            }''')
+
+            if cloudflare_detected:
+                return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2)  # Reduced retries
+
+            # Wait for page to be ready (shorter wait)
+            try:
+                await page.wait_for_load_state('domcontentloaded', timeout=10000)  # 10 second timeout
+            except:
+                pass  # Continue even if load state timeout
 
             return True
 
         except Exception as e:
-            print(f"Error during stealth navigation: {e}")
+            print(f"Error during stealth navigation to {url}: {e}")
             return False
 
     @staticmethod