potential bypass cloudflare

Dockers/puppeteer-api/main.py

@@ -20,6 +20,7 @@ from app.config import BROWSER_INSTANCE_TIMEOUT_MINUTES
 from asyncio import Queue, Lock, Semaphore
 from contextlib import asynccontextmanager
 from app.utils.browser_utils import force_cleanup_old_pages
+from app.utils.cloudflare_bypass import CloudflareBypass
 
 app = FastAPI()
 
@@ -106,37 +107,15 @@ async def create_browser():
         if playwright_instance is None:
             playwright_instance = await async_playwright().start()
 
+        # Get proxy configuration from environment
+        proxy_url = os.getenv('PROXY_URL')
+        proxy_config = CloudflareBypass.get_proxy_config(proxy_url)
+
         browser = await playwright_instance.chromium.launch(
             headless=True,
-            args=[
-                '--no-sandbox',
-                '--disable-setuid-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-accelerated-2d-canvas',
-                '--disable-gpu',
-                '--disable-extensions',
-                '--disable-sync',
-                '--disable-background-networking',
-                '--disable-default-apps',
-                '--disable-translate',
-                '--disable-background-timer-throttling',
-                '--disable-backgrounding-occluded-windows',
-                '--disable-client-side-phishing-detection',
-                '--disable-features=site-per-process',
-                '--disable-hang-monitor',
-                '--disable-ipc-flooding-protection',
-                '--disable-popup-blocking',
-                '--disable-prompt-on-repost',
-                '--disable-renderer-backgrounding',
-                '--memory-pressure-off',
-                '--no-first-run',
-                '--safebrowsing-disable-auto-update',
-                '--max_old_space_size=512',  # Limit memory usage
-                '--single-process',  # Use single process to reduce resource usage
-                '--disable-web-security',
-                '--disable-features=VizDisplayCompositor',
-            ],
+            args=CloudflareBypass.get_stealth_args(),
             ignore_default_args=['--enable-automation'],
+            **proxy_config
         )
 
         browser_creation_times[browser] = time.time()
@@ -493,12 +472,8 @@ async def safe_browser_operation(url, operation_func):
                 print("Timeout getting browser from pool, creating new one")
                 browser = await create_browser()
 
-            # Create context and page
-            context = await browser.new_context(
-                user_agent=CUSTOM_USER_AGENT,
-                viewport={'width': 1920, 'height': 1080},
-                ignore_https_errors=True,
-            )
+            # Create stealth context with Cloudflare bypass
+            context = await CloudflareBypass.setup_stealth_context(browser)
 
             page = await context.new_page()
 
@@ -506,7 +481,10 @@ async def safe_browser_operation(url, operation_func):
             page_creation_times[page] = time.time()
             active_pages.add(page)
 
-            # Set up request interception for better performance
+            # Setup stealth page with additional measures
+            await CloudflareBypass.setup_stealth_page(page)
+
+            # Set up request interception for better performance (but allow essential resources)
             await page.route("**/*", lambda route: route.abort()
                            if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
                            else route.continue_())
@@ -562,9 +540,11 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
     try:
         async def visit_operation(page):
             try:
-                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
-                if not response:
-                    print(f"Warning: No response object returned for {decoded_url}")
+                # Use Cloudflare bypass navigation
+                success = await CloudflareBypass.navigate_with_stealth(page, decoded_url)
+
+                if not success:
+                    return {"status": "error", "error": "Failed to bypass Cloudflare protection", "url": decoded_url}
 
                 # Get page content
                 content = await page.content()
@@ -604,7 +584,11 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
     try:
         async def seo_operation(page):
             try:
-                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
+                # Use Cloudflare bypass navigation
+                success = await CloudflareBypass.navigate_with_stealth(page, decoded_url)
+
+                if not success:
+                    return {"status": "error", "error": "Failed to bypass Cloudflare protection", "url": decoded_url}
 
                 # Extract SEO information
                 seo_data = await page.evaluate('''() => {
@@ -688,7 +672,11 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
     try:
         async def meta_operation(page):
             try:
-                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
+                # Use Cloudflare bypass navigation
+                success = await CloudflareBypass.navigate_with_stealth(page, decoded_url)
+
+                if not success:
+                    return {"status": "error", "error": "Failed to bypass Cloudflare protection", "url": decoded_url}
 
                 # Extract meta tags using Playwright
                 meta_data = await page.evaluate('''() => {
@@ -896,6 +884,72 @@ async def force_cleanup_old(x_api_key: Optional[str] = Header(None)):
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
+@app.get("/test-cloudflare")
+async def test_cloudflare_bypass(url: str, x_api_key: Optional[str] = Header(None)):
+    """Test Cloudflare bypass functionality on a specific URL"""
+    # Validate API key
+    if not x_api_key or x_api_key != API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    # Decode URL if it's encoded
+    decoded_url = unquote(url)
+
+    try:
+        async def test_operation(page):
+            try:
+                # Use Cloudflare bypass navigation
+                success = await CloudflareBypass.navigate_with_stealth(page, decoded_url)
+
+                if not success:
+                    return {
+                        "status": "error",
+                        "error": "Failed to bypass Cloudflare protection",
+                        "url": decoded_url,
+                        "cloudflare_detected": True
+                    }
+
+                # Get page information
+                title = await page.title()
+                url_after_navigation = page.url
+
+                # Check for Cloudflare indicators
+                cloudflare_indicators = await page.evaluate('''() => {
+                    const indicators = {
+                        has_cloudflare_title: document.title.toLowerCase().includes('cloudflare'),
+                        has_challenge_form: !!document.querySelector('#challenge-form'),
+                        has_cf_wrapper: !!document.querySelector('#cf-wrapper'),
+                        has_please_wait: !!document.querySelector('#cf-please-wait'),
+                        has_browser_verification: !!document.querySelector('.cf-browser-verification')
+                    };
+                    return indicators;
+                }''')
+
+                return {
+                    "status": "success",
+                    "url": decoded_url,
+                    "final_url": url_after_navigation,
+                    "title": title,
+                    "cloudflare_bypassed": True,
+                    "cloudflare_indicators": cloudflare_indicators,
+                    "content_length": len(await page.content())
+                }
+
+            except Exception as e:
+                print(f"Error during Cloudflare test: {e}")
+                return {
+                    "status": "error",
+                    "error": str(e),
+                    "url": decoded_url,
+                    "cloudflare_detected": False
+                }
+
+        result = await safe_browser_operation(decoded_url, test_operation)
+        return result
+
+    except Exception as e:
+        print(f"Error testing Cloudflare bypass for {decoded_url}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
 @asynccontextmanager
 async def get_browser():
     """Context manager for getting a browser from the pool"""