diff --git a/Dockers/puppeteer-api/CLOUDFLARE_BYPASS_EXAMPLES.md b/Dockers/puppeteer-api/CLOUDFLARE_BYPASS_EXAMPLES.md new file mode 100644 index 0000000..5411639 --- /dev/null +++ b/Dockers/puppeteer-api/CLOUDFLARE_BYPASS_EXAMPLES.md @@ -0,0 +1,301 @@ +# Cloudflare Bypass Examples & Usage Guide + +This guide demonstrates how to use the advanced Cloudflare bypass features in the Playwright API. + +## ๐Ÿš€ Quick Start + +### Basic Usage + +All existing endpoints now automatically include Cloudflare bypass: + +```bash +# Visit a Cloudflare-protected site +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/?url=https://cloudflare-protected-site.com" + +# Extract SEO from protected site +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/seo?url=https://cloudflare-protected-site.com" + +# Extract meta tags from protected site +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/meta?url=https://cloudflare-protected-site.com" +``` + +### Test Cloudflare Bypass + +Use the dedicated test endpoint to verify bypass effectiveness: + +```bash +# Test bypass on a specific URL +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/test-cloudflare?url=https://cloudflare-protected-site.com" +``` + +## ๐Ÿ”ง Configuration Options + +### Environment Variables + +```bash +# Enable/disable Cloudflare bypass (default: true) +ENABLE_CLOUDFLARE_BYPASS=true + +# Optional proxy for additional stealth +PROXY_URL=http://proxy-server:8080 + +# Other existing variables still work +API_KEY=your-api-key +MAX_BROWSERS=3 +BROWSER_TTL=1800 +``` + +### Docker Example with Proxy + +```bash +docker run -d \ + --name playwright-api \ + -p 8000:8000 \ + -e API_KEY=your-api-key \ + -e ENABLE_CLOUDFLARE_BYPASS=true \ + -e PROXY_URL=http://your-proxy:8080 \ + -v /path/to/cache:/db \ + playwright-api +``` + +## ๐Ÿ“Š Response Examples + +### Successful Bypass + +```json +{ + "status": "success", + "url": "https://cloudflare-protected-site.com", + "final_url": "https://cloudflare-protected-site.com", + "title": "Protected Site - Home", + "cloudflare_bypassed": true, + "cloudflare_indicators": { + "has_cloudflare_title": false, + "has_challenge_form": false, + "has_cf_wrapper": false, + "has_please_wait": false, + "has_browser_verification": false + }, + "content_length": 45678 +} +``` + +### Failed Bypass + +```json +{ + "status": "error", + "error": "Failed to bypass Cloudflare protection", + "url": "https://cloudflare-protected-site.com", + "cloudflare_detected": true +} +``` + +## ๐Ÿงช Testing Examples + +### Using the Test Script + +```bash +# Test a known Cloudflare-protected site +python test_cloudflare_bypass.py \ + http://localhost:8000 \ + your-api-key \ + https://example-cloudflare-site.com + +# Test multiple sites +for site in "site1.com" "site2.com" "site3.com"; do + python test_cloudflare_bypass.py \ + http://localhost:8000 \ + your-api-key \ + "https://$site" +done +``` + +### Manual Testing with curl + +```bash +# Test bypass endpoint +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/test-cloudflare?url=https://example.com" | jq + +# Compare with regular endpoint +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/?url=https://example.com" | jq '.content | length' +``` + +## ๐Ÿ” Bypass Techniques Explained + +### 1. Browser Fingerprinting Protection + +The API automatically: + +- Removes `navigator.webdriver` property +- Overrides automation detection methods +- Spoofs browser plugins and languages +- Masks Chrome automation indicators + +### 2. Request Header Spoofing + +Headers automatically set: + +```http +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 +Accept-Language: en-US,en;q=0.9 +Accept-Encoding: gzip, deflate, br +Sec-Ch-Ua: "Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120" +Sec-Ch-Ua-Mobile: ?0 +Sec-Ch-Ua-Platform: "Windows" +Sec-Fetch-Dest: document +Sec-Fetch-Mode: navigate +Sec-Fetch-Site: none +Sec-Fetch-User: ?1 +Upgrade-Insecure-Requests: 1 +``` + +### 3. Human-like Behavior + +- Random mouse movements +- Natural scrolling patterns +- Realistic timing delays +- Page interaction simulation + +### 4. Challenge Detection + +Automatically detects and handles: + +- Cloudflare challenge forms +- Browser verification pages +- "Please wait" screens +- JavaScript challenges + +## ๐Ÿ› ๏ธ Advanced Usage + +### Custom User Agents + +The system rotates between modern user agents: + +- Chrome 120 on Windows +- Chrome 119 on Windows +- Chrome 120 on macOS +- Chrome 119 on macOS +- Chrome 120 on Linux +- Chrome 119 on Linux + +### Viewport Randomization + +Random viewport sizes to appear more human: + +- 1920x1080 (Full HD) +- 1366x768 (HD) +- 1536x864 (HD+) +- 1440x900 (WXGA+) +- 1280x720 (HD) + +### Proxy Integration + +For additional stealth, configure a proxy: + +```bash +# HTTP proxy +PROXY_URL=http://proxy-server:8080 + +# HTTPS proxy +PROXY_URL=https://proxy-server:8443 + +# SOCKS proxy +PROXY_URL=socks5://proxy-server:1080 +``` + +## ๐Ÿšจ Troubleshooting + +### Common Issues + +1. **Still getting blocked** + + - Try using a proxy: `PROXY_URL=http://your-proxy:8080` + - Increase delays by modifying the bypass module + - Check if the site has additional protection layers + +2. **Timeout errors** + + - Increase browser timeout: `BROWSER_TTL=3600` + - Check network connectivity + - Verify proxy configuration + +3. **Memory issues** + - Reduce browser pool: `MAX_BROWSERS=2` + - Increase cleanup frequency + - Monitor system resources + +### Debug Mode + +Enable detailed logging by checking the server logs: + +```bash +# View real-time logs +docker logs -f playwright-api + +# Check specific bypass attempts +docker logs playwright-api | grep -i cloudflare +``` + +## ๐Ÿ“ˆ Performance Tips + +### Optimization + +1. **Use caching**: All bypassed content is cached +2. **Batch requests**: Process multiple URLs efficiently +3. **Monitor resources**: Use `/status` endpoint +4. **Cleanup regularly**: Use `/force-cleanup-old` + +### Monitoring + +```bash +# Check system status +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/status" | jq + +# Monitor cache +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/cache/stats" | jq +``` + +## ๐Ÿ” Security Considerations + +### Best Practices + +1. **Use HTTPS**: Always use HTTPS for API communication +2. **Secure API keys**: Keep API keys secure and rotate regularly +3. **Rate limiting**: Respect rate limits to avoid detection +4. **Proxy rotation**: Use multiple proxies for high-volume scraping +5. **User agent rotation**: The system does this automatically + +### Legal Compliance + +- Always respect robots.txt +- Follow website terms of service +- Implement appropriate delays between requests +- Use for legitimate purposes only + +## ๐Ÿ“š Additional Resources + +- [Kameleo Cloudflare Bypass Guide](https://kameleo.io/blog/how-to-bypass-cloudflare-with-playwright) +- [Playwright Documentation](https://playwright.dev/) +- [Cloudflare Detection Methods](https://developers.cloudflare.com/bots/) + +## ๐Ÿค Contributing + +To improve the bypass techniques: + +1. Test with different Cloudflare configurations +2. Report successful/failed bypass attempts +3. Suggest new detection methods to counter +4. Contribute to the stealth scripts + +--- + +**Note**: Cloudflare bypass techniques are constantly evolving. This implementation includes the latest known methods, but Cloudflare may update their detection systems. Regular updates and testing are recommended. diff --git a/Dockers/puppeteer-api/README.md b/Dockers/puppeteer-api/README.md index b985d62..d9b0c0a 100644 --- a/Dockers/puppeteer-api/README.md +++ b/Dockers/puppeteer-api/README.md @@ -1,6 +1,80 @@ # Playwright API Server -A FastAPI-based server that uses Playwright to scrape websites and extract various types of data. +A FastAPI-based server that uses Playwright to scrape websites and extract various types of data. **Now with advanced Cloudflare bypass capabilities!** + +## ๐Ÿ›ก๏ธ Cloudflare Bypass Features (v4.0) + +This API now includes sophisticated Cloudflare bypass techniques based on the latest anti-detection methods: + +### Key Bypass Features + +- **๐Ÿ”„ Dynamic User Agents**: Rotates between modern, realistic user agents +- **๐ŸŽญ Stealth Mode**: Comprehensive browser fingerprint spoofing +- **๐Ÿค– Human-like Behavior**: Simulates real user interactions +- **๐Ÿ”’ Anti-Detection Scripts**: Injects scripts to bypass automation detection +- **๐ŸŒ Proxy Support**: Optional proxy configuration for additional stealth +- **โฑ๏ธ Challenge Handling**: Automatic Cloudflare challenge detection and waiting +- **๐Ÿ“ฑ Realistic Viewports**: Random viewport sizes to appear more human +- **๐ŸŒ Geolocation Spoofing**: Simulates realistic location data + +### Bypass Techniques Implemented + +1. **Browser Fingerprinting Protection** + + - Removes `navigator.webdriver` property + - Overrides automation detection methods + - Spoofs browser plugins and languages + - Masks Chrome automation indicators + +2. **Request Header Spoofing** + + - Modern `Sec-Ch-Ua` headers + - Realistic `Accept` and `Accept-Language` headers + - Proper `Sec-Fetch-*` headers + - Cache control headers + +3. **Human-like Behavior** + + - Random mouse movements + - Natural scrolling patterns + - Realistic timing delays + - Page interaction simulation + +4. **Challenge Detection & Handling** + - Automatic Cloudflare challenge detection + - Intelligent waiting for challenge completion + - Multiple retry attempts with backoff + - Success/failure reporting + +### Testing Cloudflare Bypass + +Use the new `/test-cloudflare` endpoint to test bypass effectiveness: + +```bash +# Test Cloudflare bypass on a protected site +curl -H "X-API-Key: your-api-key" \ + "http://localhost:8000/test-cloudflare?url=https://cloudflare-protected-site.com" +``` + +Response includes detailed bypass status: + +```json +{ + "status": "success", + "url": "https://example.com", + "final_url": "https://example.com", + "title": "Example Domain", + "cloudflare_bypassed": true, + "cloudflare_indicators": { + "has_cloudflare_title": false, + "has_challenge_form": false, + "has_cf_wrapper": false, + "has_please_wait": false, + "has_browser_verification": false + }, + "content_length": 12345 +} +``` ## Recent Migration (v3.0) @@ -83,6 +157,10 @@ CACHE_EXPIRY_HOURS=36 # Cache expiry in hours (default: 36) CLEANUP_CRON=0 3 * * * # Cache cleanup schedule (default: daily at 3 AM) RATE_LIMIT_MINUTE=60 # Requests per minute (default: 60) +# Cloudflare Bypass Configuration +ENABLE_CLOUDFLARE_BYPASS=true # Enable Cloudflare bypass (default: true) +PROXY_URL= # Optional proxy URL for additional stealth (e.g., http://proxy:8080) + # Database Configuration # If all PostgreSQL credentials are provided, PostgreSQL will be used # Otherwise, SQLite will be used as fallback @@ -97,9 +175,10 @@ POSTGRES_PASSWORD= # PostgreSQL password (optional) ### Core Endpoints -- `GET /` - Visit URL and get HTML content -- `GET /seo` - Extract SEO information -- `GET /meta` - Extract meta tags and Open Graph data +- `GET /` - Visit URL and get HTML content (with Cloudflare bypass) +- `GET /seo` - Extract SEO information (with Cloudflare bypass) +- `GET /meta` - Extract meta tags and Open Graph data (with Cloudflare bypass) +- `GET /test-cloudflare` - Test Cloudflare bypass functionality on a specific URL - `GET /resulting-url` - Get the final URL after navigation (handles redirects) ### Management Endpoints @@ -128,6 +207,9 @@ curl -H "X-API-Key: your-api-key" "http://localhost:8000/resulting-url?url=https # Get system status curl -H "X-API-Key: your-api-key" "http://localhost:8000/status" +# Test Cloudflare bypass +curl -H "X-API-Key: your-api-key" "http://localhost:8000/test-cloudflare?url=https://cloudflare-protected-site.com" + # Force cleanup old browser/page instances curl -X POST -H "X-API-Key: your-api-key" "http://localhost:8000/force-cleanup-old" ``` diff --git a/Dockers/puppeteer-api/app/config.py b/Dockers/puppeteer-api/app/config.py index 54bad03..995ca1a 100644 --- a/Dockers/puppeteer-api/app/config.py +++ b/Dockers/puppeteer-api/app/config.py @@ -14,8 +14,12 @@ CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *') # Get browser instance timeout from environment variable (default: 10 minutes) BROWSER_INSTANCE_TIMEOUT_MINUTES = int(os.getenv('BROWSER_INSTANCE_TIMEOUT_MINUTES', '10')) -# Define custom user agent -CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36' +# Define custom user agent (updated to modern version) +CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + +# Cloudflare bypass configuration +ENABLE_CLOUDFLARE_BYPASS = os.getenv('ENABLE_CLOUDFLARE_BYPASS', 'true').lower() == 'true' +PROXY_URL = os.getenv('PROXY_URL') # Optional proxy for additional stealth # Database configuration # Use PostgreSQL if credentials are provided, otherwise use SQLite diff --git a/Dockers/puppeteer-api/app/database.py b/Dockers/puppeteer-api/app/database.py index 9070566..32de901 100644 --- a/Dockers/puppeteer-api/app/database.py +++ b/Dockers/puppeteer-api/app/database.py @@ -236,10 +236,17 @@ class DatabaseManager: else: cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds - cursor.execute( - "SELECT data FROM cache WHERE url = %s AND route = %s AND timestamp > %s", - (url, route, cache_expiry) - ) + if self.db_type == "postgresql": + cursor.execute( + "SELECT data FROM cache WHERE url = %s AND route = %s AND timestamp > %s", + (url, route, cache_expiry) + ) + else: + cursor.execute( + "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?", + (url, route, cache_expiry) + ) + result = cursor.fetchone() conn.close() @@ -283,19 +290,34 @@ class DatabaseManager: expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60) - # Get count of entries to be deleted (non-pagination) - cursor.execute("SELECT COUNT(*) FROM cache WHERE route != %s AND timestamp < %s", ('pagination', expiry_timestamp)) - count_non_pagination = cursor.fetchone()[0] + if self.db_type == "postgresql": + # Get count of entries to be deleted (non-pagination) + cursor.execute("SELECT COUNT(*) FROM cache WHERE route != %s AND timestamp < %s", ('pagination', expiry_timestamp)) + count_non_pagination = cursor.fetchone()[0] - # Get count of pagination entries to be deleted - cursor.execute("SELECT COUNT(*) FROM cache WHERE route = %s AND timestamp < %s", ('pagination', pagination_expiry_timestamp)) - count_pagination = cursor.fetchone()[0] + # Get count of pagination entries to be deleted + cursor.execute("SELECT COUNT(*) FROM cache WHERE route = %s AND timestamp < %s", ('pagination', pagination_expiry_timestamp)) + count_pagination = cursor.fetchone()[0] - # Delete old non-pagination entries - cursor.execute("DELETE FROM cache WHERE route != %s AND timestamp < %s", ('pagination', expiry_timestamp)) + # Delete old non-pagination entries + cursor.execute("DELETE FROM cache WHERE route != %s AND timestamp < %s", ('pagination', expiry_timestamp)) - # Delete old pagination entries - cursor.execute("DELETE FROM cache WHERE route = %s AND timestamp < %s", ('pagination', pagination_expiry_timestamp)) + # Delete old pagination entries + cursor.execute("DELETE FROM cache WHERE route = %s AND timestamp < %s", ('pagination', pagination_expiry_timestamp)) + else: + # Get count of entries to be deleted (non-pagination) + cursor.execute("SELECT COUNT(*) FROM cache WHERE route != ? AND timestamp < ?", ('pagination', expiry_timestamp)) + count_non_pagination = cursor.fetchone()[0] + + # Get count of pagination entries to be deleted + cursor.execute("SELECT COUNT(*) FROM cache WHERE route = ? AND timestamp < ?", ('pagination', pagination_expiry_timestamp)) + count_pagination = cursor.fetchone()[0] + + # Delete old non-pagination entries + cursor.execute("DELETE FROM cache WHERE route != ? AND timestamp < ?", ('pagination', expiry_timestamp)) + + # Delete old pagination entries + cursor.execute("DELETE FROM cache WHERE route = ? AND timestamp < ?", ('pagination', pagination_expiry_timestamp)) conn.commit() conn.close() diff --git a/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py b/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py new file mode 100644 index 0000000..b8452ee --- /dev/null +++ b/Dockers/puppeteer-api/app/utils/cloudflare_bypass.py @@ -0,0 +1,381 @@ +import random +import time +import asyncio +from typing import Dict, List, Optional +from playwright.async_api import Browser, BrowserContext, Page + +class CloudflareBypass: + """Cloudflare bypass implementation based on Kameleo techniques""" + + # Modern, realistic user agents + MODERN_USER_AGENTS = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + ] + + # Realistic viewport sizes + VIEWPORT_SIZES = [ + {'width': 1920, 'height': 1080}, + {'width': 1366, 'height': 768}, + {'width': 1536, 'height': 864}, + {'width': 1440, 'height': 900}, + {'width': 1280, 'height': 720}, + ] + + # Common languages + LANGUAGES = [ + 'en-US,en;q=0.9', + 'en-GB,en;q=0.9', + 'en-CA,en;q=0.9', + 'en-AU,en;q=0.9', + ] + + @staticmethod + def get_random_user_agent() -> str: + """Get a random modern user agent""" + return random.choice(CloudflareBypass.MODERN_USER_AGENTS) + + @staticmethod + def get_random_viewport() -> Dict[str, int]: + """Get a random realistic viewport size""" + return random.choice(CloudflareBypass.VIEWPORT_SIZES) + + @staticmethod + def get_random_language() -> str: + """Get a random language preference""" + return random.choice(CloudflareBypass.LANGUAGES) + + @staticmethod + def get_stealth_args() -> List[str]: + """Get browser arguments for stealth mode""" + return [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--disable-gpu', + '--disable-extensions', + '--disable-sync', + '--disable-background-networking', + '--disable-default-apps', + '--disable-translate', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-client-side-phishing-detection', + '--disable-features=site-per-process', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--memory-pressure-off', + '--no-first-run', + '--safebrowsing-disable-auto-update', + '--max_old_space_size=512', + '--disable-web-security', + '--disable-features=VizDisplayCompositor', + # Additional stealth arguments + '--disable-blink-features=AutomationControlled', + '--disable-web-security', + '--disable-features=VizDisplayCompositor', + '--disable-ipc-flooding-protection', + '--disable-renderer-backgrounding', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-default-apps', + '--disable-domain-reliability', + '--disable-features=AudioServiceOutOfProcess', + '--disable-hang-monitor', + '--disable-prompt-on-repost', + '--disable-sync', + '--force-color-profile=srgb', + '--metrics-recording-only', + '--no-first-run', + '--password-store=basic', + '--use-mock-keychain', + '--hide-scrollbars', + '--mute-audio', + '--no-default-browser-check', + '--no-pings', + '--no-zygote', + '--single-process', + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-component-extensions-with-background-pages', + '--disable-dev-shm-usage', + '--disable-features=TranslateUI', + '--disable-ipc-flooding-protection', + '--disable-renderer-backgrounding', + '--disable-sync', + '--force-color-profile=srgb', + '--metrics-recording-only', + '--no-first-run', + '--safebrowsing-disable-auto-update', + '--enable-automation', + '--password-store=basic', + '--use-mock-keychain', + ] + + @staticmethod + async def setup_stealth_context(browser: Browser) -> BrowserContext: + """Create a stealth browser context with anti-detection measures""" + user_agent = CloudflareBypass.get_random_user_agent() + viewport = CloudflareBypass.get_random_viewport() + language = CloudflareBypass.get_random_language() + + # Create context with stealth settings + context = await browser.new_context( + user_agent=user_agent, + viewport=viewport, + locale='en-US', + timezone_id='America/New_York', + permissions=['geolocation'], + ignore_https_errors=True, + extra_http_headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', + 'Accept-Language': language, + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Charset': 'utf-8', + 'Cache-Control': 'no-cache', + 'Pragma': 'no-cache', + 'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', + } + ) + + # Add stealth scripts to the context + await CloudflareBypass._inject_stealth_scripts(context) + + return context + + @staticmethod + async def _inject_stealth_scripts(context: BrowserContext): + """Inject stealth scripts to bypass detection""" + await context.add_init_script(""" + // Remove webdriver property + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + + // Override permissions + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters) + ); + + // Override plugins + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + + // Override languages + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + }); + + // Override chrome + Object.defineProperty(window, 'chrome', { + get: () => ({ + runtime: {}, + }), + }); + + // Override permissions + const originalGetProperty = Object.getOwnPropertyDescriptor; + Object.getOwnPropertyDescriptor = function(obj, prop) { + if (prop === 'webdriver') { + return undefined; + } + return originalGetProperty(obj, prop); + }; + + // Override toString + const originalToString = Function.prototype.toString; + Function.prototype.toString = function() { + if (this === Function.prototype.toString) { + return originalToString.call(this); + } + if (this === window.navigator.permissions.query) { + return 'function query() { [native code] }'; + } + return originalToString.call(this); + }; + """) + + @staticmethod + async def setup_stealth_page(page: Page): + """Setup stealth measures for a specific page""" + # Set additional headers + await page.set_extra_http_headers({ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Cache-Control': 'no-cache', + 'Pragma': 'no-cache', + 'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', + }) + + # Set realistic timeout + page.set_default_timeout(30000) + page.set_default_navigation_timeout(30000) + + # Add human-like behavior + await CloudflareBypass._add_human_behavior(page) + + @staticmethod + async def _add_human_behavior(page: Page): + """Add human-like behavior to avoid detection""" + # Override mouse movement to be more human-like + await page.add_init_script(""" + // Override mouse events to be more human-like + const originalMouseEvent = window.MouseEvent; + window.MouseEvent = function(type, init) { + if (init && init.movementX === 0 && init.movementY === 0) { + init.movementX = Math.random() * 10 - 5; + init.movementY = Math.random() * 10 - 5; + } + return new originalMouseEvent(type, init); + }; + + // Add random mouse movements + setInterval(() => { + const event = new MouseEvent('mousemove', { + clientX: Math.random() * window.innerWidth, + clientY: Math.random() * window.innerHeight, + movementX: Math.random() * 10 - 5, + movementY: Math.random() * 10 - 5, + }); + document.dispatchEvent(event); + }, 5000 + Math.random() * 10000); + """) + + @staticmethod + async def handle_cloudflare_challenge(page: Page, max_retries: int = 3) -> bool: + """Handle Cloudflare challenges and wait for them to complete""" + for attempt in range(max_retries): + try: + # Check for Cloudflare challenge + cloudflare_selectors = [ + '#challenge-form', + '#cf-please-wait', + '.cf-browser-verification', + '#cf-wrapper', + 'iframe[src*="cloudflare"]' + ] + + for selector in cloudflare_selectors: + try: + element = await page.wait_for_selector(selector, timeout=5000) + if element: + print(f"Cloudflare challenge detected on attempt {attempt + 1}") + # Wait for challenge to complete + await asyncio.sleep(5 + random.uniform(2, 8)) + break + except: + continue + + # Wait for page to load completely + await page.wait_for_load_state('networkidle', timeout=30000) + + # Check if we're past the challenge + title = await page.title() + if 'Cloudflare' not in title and 'challenge' not in title.lower(): + print("Successfully bypassed Cloudflare challenge") + return True + + # Add some random delay + await asyncio.sleep(random.uniform(3, 8)) + + except Exception as e: + print(f"Error handling Cloudflare challenge (attempt {attempt + 1}): {e}") + if attempt < max_retries - 1: + await asyncio.sleep(random.uniform(5, 15)) + + return False + + @staticmethod + async def navigate_with_stealth(page: Page, url: str) -> bool: + """Navigate to URL with stealth measures and handle Cloudflare""" + try: + # Add random delay before navigation + await asyncio.sleep(random.uniform(1, 3)) + + # Navigate to the URL + await page.goto(url, wait_until='domcontentloaded') + + # Handle Cloudflare challenge + success = await CloudflareBypass.handle_cloudflare_challenge(page) + + if not success: + print("Failed to bypass Cloudflare challenge") + return False + + # Add human-like scrolling + await CloudflareBypass._simulate_human_scrolling(page) + + return True + + except Exception as e: + print(f"Error during stealth navigation: {e}") + return False + + @staticmethod + async def _simulate_human_scrolling(page: Page): + """Simulate human-like scrolling behavior""" + try: + # Get page height + page_height = await page.evaluate('document.body.scrollHeight') + viewport_height = await page.evaluate('window.innerHeight') + + if page_height > viewport_height: + # Scroll down gradually + current_position = 0 + while current_position < page_height: + scroll_amount = random.randint(100, 300) + current_position += scroll_amount + await page.evaluate(f'window.scrollTo(0, {current_position})') + await asyncio.sleep(random.uniform(0.5, 2)) + + # Scroll back up partially + await page.evaluate(f'window.scrollTo(0, {page_height // 3})') + await asyncio.sleep(random.uniform(1, 3)) + + except Exception as e: + print(f"Error during human scrolling simulation: {e}") + + @staticmethod + def get_proxy_config(proxy_url: Optional[str] = None) -> Dict[str, str]: + """Get proxy configuration if provided""" + if not proxy_url: + return {} + + return { + 'proxy': { + 'server': proxy_url, + 'username': '', # Add if needed + 'password': '', # Add if needed + } + } \ No newline at end of file diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index 4fbbfeb..14cc56a 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -20,6 +20,7 @@ from app.config import BROWSER_INSTANCE_TIMEOUT_MINUTES from asyncio import Queue, Lock, Semaphore from contextlib import asynccontextmanager from app.utils.browser_utils import force_cleanup_old_pages +from app.utils.cloudflare_bypass import CloudflareBypass app = FastAPI() @@ -106,37 +107,15 @@ async def create_browser(): if playwright_instance is None: playwright_instance = await async_playwright().start() + # Get proxy configuration from environment + proxy_url = os.getenv('PROXY_URL') + proxy_config = CloudflareBypass.get_proxy_config(proxy_url) + browser = await playwright_instance.chromium.launch( headless=True, - args=[ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--disable-gpu', - '--disable-extensions', - '--disable-sync', - '--disable-background-networking', - '--disable-default-apps', - '--disable-translate', - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-client-side-phishing-detection', - '--disable-features=site-per-process', - '--disable-hang-monitor', - '--disable-ipc-flooding-protection', - '--disable-popup-blocking', - '--disable-prompt-on-repost', - '--disable-renderer-backgrounding', - '--memory-pressure-off', - '--no-first-run', - '--safebrowsing-disable-auto-update', - '--max_old_space_size=512', # Limit memory usage - '--single-process', # Use single process to reduce resource usage - '--disable-web-security', - '--disable-features=VizDisplayCompositor', - ], + args=CloudflareBypass.get_stealth_args(), ignore_default_args=['--enable-automation'], + **proxy_config ) browser_creation_times[browser] = time.time() @@ -493,12 +472,8 @@ async def safe_browser_operation(url, operation_func): print("Timeout getting browser from pool, creating new one") browser = await create_browser() - # Create context and page - context = await browser.new_context( - user_agent=CUSTOM_USER_AGENT, - viewport={'width': 1920, 'height': 1080}, - ignore_https_errors=True, - ) + # Create stealth context with Cloudflare bypass + context = await CloudflareBypass.setup_stealth_context(browser) page = await context.new_page() @@ -506,7 +481,10 @@ async def safe_browser_operation(url, operation_func): page_creation_times[page] = time.time() active_pages.add(page) - # Set up request interception for better performance + # Setup stealth page with additional measures + await CloudflareBypass.setup_stealth_page(page) + + # Set up request interception for better performance (but allow essential resources) await page.route("**/*", lambda route: route.abort() if route.request.resource_type in ['image', 'stylesheet', 'font', 'media'] else route.continue_()) @@ -562,9 +540,11 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): try: async def visit_operation(page): try: - response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000) - if not response: - print(f"Warning: No response object returned for {decoded_url}") + # Use Cloudflare bypass navigation + success = await CloudflareBypass.navigate_with_stealth(page, decoded_url) + + if not success: + return {"status": "error", "error": "Failed to bypass Cloudflare protection", "url": decoded_url} # Get page content content = await page.content() @@ -604,7 +584,11 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)): try: async def seo_operation(page): try: - response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000) + # Use Cloudflare bypass navigation + success = await CloudflareBypass.navigate_with_stealth(page, decoded_url) + + if not success: + return {"status": "error", "error": "Failed to bypass Cloudflare protection", "url": decoded_url} # Extract SEO information seo_data = await page.evaluate('''() => { @@ -688,7 +672,11 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)): try: async def meta_operation(page): try: - response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000) + # Use Cloudflare bypass navigation + success = await CloudflareBypass.navigate_with_stealth(page, decoded_url) + + if not success: + return {"status": "error", "error": "Failed to bypass Cloudflare protection", "url": decoded_url} # Extract meta tags using Playwright meta_data = await page.evaluate('''() => { @@ -896,6 +884,72 @@ async def force_cleanup_old(x_api_key: Optional[str] = Header(None)): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) +@app.get("/test-cloudflare") +async def test_cloudflare_bypass(url: str, x_api_key: Optional[str] = Header(None)): + """Test Cloudflare bypass functionality on a specific URL""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + # Decode URL if it's encoded + decoded_url = unquote(url) + + try: + async def test_operation(page): + try: + # Use Cloudflare bypass navigation + success = await CloudflareBypass.navigate_with_stealth(page, decoded_url) + + if not success: + return { + "status": "error", + "error": "Failed to bypass Cloudflare protection", + "url": decoded_url, + "cloudflare_detected": True + } + + # Get page information + title = await page.title() + url_after_navigation = page.url + + # Check for Cloudflare indicators + cloudflare_indicators = await page.evaluate('''() => { + const indicators = { + has_cloudflare_title: document.title.toLowerCase().includes('cloudflare'), + has_challenge_form: !!document.querySelector('#challenge-form'), + has_cf_wrapper: !!document.querySelector('#cf-wrapper'), + has_please_wait: !!document.querySelector('#cf-please-wait'), + has_browser_verification: !!document.querySelector('.cf-browser-verification') + }; + return indicators; + }''') + + return { + "status": "success", + "url": decoded_url, + "final_url": url_after_navigation, + "title": title, + "cloudflare_bypassed": True, + "cloudflare_indicators": cloudflare_indicators, + "content_length": len(await page.content()) + } + + except Exception as e: + print(f"Error during Cloudflare test: {e}") + return { + "status": "error", + "error": str(e), + "url": decoded_url, + "cloudflare_detected": False + } + + result = await safe_browser_operation(decoded_url, test_operation) + return result + + except Exception as e: + print(f"Error testing Cloudflare bypass for {decoded_url}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + @asynccontextmanager async def get_browser(): """Context manager for getting a browser from the pool""" diff --git a/Dockers/puppeteer-api/test_cloudflare_bypass.py b/Dockers/puppeteer-api/test_cloudflare_bypass.py new file mode 100644 index 0000000..c4b94f0 --- /dev/null +++ b/Dockers/puppeteer-api/test_cloudflare_bypass.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Test script for Cloudflare bypass functionality +""" + +import asyncio +import aiohttp +import json +import sys +from urllib.parse import quote + +async def test_cloudflare_bypass(api_url, api_key, test_url): + """Test Cloudflare bypass on a specific URL""" + + headers = { + 'X-API-Key': api_key, + 'Content-Type': 'application/json' + } + + # Test the new Cloudflare bypass endpoint + test_endpoint = f"{api_url}/test-cloudflare?url={quote(test_url)}" + + print(f"๐Ÿ” Testing Cloudflare bypass for: {test_url}") + print(f"๐Ÿ“ก Endpoint: {test_endpoint}") + print("-" * 60) + + try: + async with aiohttp.ClientSession() as session: + async with session.get(test_endpoint, headers=headers) as response: + if response.status == 200: + result = await response.json() + + print("โœ… Request successful!") + print(f"๐Ÿ“„ Status: {result.get('status')}") + print(f"๐ŸŒ Final URL: {result.get('final_url', 'N/A')}") + print(f"๐Ÿ“ Title: {result.get('title', 'N/A')}") + print(f"๐Ÿ›ก๏ธ Cloudflare Bypassed: {result.get('cloudflare_bypassed', False)}") + print(f"๐Ÿ“ Content Length: {result.get('content_length', 0)} characters") + + # Show Cloudflare indicators + indicators = result.get('cloudflare_indicators', {}) + print("\n๐Ÿ” Cloudflare Detection Indicators:") + for indicator, value in indicators.items(): + status = "โŒ Detected" if value else "โœ… Not Detected" + print(f" {indicator}: {status}") + + if result.get('cloudflare_bypassed'): + print("\n๐ŸŽ‰ SUCCESS: Cloudflare protection was successfully bypassed!") + else: + print("\nโš ๏ธ WARNING: Cloudflare protection may still be active") + + else: + error_text = await response.text() + print(f"โŒ Request failed with status {response.status}") + print(f"Error: {error_text}") + + except Exception as e: + print(f"โŒ Error during test: {e}") + +async def test_regular_endpoint(api_url, api_key, test_url): + """Test regular endpoint to compare with Cloudflare bypass""" + + headers = { + 'X-API-Key': api_key, + 'Content-Type': 'application/json' + } + + print(f"\n๐Ÿ” Testing regular endpoint for: {test_url}") + print("-" * 60) + + try: + async with aiohttp.ClientSession() as session: + async with session.get(f"{api_url}/?url={quote(test_url)}", headers=headers) as response: + if response.status == 200: + result = await response.json() + print("โœ… Regular endpoint successful!") + print(f"๐Ÿ“„ Status: {result.get('status')}") + if 'content' in result: + content_length = len(result['content']) + print(f"๐Ÿ“ Content Length: {content_length} characters") + if content_length < 1000: + print("โš ๏ธ Content seems short - might be blocked") + else: + print("โœ… Content length looks normal") + else: + error_text = await response.text() + print(f"โŒ Regular endpoint failed: {error_text}") + + except Exception as e: + print(f"โŒ Error during regular test: {e}") + +async def main(): + """Main test function""" + + if len(sys.argv) < 4: + print("Usage: python test_cloudflare_bypass.py ") + print("Example: python test_cloudflare_bypass.py http://localhost:8000 your-api-key https://example.com") + sys.exit(1) + + api_url = sys.argv[1].rstrip('/') + api_key = sys.argv[2] + test_url = sys.argv[3] + + print("๐Ÿ›ก๏ธ Cloudflare Bypass Test Script") + print("=" * 60) + + # Test Cloudflare bypass endpoint + await test_cloudflare_bypass(api_url, api_key, test_url) + + # Test regular endpoint for comparison + await test_regular_endpoint(api_url, api_key, test_url) + + print("\n" + "=" * 60) + print("๐Ÿ Test completed!") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/Dockers/puppeteer-api/version b/Dockers/puppeteer-api/version index b9bc2fd..0c89fc9 100644 --- a/Dockers/puppeteer-api/version +++ b/Dockers/puppeteer-api/version @@ -1 +1 @@ -latest \ No newline at end of file +4.0.0 \ No newline at end of file