potential performance upgrade
Build and Push Docker Images / build-and-push (push) Successful in 21s
Build and Push Docker Images / build-and-push (push) Successful in 21s
This commit is contained in:
@@ -273,73 +273,101 @@ class CloudflareBypass:
|
||||
""")
|
||||
|
||||
@staticmethod
|
||||
async def handle_cloudflare_challenge(page: Page, max_retries: int = 3) -> bool:
|
||||
"""Handle Cloudflare challenges and wait for them to complete"""
|
||||
async def handle_cloudflare_challenge(page: Page, max_retries: int = 2) -> bool:
|
||||
"""Handle Cloudflare challenge with improved efficiency"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
# Check for Cloudflare challenge
|
||||
cloudflare_selectors = [
|
||||
'#challenge-form',
|
||||
'#cf-please-wait',
|
||||
'.cf-browser-verification',
|
||||
'#cf-wrapper',
|
||||
'iframe[src*="cloudflare"]'
|
||||
]
|
||||
print(f"Handling Cloudflare challenge (attempt {attempt + 1}/{max_retries})")
|
||||
|
||||
for selector in cloudflare_selectors:
|
||||
try:
|
||||
element = await page.wait_for_selector(selector, timeout=5000)
|
||||
if element:
|
||||
print(f"Cloudflare challenge detected on attempt {attempt + 1}")
|
||||
# Wait for challenge to complete
|
||||
await asyncio.sleep(5 + random.uniform(2, 8))
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
# Wait for page to load completely
|
||||
await page.wait_for_load_state('networkidle', timeout=30000)
|
||||
|
||||
# Check if we're past the challenge
|
||||
title = await page.title()
|
||||
if 'Cloudflare' not in title and 'challenge' not in title.lower():
|
||||
print("Successfully bypassed Cloudflare challenge")
|
||||
# Wait for challenge form with shorter timeout
|
||||
try:
|
||||
await page.wait_for_selector('#challenge-form', timeout=10000) # 10 seconds
|
||||
except:
|
||||
# If no challenge form, might already be bypassed
|
||||
return True
|
||||
|
||||
# Add some random delay
|
||||
await asyncio.sleep(random.uniform(3, 8))
|
||||
# Wait for the challenge to complete with shorter timeout
|
||||
try:
|
||||
await page.wait_for_function('''
|
||||
() => {
|
||||
return !document.querySelector('#challenge-form') &&
|
||||
!document.querySelector('#cf-wrapper') &&
|
||||
!document.querySelector('#cf-please-wait') &&
|
||||
!document.querySelector('.cf-browser-verification');
|
||||
}
|
||||
''', timeout=15000) # 15 seconds
|
||||
|
||||
print("Cloudflare challenge completed")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Challenge completion timeout (attempt {attempt + 1}): {e}")
|
||||
|
||||
# Try to click any "Verify" buttons
|
||||
try:
|
||||
verify_buttons = await page.query_selector_all('button[type="submit"], input[type="submit"]')
|
||||
for button in verify_buttons:
|
||||
try:
|
||||
await button.click(timeout=5000)
|
||||
await asyncio.sleep(2)
|
||||
except:
|
||||
continue
|
||||
except:
|
||||
pass
|
||||
|
||||
# Short delay before retry
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(2)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error handling Cloudflare challenge (attempt {attempt + 1}): {e}")
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(random.uniform(5, 15))
|
||||
await asyncio.sleep(2)
|
||||
|
||||
print("Failed to bypass Cloudflare challenge after all attempts")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
async def navigate_with_stealth(page: Page, url: str) -> bool:
|
||||
"""Navigate to URL with stealth measures and handle Cloudflare"""
|
||||
"""Navigate to URL with stealth measures and improved performance"""
|
||||
try:
|
||||
# Add random delay before navigation
|
||||
await asyncio.sleep(random.uniform(1, 3))
|
||||
# Set shorter timeouts for better performance
|
||||
page.set_default_timeout(30000) # 30 seconds
|
||||
page.set_default_navigation_timeout(30000) # 30 seconds
|
||||
|
||||
# Navigate to the URL
|
||||
await page.goto(url, wait_until='domcontentloaded')
|
||||
# Navigate with stealth measures
|
||||
response = await page.goto(url, wait_until='domcontentloaded') # Changed from 'networkidle' to 'domcontentloaded' for speed
|
||||
|
||||
# Handle Cloudflare challenge
|
||||
success = await CloudflareBypass.handle_cloudflare_challenge(page)
|
||||
|
||||
if not success:
|
||||
print("Failed to bypass Cloudflare challenge")
|
||||
if not response:
|
||||
return False
|
||||
|
||||
# Add human-like scrolling
|
||||
await CloudflareBypass._simulate_human_scrolling(page)
|
||||
# Quick check for Cloudflare challenge
|
||||
if response.status == 403 or response.status == 503:
|
||||
# Handle Cloudflare challenge
|
||||
return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2) # Reduced retries
|
||||
|
||||
# Check for Cloudflare indicators in the page
|
||||
cloudflare_detected = await page.evaluate('''() => {
|
||||
return document.title.toLowerCase().includes('cloudflare') ||
|
||||
!!document.querySelector('#challenge-form') ||
|
||||
!!document.querySelector('#cf-wrapper') ||
|
||||
!!document.querySelector('#cf-please-wait') ||
|
||||
!!document.querySelector('.cf-browser-verification');
|
||||
}''')
|
||||
|
||||
if cloudflare_detected:
|
||||
return await CloudflareBypass.handle_cloudflare_challenge(page, max_retries=2) # Reduced retries
|
||||
|
||||
# Wait for page to be ready (shorter wait)
|
||||
try:
|
||||
await page.wait_for_load_state('domcontentloaded', timeout=10000) # 10 second timeout
|
||||
except:
|
||||
pass # Continue even if load state timeout
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during stealth navigation: {e}")
|
||||
print(f"Error during stealth navigation to {url}: {e}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user