Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

try another fix
Build and Push Docker Images / build-and-push (push) Successful in 18s
Details

Browse Source
This commit is contained in:
Bram
2025-07-01 09:54:11 +02:00
parent bde19374c0
commit 96eb344191
3 changed files with 562 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/main.py
+268 -128
View File
@@ -6,6 +6,8 @@ import json
import re
import sqlite3
import time
import signal
import psutil
from datetime import datetime, timedelta
from typing import Optional, Dict, List, Any
from urllib.parse import unquote
@@ -14,7 +16,7 @@ from apscheduler.triggers.cron import CronTrigger
from fastapi.middleware.base import BaseHTTPMiddleware
# Add imports for browser pool
from asyncio import Queue, Lock
from asyncio import Queue, Lock, Semaphore
from contextlib import asynccontextmanager
app = FastAPI()
@@ -33,12 +35,17 @@ CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
# Define custom user agent
CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
# Browser pool configuration
MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '5')) # Maximum number of browser instances
BROWSER_TTL = int(os.getenv('BROWSER_TTL', '3600')) # Time to live for browser instances in seconds
browser_pool = Queue()
# Browser pool configuration - Reduced for better resource management
MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '3')) # Reduced from 5 to 3
BROWSER_TTL = int(os.getenv('BROWSER_TTL', '1800')) # Reduced from 3600 to 1800 seconds (30 minutes)
MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '5')) # New: limit concurrent operations
# Browser pool management
browser_pool = Queue(maxsize=MAX_BROWSERS) # Add maxsize to prevent unbounded growth
browser_lock = Lock()
browser_creation_times = {}
active_browsers = set() # Track active browsers
operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS) # Limit concurrent operations
# Rate limiting configuration
RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60')) # requests per minute
@@ -87,105 +94,164 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
app.add_middleware(RateLimitMiddleware)
async def create_browser():
"""Create a new browser instance"""
browser = await launch(
headless=True,
executablePath='/usr/bin/google-chrome',
args=[
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--disable-extensions',
'--disable-sync',
'--disable-background-networking',
'--disable-default-apps',
'--disable-translate',
'--disable-background-timer-throttling',
'--disable-backgrounding-occluded-windows',
'--disable-client-side-phishing-detection',
'--disable-features=site-per-process',
'--disable-hang-monitor',
'--disable-ipc-flooding-protection',
'--disable-popup-blocking',
'--disable-prompt-on-repost',
'--disable-renderer-backgrounding',
'--memory-pressure-off',
'--no-first-run',
'--safebrowsing-disable-auto-update',
],
handleSIGINT=False,
handleSIGTERM=False,
handleSIGHUP=False,
ignoreHTTPSErrors=True
)
browser_creation_times[browser] = time.time()
return browser
"""Create a new browser instance with improved resource management"""
try:
browser = await launch(
headless=True,
executablePath='/usr/bin/google-chrome',
args=[
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--disable-extensions',
'--disable-sync',
'--disable-background-networking',
'--disable-default-apps',
'--disable-translate',
'--disable-background-timer-throttling',
'--disable-backgrounding-occluded-windows',
'--disable-client-side-phishing-detection',
'--disable-features=site-per-process',
'--disable-hang-monitor',
'--disable-ipc-flooding-protection',
'--disable-popup-blocking',
'--disable-prompt-on-repost',
'--disable-renderer-backgrounding',
'--memory-pressure-off',
'--no-first-run',
'--safebrowsing-disable-auto-update',
'--max_old_space_size=512', # Limit memory usage
'--single-process', # Use single process to reduce resource usage
'--disable-web-security',
'--disable-features=VizDisplayCompositor',
],
handleSIGINT=False,
handleSIGTERM=False,
handleSIGHUP=False,
ignoreHTTPSErrors=True,
autoClose=True, # Ensure browser closes automatically
)
browser_creation_times[browser] = time.time()
active_browsers.add(browser)
print(f"Created new browser instance. Total active browsers: {len(active_browsers)}")
return browser
except Exception as e:
print(f"Error creating browser: {e}")
raise
async def cleanup_browser(browser):
"""Safely cleanup a browser instance"""
try:
if browser in active_browsers:
active_browsers.remove(browser)
if browser in browser_creation_times:
del browser_creation_times[browser]
# Close all pages first
pages = await browser.pages()
for page in pages:
try:
await page.close()
except Exception as e:
print(f"Error closing page: {e}")
# Close browser
await browser.close()
print(f"Browser cleaned up. Total active browsers: {len(active_browsers)}")
except Exception as e:
print(f"Error during browser cleanup: {e}")
async def check_browser_health():
"""Check browser health and recycle if needed"""
while True:
try:
# Sleep for 5 minutes between checks
await asyncio.sleep(300)
# Sleep for 2 minutes between checks (reduced from 5 minutes)
await asyncio.sleep(120)
async with browser_lock:
# Get all browsers from the pool
browsers = []
while not browser_pool.empty():
browsers.append(await browser_pool.get())
try:
browsers.append(await browser_pool.get_nowait())
except asyncio.QueueEmpty:
break
# Check each browser
for browser in browsers:
try:
# Check if browser is too old
if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
await browser.close()
del browser_creation_times[browser]
print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
await cleanup_browser(browser)
browser = await create_browser()
else:
# Quick health check
await browser.pages()
# Put back in pool if healthy
await browser_pool.put(browser)
except Exception:
if not browser_pool.full():
await browser_pool.put(browser)
else:
# Pool is full, cleanup this browser
await cleanup_browser(browser)
except Exception as e:
print(f"Browser health check failed: {e}")
# If unhealthy, close and create new
try:
await browser.close()
except:
pass
if browser in browser_creation_times:
del browser_creation_times[browser]
new_browser = await create_browser()
await browser_pool.put(new_browser)
await cleanup_browser(browser)
if not browser_pool.full():
new_browser = await create_browser()
await browser_pool.put(new_browser)
except Exception as e:
print(f"Error in browser health check: {str(e)}")
async def force_cleanup_all_browsers():
"""Force cleanup all browsers in emergency situations"""
print("Force cleaning up all browsers...")
# Clean up pool
while not browser_pool.empty():
try:
browser = await browser_pool.get_nowait()
await cleanup_browser(browser)
except asyncio.QueueEmpty:
break
# Clean up active browsers
for browser in list(active_browsers):
await cleanup_browser(browser)
# Initialize browser pool
@app.on_event("startup")
async def init_browser_pool():
"""Initialize the browser pool with some browsers"""
for _ in range(min(3, MAX_BROWSERS)): # Start with 3 browsers or MAX_BROWSERS, whichever is smaller
browser = await create_browser()
await browser_pool.put(browser)
print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
try:
for _ in range(min(2, MAX_BROWSERS)): # Start with 2 browsers instead of 3
browser = await create_browser()
await browser_pool.put(browser)
print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
# Start browser health check task
asyncio.create_task(check_browser_health())
# Start browser health check task
asyncio.create_task(check_browser_health())
except Exception as e:
print(f"Error initializing browser pool: {e}")
@app.on_event("shutdown")
async def cleanup_browser_pool():
"""Clean up all browsers in the pool"""
while not browser_pool.empty():
try:
browser = await browser_pool.get_nowait()
await browser.close()
if browser in browser_creation_times:
del browser_creation_times[browser]
except:
pass
await force_cleanup_all_browsers()
# Signal handlers for graceful shutdown
def signal_handler(signum, frame):
print(f"Received signal {signum}, shutting down gracefully...")
asyncio.create_task(force_cleanup_all_browsers())
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
# Initialize SQLite database
def init_db():
@@ -371,47 +437,49 @@ async def health_check():
return {"status": "ok"}
async def safe_browser_operation(url, operation_func):
"""Safely perform a browser operation with proper cleanup"""
async with get_browser() as browser:
try:
# Create a new page
page = await browser.newPage()
# Set reasonable viewport
await page.setViewport({'width': 1280, 'height': 800})
# Set user agent
await page.setUserAgent(CUSTOM_USER_AGENT)
# Set reasonable timeout
page.setDefaultNavigationTimeout(30000)
# Enable request interception to block unnecessary resources
await page.setRequestInterception(True)
async def intercept(request):
# Block unnecessary resource types
if request.resourceType in ['image', 'media', 'font', 'stylesheet']:
await request.abort()
else:
await request.continue_()
page.on('request', lambda req: asyncio.ensure_future(intercept(req)))
# Perform the operation
result = await operation_func(page)
return result
except Exception as e:
print(f"Error during browser operation: {str(e)}")
raise
finally:
"""Safely perform a browser operation with proper cleanup and resource limits"""
async with operation_semaphore: # Limit concurrent operations
async with get_browser() as browser:
page = None
try:
# Ensure page is properly closed
if 'page' in locals():
await page.close()
# Create a new page
page = await browser.newPage()
# Set reasonable viewport
await page.setViewport({'width': 1280, 'height': 800})
# Set user agent
await page.setUserAgent(CUSTOM_USER_AGENT)
# Set reasonable timeout
page.setDefaultNavigationTimeout(30000)
# Enable request interception to block unnecessary resources
await page.setRequestInterception(True)
async def intercept(request):
# Block unnecessary resource types
if request.resourceType in ['image', 'media', 'font', 'stylesheet']:
await request.abort()
else:
await request.continue_()
page.on('request', lambda req: asyncio.ensure_future(intercept(req)))
# Perform the operation
result = await operation_func(page)
return result
except Exception as e:
print(f"Error closing page: {str(e)}")
print(f"Error during browser operation: {str(e)}")
raise
finally:
try:
# Ensure page is properly closed
if page:
await page.close()
except Exception as e:
print(f"Error closing page: {str(e)}")
@app.get("/")
async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
@@ -682,23 +750,93 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.get("/status")
async def system_status(x_api_key: Optional[str] = Header(None)):
"""Get system status and browser pool information"""
# Validate API key
if not x_api_key or x_api_key != API_KEY:
raise HTTPException(status_code=401, detail="Invalid API key")
try:
# Get system information
process = psutil.Process()
memory_info = process.memory_info()
# Get browser pool information
pool_size = browser_pool.qsize()
active_browser_count = len(active_browsers)
# Calculate browser ages
browser_ages = []
for browser, creation_time in browser_creation_times.items():
age = time.time() - creation_time
browser_ages.append(age)
return {
"system": {
"cpu_percent": process.cpu_percent(),
"memory_mb": memory_info.rss / 1024 / 1024,
"memory_percent": process.memory_percent(),
"open_files": len(process.open_files()),
"connections": len(process.connections()),
"threads": process.num_threads()
},
"browser_pool": {
"pool_size": pool_size,
"active_browsers": active_browser_count,
"max_browsers": MAX_BROWSERS,
"browser_ttl_seconds": BROWSER_TTL,
"browser_ages_seconds": browser_ages,
"concurrent_operations_limit": MAX_CONCURRENT_OPERATIONS
},
"timestamp": time.time()
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/emergency-cleanup")
async def emergency_cleanup(x_api_key: Optional[str] = Header(None)):
"""Force emergency cleanup of all browsers"""
# Validate API key
if not x_api_key or x_api_key != API_KEY:
raise HTTPException(status_code=401, detail="Invalid API key")
try:
await force_cleanup_all_browsers()
return {"status": "success", "message": "Emergency cleanup completed"}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@asynccontextmanager
async def get_browser():
"""Get a browser from the pool or create a new one if needed"""
browser = None
try:
# Try to get a browser from the pool
# Try to get a browser from the pool with timeout
try:
browser = await browser_pool.get_nowait()
except asyncio.QueueEmpty:
# If pool is empty, create a new browser if under the limit
browser = await asyncio.wait_for(browser_pool.get(), timeout=30.0)
except (asyncio.QueueEmpty, asyncio.TimeoutError):
# If pool is empty or timeout, create a new browser if under the limit
async with browser_lock:
if browser_pool.qsize() + 1 <= MAX_BROWSERS:
current_browser_count = len(active_browsers)
if current_browser_count < MAX_BROWSERS:
browser = await create_browser()
else:
# If at limit, wait for a browser to become available
browser = await browser_pool.get()
# If at limit, wait for a browser to become available with timeout
try:
browser = await asyncio.wait_for(browser_pool.get(), timeout=60.0)
except asyncio.TimeoutError:
# Emergency cleanup and create new browser
print("Emergency: Timeout waiting for browser, forcing cleanup")
await force_cleanup_all_browsers()
browser = await create_browser()
yield browser
except Exception as e:
print(f"Error in get_browser: {e}")
# Emergency cleanup if we can't get a browser
await force_cleanup_all_browsers()
browser = await create_browser()
yield browser
finally:
# Return browser to pool if it's still viable
@@ -708,21 +846,23 @@ async def get_browser():
await browser.pages()
# Check if browser is too old
if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
await browser.close()
if browser in browser_creation_times:
del browser_creation_times[browser]
print(f"Recycling old browser in get_browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
await cleanup_browser(browser)
browser = await create_browser()
await browser_pool.put(browser)
except Exception:
# Only put back if pool is not full
if not browser_pool.full():
await browser_pool.put(browser)
else:
# Pool is full, cleanup this browser
await cleanup_browser(browser)
except Exception as e:
print(f"Browser health check failed in get_browser: {e}")
# If browser is not usable, close it and create a new one
try:
await browser.close()
except:
pass
if browser in browser_creation_times:
del browser_creation_times[browser]
browser = await create_browser()
await browser_pool.put(browser)
await cleanup_browser(browser)
if not browser_pool.full():
new_browser = await create_browser()
await browser_pool.put(new_browser)
if __name__ == "__main__":
import uvicorn