diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index 6c7b710..c7d251e 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -8,22 +8,18 @@ import psutil from typing import Optional from urllib.parse import unquote -# Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) app = FastAPI() -# Get API key from environment variable API_KEY = os.getenv('API_KEY') if not API_KEY: raise ValueError("API_KEY environment variable must be set") -# Global browser instance browser = None async def get_browser(): - """Get or create a browser instance""" global browser if browser is None or not browser.isConnected(): logger.info("Launching new browser instance") @@ -35,136 +31,58 @@ async def get_browser(): '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', - '--disable-extensions', - '--disable-sync', - '--disable-translate', - '--hide-scrollbars', - '--mute-audio', - '--disable-background-networking', - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-breakpad', - '--disable-client-side-phishing-detection', - '--disable-component-extensions-with-background-pages', - '--disable-default-apps', - '--disable-features=TranslateUI,BlinkGenPropertyTrees', - '--disable-hang-monitor', - '--disable-ipc-flooding-protection', - '--disable-popup-blocking', - '--disable-prompt-on-repost', - '--disable-renderer-backgrounding', - '--force-color-profile=srgb', - '--metrics-recording-only', - '--no-first-run', - '--enable-automation', - '--password-store=basic', - '--use-mock-keychain', + '--single-process', + '--disable-software-rasterizer', '--js-flags=--expose-gc', '--memory-pressure-off', ] ) return browser -async def cleanup_resources(page=None, close_browser=False): - """Clean up resources to prevent memory leaks""" +async def cleanup_resources(page=None): try: if page: + try: + await page.evaluate("window.gc()") # Trigger JS garbage collection + except: + pass # Ignore if GC is not exposed await page.close() - - if close_browser and browser and browser.isConnected(): - logger.info("Closing browser instance") - await browser.close() - global browser - browser = None - - # Force garbage collection gc.collect() - - # Log memory usage process = psutil.Process(os.getpid()) - memory_info = process.memory_info() - logger.info(f"Memory usage: {memory_info.rss / 1024 / 1024:.2f} MB") + logger.info(f"Memory usage: {process.memory_info().rss / 1024 / 1024:.2f} MB") except Exception as e: - logger.error(f"Error during cleanup: {str(e)}") - -@app.head("/") -async def health_check(): - return {"status": "ok"} - -@app.get("/memory") -async def memory_status(x_api_key: Optional[str] = Header(None)): - # Validate API key - if not x_api_key or x_api_key != API_KEY: - raise HTTPException(status_code=401, detail="Invalid API key") - - process = psutil.Process(os.getpid()) - memory_info = process.memory_info() - - return { - "memory_usage_mb": memory_info.rss / 1024 / 1024, - "browser_active": browser is not None and browser.isConnected() if browser else False - } - -@app.post("/cleanup") -async def force_cleanup(x_api_key: Optional[str] = Header(None)): - # Validate API key - if not x_api_key or x_api_key != API_KEY: - raise HTTPException(status_code=401, detail="Invalid API key") - - await cleanup_resources(close_browser=True) - return {"status": "cleanup completed"} + logger.error(f"Cleanup error: {str(e)}") @app.get("/") async def visit_url(url: str, x_api_key: Optional[str] = Header(None), background_tasks: BackgroundTasks = None): - # Validate API key if not x_api_key or x_api_key != API_KEY: raise HTTPException(status_code=401, detail="Invalid API key") page = None try: - # Decode URL if it's encoded decoded_url = unquote(url) logger.info(f"Visiting URL: {decoded_url}") - - # Get browser instance browser_instance = await get_browser() - - # Create new page - page = await browser_instance.newPage() - - # Set page timeout - await page.setDefaultNavigationTimeout(30000) # 30 seconds - - # Set viewport + context = await browser_instance.createIncognitoBrowserContext() + page = await context.newPage() await page.setViewport({"width": 1280, "height": 800}) - - # Navigate to URL and wait for network idle await page.goto(decoded_url, waitUntil='networkidle2') - - # Get page content content = await page.content() - - # Schedule cleanup in background if background_tasks: - background_tasks.add_task(cleanup_resources, page, False) + background_tasks.add_task(cleanup_resources, page) else: - await cleanup_resources(page, False) - + await cleanup_resources(page) return {"status": "success", "content": content} - except Exception as e: - logger.error(f"Error processing request: {str(e)}") - # Ensure cleanup happens even on error + logger.error(f"Error: {str(e)}") if page: - await cleanup_resources(page, False) + await cleanup_resources(page) raise HTTPException(status_code=500, detail=str(e)) @app.on_event("shutdown") async def shutdown_event(): - """Clean up resources when shutting down""" - logger.info("Application shutting down, cleaning up resources") - await cleanup_resources(close_browser=True) - -if __name__ == "__main__": - import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8000) + logger.info("Shutting down, cleaning up browser instance") + global browser + if browser: + await browser.close() + browser = None diff --git a/Dockers/puppeteer-api/requirements.txt b/Dockers/puppeteer-api/requirements.txt index bfc1397..1d5de17 100644 --- a/Dockers/puppeteer-api/requirements.txt +++ b/Dockers/puppeteer-api/requirements.txt @@ -1,4 +1,3 @@ fastapi==0.68.1 uvicorn==0.15.0 -pyppeteer==1.0.2 -psutil \ No newline at end of file +pyppeteer==1.0.2 \ No newline at end of file