From 968f0d9864fa83aabe6eee09eb254b1193507f40 Mon Sep 17 00:00:00 2001 From: Bram Kelchtermans Date: Mon, 31 Mar 2025 14:13:37 +0200 Subject: [PATCH] update puppeteer server --- Dockers/puppeteer-api/Dockerfile | 1 + Dockers/puppeteer-api/main.py | 142 ++++++++++++++++++++++--- Dockers/puppeteer-api/requirements.txt | 3 +- Dockers/verdaccio/verdaccio.yaml | 2 - 4 files changed, 129 insertions(+), 19 deletions(-) diff --git a/Dockers/puppeteer-api/Dockerfile b/Dockers/puppeteer-api/Dockerfile index 74f6b22..73cae08 100644 --- a/Dockers/puppeteer-api/Dockerfile +++ b/Dockers/puppeteer-api/Dockerfile @@ -4,6 +4,7 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y \ wget \ gnupg2 \ + procps \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/google-chrome-keyring.gpg \ && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \ && apt-get update \ diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index fc3a7f2..6c7b710 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -1,10 +1,17 @@ -from fastapi import FastAPI, HTTPException, Header +from fastapi import FastAPI, HTTPException, Header, BackgroundTasks from pyppeteer import launch import os import asyncio +import gc +import logging +import psutil from typing import Optional from urllib.parse import unquote +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + app = FastAPI() # Get API key from environment variable @@ -12,34 +19,124 @@ API_KEY = os.getenv('API_KEY') if not API_KEY: raise ValueError("API_KEY environment variable must be set") -async def wait_for_network_idle(page): - """Wait until no network requests are in flight""" - await page.waitForNetworkIdle(idleTime=500, timeout=30000) +# Global browser instance +browser = None + +async def get_browser(): + """Get or create a browser instance""" + global browser + if browser is None or not browser.isConnected(): + logger.info("Launching new browser instance") + browser = await launch( + headless=True, + executablePath='/usr/bin/google-chrome', + args=[ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + '--disable-extensions', + '--disable-sync', + '--disable-translate', + '--hide-scrollbars', + '--mute-audio', + '--disable-background-networking', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-client-side-phishing-detection', + '--disable-component-extensions-with-background-pages', + '--disable-default-apps', + '--disable-features=TranslateUI,BlinkGenPropertyTrees', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--force-color-profile=srgb', + '--metrics-recording-only', + '--no-first-run', + '--enable-automation', + '--password-store=basic', + '--use-mock-keychain', + '--js-flags=--expose-gc', + '--memory-pressure-off', + ] + ) + return browser + +async def cleanup_resources(page=None, close_browser=False): + """Clean up resources to prevent memory leaks""" + try: + if page: + await page.close() + + if close_browser and browser and browser.isConnected(): + logger.info("Closing browser instance") + await browser.close() + global browser + browser = None + + # Force garbage collection + gc.collect() + + # Log memory usage + process = psutil.Process(os.getpid()) + memory_info = process.memory_info() + logger.info(f"Memory usage: {memory_info.rss / 1024 / 1024:.2f} MB") + except Exception as e: + logger.error(f"Error during cleanup: {str(e)}") @app.head("/") async def health_check(): return {"status": "ok"} -@app.get("/") -async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): +@app.get("/memory") +async def memory_status(x_api_key: Optional[str] = Header(None)): # Validate API key if not x_api_key or x_api_key != API_KEY: raise HTTPException(status_code=401, detail="Invalid API key") + process = psutil.Process(os.getpid()) + memory_info = process.memory_info() + + return { + "memory_usage_mb": memory_info.rss / 1024 / 1024, + "browser_active": browser is not None and browser.isConnected() if browser else False + } + +@app.post("/cleanup") +async def force_cleanup(x_api_key: Optional[str] = Header(None)): + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + await cleanup_resources(close_browser=True) + return {"status": "cleanup completed"} + +@app.get("/") +async def visit_url(url: str, x_api_key: Optional[str] = Header(None), background_tasks: BackgroundTasks = None): + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + page = None try: # Decode URL if it's encoded decoded_url = unquote(url) - print(decoded_url) + logger.info(f"Visiting URL: {decoded_url}") - # Launch browser using installed Chrome - browser = await launch( - headless=True, - executablePath='/usr/bin/google-chrome', - args=['--no-sandbox', '--disable-setuid-sandbox'] - ) + # Get browser instance + browser_instance = await get_browser() # Create new page - page = await browser.newPage() + page = await browser_instance.newPage() + + # Set page timeout + await page.setDefaultNavigationTimeout(30000) # 30 seconds + + # Set viewport + await page.setViewport({"width": 1280, "height": 800}) # Navigate to URL and wait for network idle await page.goto(decoded_url, waitUntil='networkidle2') @@ -47,14 +144,27 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): # Get page content content = await page.content() - # Close browser - await browser.close() + # Schedule cleanup in background + if background_tasks: + background_tasks.add_task(cleanup_resources, page, False) + else: + await cleanup_resources(page, False) return {"status": "success", "content": content} except Exception as e: + logger.error(f"Error processing request: {str(e)}") + # Ensure cleanup happens even on error + if page: + await cleanup_resources(page, False) raise HTTPException(status_code=500, detail=str(e)) +@app.on_event("shutdown") +async def shutdown_event(): + """Clean up resources when shutting down""" + logger.info("Application shutting down, cleaning up resources") + await cleanup_resources(close_browser=True) + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/Dockers/puppeteer-api/requirements.txt b/Dockers/puppeteer-api/requirements.txt index 1d5de17..bfc1397 100644 --- a/Dockers/puppeteer-api/requirements.txt +++ b/Dockers/puppeteer-api/requirements.txt @@ -1,3 +1,4 @@ fastapi==0.68.1 uvicorn==0.15.0 -pyppeteer==1.0.2 \ No newline at end of file +pyppeteer==1.0.2 +psutil \ No newline at end of file diff --git a/Dockers/verdaccio/verdaccio.yaml b/Dockers/verdaccio/verdaccio.yaml index dcd5deb..81ea5d8 100644 --- a/Dockers/verdaccio/verdaccio.yaml +++ b/Dockers/verdaccio/verdaccio.yaml @@ -3,8 +3,6 @@ auth: gitea-auth: url: https://gitea.bramkelchtermans.be defaultOrg: LaughNCode - htpasswd: - file: ./htpasswd uplinks: npmjs: url: https://registry.npmjs.org/