update puppeteer server

Dockers/puppeteer-api/Dockerfile

@@ -4,6 +4,7 @@ FROM python:3.9-slim
 RUN apt-get update && apt-get install -y \
     wget \
     gnupg2 \
+    procps \
     && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/google-chrome-keyring.gpg \
     && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \
     && apt-get update \

Dockers/puppeteer-api/main.py

@@ -1,10 +1,17 @@
-from fastapi import FastAPI, HTTPException, Header
+from fastapi import FastAPI, HTTPException, Header, BackgroundTasks
 from pyppeteer import launch
 import os
 import asyncio
+import gc
+import logging
+import psutil
 from typing import Optional
 from urllib.parse import unquote
 
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 app = FastAPI()
 
 # Get API key from environment variable
@@ -12,34 +19,124 @@ API_KEY = os.getenv('API_KEY')
 if not API_KEY:
     raise ValueError("API_KEY environment variable must be set")
 
-async def wait_for_network_idle(page):
+# Global browser instance
-    """Wait until no network requests are in flight"""
+browser = None
-    await page.waitForNetworkIdle(idleTime=500, timeout=30000)
+
+async def get_browser():
+    """Get or create a browser instance"""
+    global browser
+    if browser is None or not browser.isConnected():
+        logger.info("Launching new browser instance")
+        browser = await launch(
+            headless=True,
+            executablePath='/usr/bin/google-chrome',
+            args=[
+                '--no-sandbox',
+                '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage',
+                '--disable-gpu',
+                '--disable-extensions',
+                '--disable-sync',
+                '--disable-translate',
+                '--hide-scrollbars',
+                '--mute-audio',
+                '--disable-background-networking',
+                '--disable-background-timer-throttling',
+                '--disable-backgrounding-occluded-windows',
+                '--disable-breakpad',
+                '--disable-client-side-phishing-detection',
+                '--disable-component-extensions-with-background-pages',
+                '--disable-default-apps',
+                '--disable-features=TranslateUI,BlinkGenPropertyTrees',
+                '--disable-hang-monitor',
+                '--disable-ipc-flooding-protection',
+                '--disable-popup-blocking',
+                '--disable-prompt-on-repost',
+                '--disable-renderer-backgrounding',
+                '--force-color-profile=srgb',
+                '--metrics-recording-only',
+                '--no-first-run',
+                '--enable-automation',
+                '--password-store=basic',
+                '--use-mock-keychain',
+                '--js-flags=--expose-gc',
+                '--memory-pressure-off',
+            ]
+        )
+    return browser
+
+async def cleanup_resources(page=None, close_browser=False):
+    """Clean up resources to prevent memory leaks"""
+    try:
+        if page:
+            await page.close()
+
+        if close_browser and browser and browser.isConnected():
+            logger.info("Closing browser instance")
+            await browser.close()
+            global browser
+            browser = None
+
+        # Force garbage collection
+        gc.collect()
+
+        # Log memory usage
+        process = psutil.Process(os.getpid())
+        memory_info = process.memory_info()
+        logger.info(f"Memory usage: {memory_info.rss / 1024 / 1024:.2f} MB")
+    except Exception as e:
+        logger.error(f"Error during cleanup: {str(e)}")
 
 @app.head("/")
 async def health_check():
     return {"status": "ok"}
 
-@app.get("/")
+@app.get("/memory")
-async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
+async def memory_status(x_api_key: Optional[str] = Header(None)):
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
 
+    process = psutil.Process(os.getpid())
+    memory_info = process.memory_info()
+
+    return {
+        "memory_usage_mb": memory_info.rss / 1024 / 1024,
+        "browser_active": browser is not None and browser.isConnected() if browser else False
+    }
+
+@app.post("/cleanup")
+async def force_cleanup(x_api_key: Optional[str] = Header(None)):
+    # Validate API key
+    if not x_api_key or x_api_key != API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    await cleanup_resources(close_browser=True)
+    return {"status": "cleanup completed"}
+
+@app.get("/")
+async def visit_url(url: str, x_api_key: Optional[str] = Header(None), background_tasks: BackgroundTasks = None):
+    # Validate API key
+    if not x_api_key or x_api_key != API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    page = None
     try:
         # Decode URL if it's encoded
         decoded_url = unquote(url)
-        print(decoded_url)
+        logger.info(f"Visiting URL: {decoded_url}")
 
-        # Launch browser using installed Chrome
+        # Get browser instance
-        browser = await launch(
+        browser_instance = await get_browser()
-            headless=True,
-            executablePath='/usr/bin/google-chrome',
-            args=['--no-sandbox', '--disable-setuid-sandbox']
-        )
 
         # Create new page
-        page = await browser.newPage()
+        page = await browser_instance.newPage()
+
+        # Set page timeout
+        await page.setDefaultNavigationTimeout(30000)  # 30 seconds
+
+        # Set viewport
+        await page.setViewport({"width": 1280, "height": 800})
 
         # Navigate to URL and wait for network idle
         await page.goto(decoded_url, waitUntil='networkidle2')
@@ -47,14 +144,27 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
         # Get page content
         content = await page.content()
 
-        # Close browser
+        # Schedule cleanup in background
-        await browser.close()
+        if background_tasks:
+            background_tasks.add_task(cleanup_resources, page, False)
+        else:
+            await cleanup_resources(page, False)
 
         return {"status": "success", "content": content}
 
     except Exception as e:
+        logger.error(f"Error processing request: {str(e)}")
+        # Ensure cleanup happens even on error
+        if page:
+            await cleanup_resources(page, False)
         raise HTTPException(status_code=500, detail=str(e))
 
+@app.on_event("shutdown")
+async def shutdown_event():
+    """Clean up resources when shutting down"""
+    logger.info("Application shutting down, cleaning up resources")
+    await cleanup_resources(close_browser=True)
+
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8000)

Dockers/puppeteer-api/requirements.txt

@@ -1,3 +1,4 @@
 fastapi==0.68.1
 uvicorn==0.15.0
 pyppeteer==1.0.2
+psutil

Dockers/verdaccio/verdaccio.yaml

@@ -3,8 +3,6 @@ auth:
   gitea-auth:
     url: https://gitea.bramkelchtermans.be
     defaultOrg: LaughNCode
-  htpasswd:
-    file: ./htpasswd
 uplinks:
   npmjs:
     url: https://registry.npmjs.org/