replace puppeteer with playwright

Dockers/puppeteer-api/main.py

@@ -1,5 +1,5 @@
 from fastapi import FastAPI, HTTPException, Header, Request
-from pyppeteer import launch
+from playwright.async_api import async_playwright
 import os
 import asyncio
 import json
@@ -46,6 +46,7 @@ browser_lock = Lock()
 browser_creation_times = {}
 active_browsers = set()  # Track active browsers
 operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS)  # Limit concurrent operations
+playwright_instance = None  # Global playwright instance
 
 # Rate limiting configuration
 RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60'))  # requests per minute
@@ -95,10 +96,14 @@ app.add_middleware(RateLimitMiddleware)
 
 async def create_browser():
     """Create a new browser instance with improved resource management"""
+    global playwright_instance
+
     try:
-        browser = await launch(
+        if playwright_instance is None:
+            playwright_instance = await async_playwright().start()
+
+        browser = await playwright_instance.chromium.launch(
             headless=True,
-            executablePath='/usr/bin/google-chrome',
             args=[
                 '--no-sandbox',
                 '--disable-setuid-sandbox',
@@ -127,11 +132,7 @@ async def create_browser():
                 '--disable-web-security',
                 '--disable-features=VizDisplayCompositor',
             ],
-            handleSIGINT=False,
-            handleSIGTERM=False,
-            handleSIGHUP=False,
-            ignoreHTTPSErrors=True,
-            autoClose=True,  # Ensure browser closes automatically
+            ignore_default_args=['--enable-automation'],
         )
 
         browser_creation_times[browser] = time.time()
@@ -152,7 +153,7 @@ async def cleanup_browser(browser):
             del browser_creation_times[browser]
 
         # Close all pages first
-        pages = await browser.pages()
+        pages = browser.contexts[0].pages if browser.contexts else []
         for page in pages:
             try:
                 await page.close()
@@ -191,7 +192,9 @@ async def check_browser_health():
                             browser = await create_browser()
                         else:
                             # Quick health check
-                            await browser.pages()
+                            contexts = browser.contexts
+                            if contexts:
+                                pages = contexts[0].pages
 
                         # Put back in pool if healthy
                         if not browser_pool.full():
@@ -200,208 +203,190 @@ async def check_browser_health():
                             # Pool is full, cleanup this browser
                             await cleanup_browser(browser)
                     except Exception as e:
-                        print(f"Browser health check failed: {e}")
-                        # If unhealthy, close and create new
-                        await cleanup_browser(browser)
-                        if not browser_pool.full():
-                            new_browser = await create_browser()
-                            await browser_pool.put(new_browser)
+                        print(f"Error checking browser health: {e}")
+                        # Cleanup the problematic browser
+                        try:
+                            await cleanup_browser(browser)
+                        except:
+                            pass
+
+                # Create new browsers if pool is empty
+                while browser_pool.qsize() < MAX_BROWSERS:
+                    try:
+                        browser = await create_browser()
+                        await browser_pool.put(browser)
+                    except Exception as e:
+                        print(f"Error creating browser for pool: {e}")
+                        break
+
         except Exception as e:
-            print(f"Error in browser health check: {str(e)}")
+            print(f"Error in browser health check: {e}")
+            await asyncio.sleep(60)  # Wait before retrying
 
 async def force_cleanup_all_browsers():
-    """Force cleanup all browsers in emergency situations"""
+    """Force cleanup all browsers - useful for emergency situations"""
     print("Force cleaning up all browsers...")
 
-    # Clean up pool
-    while not browser_pool.empty():
-        try:
-            browser = await browser_pool.get_nowait()
-            await cleanup_browser(browser)
-        except asyncio.QueueEmpty:
-            break
+    async with browser_lock:
+        # Clean up browsers in pool
+        while not browser_pool.empty():
+            try:
+                browser = await browser_pool.get_nowait()
+                await cleanup_browser(browser)
+            except asyncio.QueueEmpty:
+                break
 
-    # Clean up active browsers
-    for browser in list(active_browsers):
-        await cleanup_browser(browser)
+        # Clean up any remaining active browsers
+        for browser in list(active_browsers):
+            try:
+                await cleanup_browser(browser)
+            except Exception as e:
+                print(f"Error force cleaning browser: {e}")
+
+    print("Force cleanup completed")
 
-# Initialize browser pool
 @app.on_event("startup")
 async def init_browser_pool():
-    """Initialize the browser pool with some browsers"""
-    try:
-        for _ in range(min(2, MAX_BROWSERS)):  # Start with 2 browsers instead of 3
+    """Initialize the browser pool on startup"""
+    print("Initializing browser pool...")
+
+    # Start browser health check task
+    asyncio.create_task(check_browser_health())
+
+    # Pre-populate pool with initial browsers
+    for _ in range(min(2, MAX_BROWSERS)):
+        try:
             browser = await create_browser()
             await browser_pool.put(browser)
-        print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
+        except Exception as e:
+            print(f"Error creating initial browser: {e}")
 
-        # Start browser health check task
-        asyncio.create_task(check_browser_health())
-    except Exception as e:
-        print(f"Error initializing browser pool: {e}")
+    print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
 
 @app.on_event("shutdown")
 async def cleanup_browser_pool():
-    """Clean up all browsers in the pool"""
+    """Cleanup browser pool on shutdown"""
+    print("Cleaning up browser pool...")
     await force_cleanup_all_browsers()
 
-# Signal handlers for graceful shutdown
+    # Stop playwright instance
+    global playwright_instance
+    if playwright_instance:
+        await playwright_instance.stop()
+        playwright_instance = None
+
+    print("Browser pool cleanup completed")
+
 def signal_handler(signum, frame):
+    """Handle shutdown signals"""
     print(f"Received signal {signum}, shutting down gracefully...")
-    asyncio.create_task(force_cleanup_all_browsers())
+    asyncio.create_task(cleanup_browser_pool())
+    exit(0)
 
-signal.signal(signal.SIGTERM, signal_handler)
+# Register signal handlers
 signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
 
-# Initialize SQLite database
 def init_db():
-    global DB_PATH
+    """Initialize the SQLite database"""
+    conn = sqlite3.connect('/db/cache.db')
+    cursor = conn.cursor()
 
-    # Try to use the mounted volume first
-    db_path = '/db/cache.db'
-    db_dir = os.path.dirname(db_path)
-
-    # Check if directory exists and is writable
-    dir_writable = False
-    if os.path.exists(db_dir):
-        try:
-            test_file = os.path.join(db_dir, '.write_test')
-            with open(test_file, 'w') as f:
-                f.write('test')
-            os.remove(test_file)
-            dir_writable = True
-        except (IOError, PermissionError):
-            print(f"Directory {db_dir} exists but is not writable")
-            dir_writable = False
-
-    # If directory doesn't exist or isn't writable, try to create it
-    if not os.path.exists(db_dir) or not dir_writable:
-        try:
-            os.makedirs(db_dir, exist_ok=True)
-            # Test if we can write to the directory
-            test_file = os.path.join(db_dir, '.write_test')
-            with open(test_file, 'w') as f:
-                f.write('test')
-            os.remove(test_file)
-            print(f"Created directory: {db_dir}")
-            dir_writable = True
-        except Exception as e:
-            print(f"Warning: Could not create or write to directory {db_dir}: {e}")
-            # Fallback to using a local database file
-            db_path = 'cache.db'
-            print(f"Using local database file: {db_path}")
-
-    try:
-        conn = sqlite3.connect(db_path)
-        cursor = conn.cursor()
-        cursor.execute('''
+    # Create cache table
+    cursor.execute('''
         CREATE TABLE IF NOT EXISTS cache (
-            url TEXT,
-            route TEXT,
-            data TEXT,
-            timestamp INTEGER,
-            PRIMARY KEY (url, route)
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            url TEXT NOT NULL,
+            route TEXT NOT NULL,
+            data TEXT NOT NULL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            UNIQUE(url, route)
         )
-        ''')
-        conn.commit()
-        conn.close()
-        print(f"Database initialized at {db_path}")
-        # Update the global DB_PATH
-        DB_PATH = db_path
-    except sqlite3.OperationalError as e:
-        print(f"Error initializing database at {db_path}: {e}")
-        # Fallback to using a local database file if the mounted volume has permission issues
-        db_path = 'cache.db'
-        print(f"Falling back to local database file: {db_path}")
-        try:
-            conn = sqlite3.connect(db_path)
-            cursor = conn.cursor()
-            cursor.execute('''
-            CREATE TABLE IF NOT EXISTS cache (
-                url TEXT,
-                route TEXT,
-                data TEXT,
-                timestamp INTEGER,
-                PRIMARY KEY (url, route)
-            )
-            ''')
-            conn.commit()
-            conn.close()
-            print(f"Local database initialized at {db_path}")
-            # Update the global DB_PATH
-            DB_PATH = db_path
-        except sqlite3.OperationalError as e2:
-            print(f"Error initializing local database: {e2}")
-            raise
+    ''')
 
-# Define the database path
-DB_PATH = '/db/cache.db'
+    # Create index for faster lookups
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_cache_url_route
+        ON cache(url, route)
+    ''')
 
-# Get cached data if it exists and is not older than the expiry time
-def get_cached_data(url, route):
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-    cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)  # Convert hours to seconds
-    cursor.execute(
-        "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
-        (url, route, cache_expiry)
-    )
-    result = cursor.fetchone()
-    conn.close()
+    # Create index for cleanup operations
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_cache_created_at
+        ON cache(created_at)
+    ''')
 
-    if result:
-        print(f"Cache hit for {url} on route {route}")
-        return json.loads(result[0])
-    return None
-
-# Save data to cache
-def save_to_cache(url, route, data):
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-    timestamp = int(time.time())
-
-    # Convert data to JSON string
-    data_json = json.dumps(data)
-
-    cursor.execute(
-        "INSERT OR REPLACE INTO cache (url, route, data, timestamp) VALUES (?, ?, ?, ?)",
-        (url, route, data_json, timestamp)
-    )
     conn.commit()
     conn.close()
-    print(f"Saved to cache: {url} on route {route}")
+    print("Database initialized")
 
-# Function to clean up old cache entries
-def cleanup_old_cache_entries():
+def get_cached_data(url, route):
+    """Get cached data for a URL and route"""
     try:
-        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
-        conn = sqlite3.connect(DB_PATH)
+        conn = sqlite3.connect('/db/cache.db')
         cursor = conn.cursor()
 
-        # Calculate the timestamp for entries older than the expiry time
-        expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
-        pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
+        cursor.execute('''
+            SELECT data, created_at FROM cache
+            WHERE url = ? AND route = ?
+        ''', (url, route))
 
-        # Get count of entries to be deleted (non-pagination)
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
-        count_non_pagination = cursor.fetchone()[0]
+        result = cursor.fetchone()
+        conn.close()
 
-        # Get count of pagination entries to be deleted
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
-        count_pagination = cursor.fetchone()[0]
+        if result:
+            data, created_at = result
+            created_time = datetime.fromisoformat(created_at)
 
-        # Delete old non-pagination entries
-        cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+            # Check if cache is still valid
+            if datetime.now() - created_time < timedelta(hours=CACHE_EXPIRY_HOURS):
+                return json.loads(data)
 
-        # Delete old pagination entries
-        cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
+        return None
+    except Exception as e:
+        print(f"Error getting cached data: {e}")
+        return None
+
+def save_to_cache(url, route, data):
+    """Save data to cache"""
+    try:
+        conn = sqlite3.connect('/db/cache.db')
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            INSERT OR REPLACE INTO cache (url, route, data, created_at)
+            VALUES (?, ?, ?, CURRENT_TIMESTAMP)
+        ''', (url, route, json.dumps(data)))
 
         conn.commit()
         conn.close()
-
-        print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
     except Exception as e:
-        print(f"Error during cache cleanup: {e}")
+        print(f"Error saving to cache: {e}")
+
+def cleanup_old_cache_entries():
+    """Clean up old cache entries"""
+    try:
+        conn = sqlite3.connect('/db/cache.db')
+        cursor = conn.cursor()
+
+        # Delete entries older than CACHE_EXPIRY_HOURS
+        cutoff_time = datetime.now() - timedelta(hours=CACHE_EXPIRY_HOURS)
+
+        cursor.execute('''
+            DELETE FROM cache
+            WHERE created_at < ?
+        ''', (cutoff_time.isoformat(),))
+
+        deleted_count = cursor.rowcount
+        conn.commit()
+        conn.close()
+
+        print(f"Cleaned up {deleted_count} old cache entries")
+    except Exception as e:
+        print(f"Error cleaning up cache: {e}")
+
+# Initialize database
+init_db()
 
 # Initialize scheduler for periodic cache cleanup
 scheduler = BackgroundScheduler()
@@ -412,74 +397,81 @@ scheduler.add_job(
     replace_existing=True
 )
 
-# Initialize database on startup
-init_db()
-
-# Start the scheduler when the application starts
 @app.on_event("startup")
 def start_scheduler():
     scheduler.start()
     print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
-    print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours")
 
-# Shutdown the scheduler when the application stops
 @app.on_event("shutdown")
 def shutdown_scheduler():
     scheduler.shutdown(wait=False)
     print("Cache cleanup scheduler stopped")
 
 async def wait_for_network_idle(page):
-    """Wait until no network requests are in flight"""
-    await page.waitForNetworkIdle(idleTime=500, timeout=30000)
+    """Wait for network to be idle"""
+    await page.wait_for_load_state('networkidle')
 
 @app.head("/")
 async def health_check():
-    return {"status": "ok"}
+    return {"status": "healthy"}
 
 async def safe_browser_operation(url, operation_func):
-    """Safely perform a browser operation with proper cleanup and resource limits"""
-    async with operation_semaphore:  # Limit concurrent operations
-        async with get_browser() as browser:
-            page = None
+    """Safely perform a browser operation with proper resource management"""
+    async with operation_semaphore:
+        browser = None
+        context = None
+        page = None
+
+        try:
+            # Get browser from pool or create new one
             try:
-                # Create a new page
-                page = await browser.newPage()
+                browser = await asyncio.wait_for(browser_pool.get(), timeout=10.0)
+            except asyncio.TimeoutError:
+                print("Timeout getting browser from pool, creating new one")
+                browser = await create_browser()
 
-                # Set reasonable viewport
-                await page.setViewport({'width': 1280, 'height': 800})
+            # Create context and page
+            context = await browser.new_context(
+                user_agent=CUSTOM_USER_AGENT,
+                viewport={'width': 1920, 'height': 1080},
+                ignore_https_errors=True,
+            )
 
-                # Set user agent
-                await page.setUserAgent(CUSTOM_USER_AGENT)
+            page = await context.new_page()
 
-                # Set reasonable timeout
-                page.setDefaultNavigationTimeout(30000)
+            # Set up request interception for better performance
+            await page.route("**/*", lambda route: route.abort()
+                           if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
+                           else route.continue_())
 
-                # Enable request interception to block unnecessary resources
-                await page.setRequestInterception(True)
+            # Perform the operation
+            result = await operation_func(page)
+            return result
 
-                async def intercept(request):
-                    # Block unnecessary resource types
-                    if request.resourceType in ['image', 'media', 'font', 'stylesheet']:
-                        await request.abort()
-                    else:
-                        await request.continue_()
-
-                page.on('request', lambda req: asyncio.ensure_future(intercept(req)))
-
-                # Perform the operation
-                result = await operation_func(page)
-
-                return result
-            except Exception as e:
-                print(f"Error during browser operation: {str(e)}")
-                raise
-            finally:
+        except Exception as e:
+            print(f"Error in browser operation: {e}")
+            raise
+        finally:
+            # Cleanup
+            if page:
                 try:
-                    # Ensure page is properly closed
-                    if page:
-                        await page.close()
-                except Exception as e:
-                    print(f"Error closing page: {str(e)}")
+                    await page.close()
+                except:
+                    pass
+            if context:
+                try:
+                    await context.close()
+                except:
+                    pass
+            if browser:
+                try:
+                    # Return browser to pool if it's still healthy
+                    if not browser_pool.full():
+                        await browser_pool.put(browser)
+                    else:
+                        await cleanup_browser(browser)
+                except:
+                    pass
 
 @app.get("/")
 async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
@@ -496,12 +488,9 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
         return cached_result
 
     try:
-        print(f"Visiting URL: {decoded_url}")
-
-        # Define the operation to perform with the browser
         async def visit_operation(page):
             try:
-                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
                 if not response:
                     print(f"Warning: No response object returned for {decoded_url}")
 
@@ -515,14 +504,10 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
                     content = await page.content()
                     return {"status": "partial", "content": content, "error": str(e)}
                 except:
-                    raise HTTPException(status_code=500, detail=f"Failed to get page content: {str(e)}")
+                    raise Exception(f"Failed to get page content: {str(e)}")
 
-        # Perform the operation
         result = await safe_browser_operation(decoded_url, visit_operation)
-
-        # Save to cache
         save_to_cache(decoded_url, "visit", result)
-
         return result
 
     except Exception as e:
@@ -545,12 +530,9 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
         return cached_result
 
     try:
-        print(f"Extracting SEO from: {decoded_url}")
-
-        # Define the operation to perform with the browser
         async def seo_operation(page):
             try:
-                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
 
                 # Extract SEO information
                 seo_data = await page.evaluate('''() => {
@@ -609,12 +591,8 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
                 print(f"Error during SEO extraction: {e}")
                 return {"status": "error", "url": decoded_url, "error": str(e)}
 
-        # Perform the operation
         result = await safe_browser_operation(decoded_url, seo_operation)
-
-        # Save to cache
         save_to_cache(decoded_url, "seo", result)
-
         return result
 
     except Exception as e:
@@ -636,52 +614,48 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
         return cached_result
 
     try:
-        print(f"Extracting meta tags from: {decoded_url}")
-
-        # Define the operation to perform with the browser
         async def meta_operation(page):
             try:
-                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
 
-                # Extract all meta tags
-                meta_tags = await page.evaluate('''() => {
-                    const metas = Array.from(document.querySelectorAll('meta'));
-                    return metas.map(meta => {
+                # Extract meta tags using Playwright
+                meta_data = await page.evaluate('''() => {
+                    const data = {
+                        meta_tags: [],
+                        open_graph: {},
+                        twitter_card: {},
+                        title: document.title || ''
+                    };
+
+                    // Extract all meta tags
+                    document.querySelectorAll('meta').forEach(meta => {
                         const attributes = {};
-                        Array.from(meta.attributes).forEach(attr => {
+                        for (let attr of meta.attributes) {
                             attributes[attr.name] = attr.value;
-                        });
-                        return attributes;
+                        }
+                        data.meta_tags.push(attributes);
                     });
-                }''')
 
-                # Extract Open Graph tags
-                og_tags = await page.evaluate('''() => {
-                    const ogTags = {};
-                    document.querySelectorAll('meta[property^="og:"]').forEach(tag => {
-                        const property = tag.getAttribute('property');
-                        ogTags[property] = tag.getAttribute('content');
+                    // Extract Open Graph tags
+                    document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
+                        data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content');
                     });
-                    return ogTags;
-                }''')
 
-                # Extract Twitter card tags
-                twitter_tags = await page.evaluate('''() => {
-                    const twitterTags = {};
-                    document.querySelectorAll('meta[name^="twitter:"]').forEach(tag => {
-                        const name = tag.getAttribute('name');
-                        twitterTags[name] = tag.getAttribute('content');
+                    // Extract Twitter card tags
+                    document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
+                        data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content');
                     });
-                    return twitterTags;
+
+                    return data;
                 }''')
 
                 result = {
                     "status": "success",
                     "url": decoded_url,
-                    "meta_tags": meta_tags,
-                    "open_graph": og_tags,
-                    "twitter_card": twitter_tags,
-                    "title": await page.title()
+                    "meta_tags": meta_data['meta_tags'],
+                    "open_graph": meta_data['open_graph'],
+                    "twitter_card": meta_data['twitter_card'],
+                    "title": meta_data['title']
                 }
 
                 return result
@@ -690,12 +664,8 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
                 print(f"Error during meta tag extraction: {e}")
                 return {"status": "error", "url": decoded_url, "error": str(e)}
 
-        # Perform the operation
         result = await safe_browser_operation(decoded_url, meta_operation)
-
-        # Save to cache
         save_to_cache(decoded_url, "meta", result)
-
         return result
 
     except Exception as e:
@@ -703,18 +673,22 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
 
 @app.get("/cache/clear")
 async def clear_cache(x_api_key: Optional[str] = Header(None)):
-    """Clear the entire cache database"""
+    """Clear all cached data"""
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
 
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("DELETE FROM cache")
-    conn.commit()
-    conn.close()
+    try:
+        conn = sqlite3.connect('/db/cache.db')
+        cursor = conn.cursor()
+        cursor.execute('DELETE FROM cache')
+        deleted_count = cursor.rowcount
+        conn.commit()
+        conn.close()
 
-    return {"status": "success", "message": "Cache cleared successfully"}
+        return {"status": "success", "message": f"Cleared {deleted_count} cache entries"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
 
 @app.get("/cache/stats")
 async def cache_stats(x_api_key: Optional[str] = Header(None)):
@@ -724,79 +698,103 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
         raise HTTPException(status_code=401, detail="Invalid API key")
 
     try:
-        conn = sqlite3.connect(DB_PATH)
+        conn = sqlite3.connect('/db/cache.db')
         cursor = conn.cursor()
 
         # Get total count
-        cursor.execute("SELECT COUNT(*) FROM cache")
+        cursor.execute('SELECT COUNT(*) FROM cache')
         total_count = cursor.fetchone()[0]
 
         # Get count by route
-        cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route")
+        cursor.execute('''
+            SELECT route, COUNT(*) as count
+            FROM cache
+            GROUP BY route
+        ''')
         route_counts = dict(cursor.fetchall())
 
         # Get oldest and newest entries
-        cursor.execute("SELECT MIN(timestamp), MAX(timestamp) FROM cache")
-        min_time, max_time = cursor.fetchone()
+        cursor.execute('''
+            SELECT MIN(created_at), MAX(created_at)
+            FROM cache
+        ''')
+        oldest, newest = cursor.fetchone()
+
+        # Get database size
+        cursor.execute('PRAGMA page_count')
+        page_count = cursor.fetchone()[0]
+        cursor.execute('PRAGMA page_size')
+        page_size = cursor.fetchone()[0]
+        db_size = page_count * page_size
 
         conn.close()
 
         return {
+            "status": "success",
             "total_entries": total_count,
             "route_counts": route_counts,
-            "oldest_entry": min_time,
-            "newest_entry": max_time
+            "oldest_entry": oldest,
+            "newest_entry": newest,
+            "database_size_bytes": db_size,
+            "cache_expiry_hours": CACHE_EXPIRY_HOURS
         }
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.get("/status")
 async def system_status(x_api_key: Optional[str] = Header(None)):
-    """Get system status and browser pool information"""
+    """Get system status and health information"""
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
 
     try:
         # Get system information
-        process = psutil.Process()
-        memory_info = process.memory_info()
+        cpu_percent = psutil.cpu_percent(interval=1)
+        memory = psutil.virtual_memory()
+        disk = psutil.disk_usage('/')
 
         # Get browser pool information
         pool_size = browser_pool.qsize()
         active_browser_count = len(active_browsers)
 
-        # Calculate browser ages
-        browser_ages = []
-        for browser, creation_time in browser_creation_times.items():
-            age = time.time() - creation_time
-            browser_ages.append(age)
+        # Get cache statistics
+        conn = sqlite3.connect('/db/cache.db')
+        cursor = conn.cursor()
+        cursor.execute('SELECT COUNT(*) FROM cache')
+        cache_count = cursor.fetchone()[0]
+        conn.close()
 
         return {
+            "status": "success",
             "system": {
-                "cpu_percent": process.cpu_percent(),
-                "memory_mb": memory_info.rss / 1024 / 1024,
-                "memory_percent": process.memory_percent(),
-                "open_files": len(process.open_files()),
-                "connections": len(process.connections()),
-                "threads": process.num_threads()
+                "cpu_percent": cpu_percent,
+                "memory_percent": memory.percent,
+                "memory_available_gb": round(memory.available / (1024**3), 2),
+                "disk_percent": disk.percent,
+                "disk_free_gb": round(disk.free / (1024**3), 2)
             },
             "browser_pool": {
                 "pool_size": pool_size,
-                "active_browsers": active_browser_count,
                 "max_browsers": MAX_BROWSERS,
-                "browser_ttl_seconds": BROWSER_TTL,
-                "browser_ages_seconds": browser_ages,
-                "concurrent_operations_limit": MAX_CONCURRENT_OPERATIONS
+                "active_browsers": active_browser_count,
+                "browser_ttl_seconds": BROWSER_TTL
             },
-            "timestamp": time.time()
+            "cache": {
+                "total_entries": cache_count,
+                "expiry_hours": CACHE_EXPIRY_HOURS
+            },
+            "rate_limiting": {
+                "requests_per_minute": RATE_LIMIT_MINUTE,
+                "window_seconds": RATE_LIMIT_WINDOW
+            }
         }
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.post("/emergency-cleanup")
 async def emergency_cleanup(x_api_key: Optional[str] = Header(None)):
-    """Force emergency cleanup of all browsers"""
+    """Emergency cleanup endpoint to force cleanup all browsers"""
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
@@ -809,60 +807,14 @@ async def emergency_cleanup(x_api_key: Optional[str] = Header(None)):
 
 @asynccontextmanager
 async def get_browser():
-    """Get a browser from the pool or create a new one if needed"""
+    """Context manager for getting a browser from the pool"""
     browser = None
     try:
-        # Try to get a browser from the pool with timeout
-        try:
-            browser = await asyncio.wait_for(browser_pool.get(), timeout=30.0)
-        except (asyncio.QueueEmpty, asyncio.TimeoutError):
-            # If pool is empty or timeout, create a new browser if under the limit
-            async with browser_lock:
-                current_browser_count = len(active_browsers)
-                if current_browser_count < MAX_BROWSERS:
-                    browser = await create_browser()
-                else:
-                    # If at limit, wait for a browser to become available with timeout
-                    try:
-                        browser = await asyncio.wait_for(browser_pool.get(), timeout=60.0)
-                    except asyncio.TimeoutError:
-                        # Emergency cleanup and create new browser
-                        print("Emergency: Timeout waiting for browser, forcing cleanup")
-                        await force_cleanup_all_browsers()
-                        browser = await create_browser()
-
-        yield browser
-    except Exception as e:
-        print(f"Error in get_browser: {e}")
-        # Emergency cleanup if we can't get a browser
-        await force_cleanup_all_browsers()
-        browser = await create_browser()
+        browser = await browser_pool.get()
         yield browser
     finally:
-        # Return browser to pool if it's still viable
         if browser:
-            try:
-                # Quick check if browser is still usable
-                await browser.pages()
-                # Check if browser is too old
-                if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
-                    print(f"Recycling old browser in get_browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
-                    await cleanup_browser(browser)
-                    browser = await create_browser()
-
-                # Only put back if pool is not full
-                if not browser_pool.full():
-                    await browser_pool.put(browser)
-                else:
-                    # Pool is full, cleanup this browser
-                    await cleanup_browser(browser)
-            except Exception as e:
-                print(f"Browser health check failed in get_browser: {e}")
-                # If browser is not usable, close it and create a new one
-                await cleanup_browser(browser)
-                if not browser_pool.full():
-                    new_browser = await create_browser()
-                    await browser_pool.put(new_browser)
+            await browser_pool.put(browser)
 
 if __name__ == "__main__":
     import uvicorn