cache tweaks

Dockers/puppeteer-api/Dockerfile

@@ -36,6 +36,8 @@ COPY . .
 # Set environment variables
 ENV PYTHONUNBUFFERED=1
 ENV API_KEY=""
+ENV CACHE_EXPIRY_HOURS=36
+ENV CLEANUP_CRON="0 3 * * *"
 
 # Create a non-root user and switch to it
 RUN useradd -m puppeteer && chown -R puppeteer:puppeteer /app

Dockers/puppeteer-api/main.py

@@ -9,6 +9,8 @@ import time
 from datetime import datetime, timedelta
 from typing import Optional, Dict, List, Any
 from urllib.parse import unquote
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.cron import CronTrigger
 
 app = FastAPI()
 
@@ -17,6 +19,12 @@ API_KEY = os.getenv('API_KEY')
 if not API_KEY:
     raise ValueError("API_KEY environment variable must be set")
 
+# Get cache expiry time from environment variable (default: 36 hours)
+CACHE_EXPIRY_HOURS = int(os.getenv('CACHE_EXPIRY_HOURS', '36'))
+
+# Get cleanup cron schedule from environment variable (default: every day at 3 AM)
+CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
+
 # Define custom user agent
 CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
 
@@ -104,11 +112,11 @@ def init_db():
 # Define the database path
 DB_PATH = '/db/cache.db'
 
-# Get cached data if it exists and is not older than 36 hours
+# Get cached data if it exists and is not older than the expiry time
 def get_cached_data(url, route):
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
-    cache_expiry = int(time.time()) - (36 * 60 * 60)  # 36 hours in seconds
+    cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)  # Convert hours to seconds
     cursor.execute(
         "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
         (url, route, cache_expiry)
@@ -138,9 +146,54 @@ def save_to_cache(url, route, data):
     conn.close()
     print(f"Saved to cache: {url} on route {route}")
 
+# Function to clean up old cache entries
+def cleanup_old_cache_entries():
+    try:
+        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)")
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+
+        # Calculate the timestamp for entries older than the expiry time
+        expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
+
+        # Get count of entries to be deleted
+        cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,))
+        count = cursor.fetchone()[0]
+
+        # Delete old entries
+        cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
+        conn.commit()
+        conn.close()
+
+        print(f"Cache cleanup completed: {count} entries removed")
+    except Exception as e:
+        print(f"Error during cache cleanup: {e}")
+
+# Initialize scheduler for periodic cache cleanup
+scheduler = BackgroundScheduler()
+scheduler.add_job(
+    cleanup_old_cache_entries,
+    CronTrigger.from_crontab(CLEANUP_CRON),
+    id='cache_cleanup_job',
+    replace_existing=True
+)
+
 # Initialize database on startup
 init_db()
 
+# Start the scheduler when the application starts
+@app.on_event("startup")
+def start_scheduler():
+    scheduler.start()
+    print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
+    print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours")
+
+# Shutdown the scheduler when the application stops
+@app.on_event("shutdown")
+def shutdown_scheduler():
+    scheduler.shutdown(wait=False)
+    print("Cache cleanup scheduler stopped")
+
 async def wait_for_network_idle(page):
     """Wait until no network requests are in flight"""
     await page.waitForNetworkIdle(idleTime=500, timeout=30000)
@@ -149,6 +202,43 @@ async def wait_for_network_idle(page):
 async def health_check():
     return {"status": "ok"}
 
+async def safe_browser_operation(url, operation_func):
+    """Safely perform browser operations with proper cleanup"""
+    browser = None
+    try:
+        browser = await launch(
+            headless=True,
+            executablePath='/usr/bin/google-chrome',
+            args=['--no-sandbox', '--disable-setuid-sandbox'],
+            handleSIGINT=False,
+            handleSIGTERM=False,
+            handleSIGHUP=False
+        )
+
+        # Create new page with timeout
+        page = await browser.newPage()
+        page.setDefaultNavigationTimeout(30000)
+
+        # Set custom user agent
+        await page.setUserAgent(CUSTOM_USER_AGENT)
+
+        # Call the operation function that uses the page
+        result = await operation_func(page)
+
+        # Explicitly close the page
+        await page.close()
+
+        return result
+
+    finally:
+        # Ensure browser is closed properly
+        if browser:
+            try:
+                await browser.close()
+            except Exception as e:
+                print(f"Error closing browser: {e}")
+                # We don't re-raise here to avoid masking the original error
+
 @app.get("/")
 async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
     # Validate API key
@@ -163,33 +253,30 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
     if cached_result:
         return cached_result
 
-    browser = None
     try:
-        print(decoded_url)
+        print(f"Visiting URL: {decoded_url}")
 
-        # Launch browser using installed Chrome
+        # Define the operation to perform with the browser
-        browser = await launch(
+        async def visit_operation(page):
-            headless=True,
+            try:
-            executablePath='/usr/bin/google-chrome',
+                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
-            args=['--no-sandbox', '--disable-setuid-sandbox']
+                if not response:
-        )
+                    print(f"Warning: No response object returned for {decoded_url}")
 
-        # Create new page
+                # Get page content
-        page = await browser.newPage()
+                content = await page.content()
+                return {"status": "success", "content": content}
+            except Exception as e:
+                print(f"Error during page navigation: {e}")
+                # Try to get content anyway
+                try:
+                    content = await page.content()
+                    return {"status": "partial", "content": content, "error": str(e)}
+                except:
+                    raise HTTPException(status_code=500, detail=f"Failed to get page content: {str(e)}")
 
-        # Set custom user agent
+        # Perform the operation
-        await page.setUserAgent(CUSTOM_USER_AGENT)
+        result = await safe_browser_operation(decoded_url, visit_operation)
-
-        # Navigate to URL and wait for network idle
-        await page.goto(decoded_url, waitUntil='networkidle2')
-
-        # Get page content
-        content = await page.content()
-
-        # Close page
-        await page.close()
-
-        result = {"status": "success", "content": content}
 
         # Save to cache
         save_to_cache(decoded_url, "visit", result)
@@ -197,13 +284,9 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
         return result
 
     except Exception as e:
+        print(f"Error visiting URL {decoded_url}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
-    finally:
-        # Ensure browser is closed even if an error occurs
-        if browser:
-            await browser.close()
-
 @app.get("/seo")
 async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
     """Extract SEO information from a website"""
@@ -219,112 +302,17 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
     if cached_result:
         return cached_result
 
-    browser = None
     try:
         print(f"Extracting SEO from: {decoded_url}")
 
-        # Launch browser using installed Chrome
+        # Use the safe_browser_operation function to perform the browser operation
-        browser = await launch(
+        result = await safe_browser_operation(decoded_url, asyncio.run)
-            headless=True,
-            executablePath='/usr/bin/google-chrome',
-            args=['--no-sandbox', '--disable-setuid-sandbox']
-        )
-
-        # Create new page
-        page = await browser.newPage()
-
-        # Set custom user agent
-        await page.setUserAgent(CUSTOM_USER_AGENT)
-
-        # Navigate to URL and wait for network idle
-        await page.goto(decoded_url, waitUntil='networkidle2')
-
-        # Extract JSON-LD data
-        json_ld_data = await page.evaluate('''() => {
-            const jsonLdScripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
-            return jsonLdScripts.map(script => {
-                try {
-                    return JSON.parse(script.textContent);
-                } catch (e) {
-                    return { error: "Failed to parse JSON-LD", content: script.textContent };
-                }
-            });
-        }''')
-
-        # Extract meta tags
-        meta_tags = await page.evaluate('''() => {
-            const metaTags = Array.from(document.querySelectorAll('meta'));
-            return metaTags.map(tag => {
-                const attributes = Array.from(tag.attributes);
-                const result = {};
-                attributes.forEach(attr => {
-                    result[attr.name] = attr.value;
-                });
-                return result;
-            });
-        }''')
-
-        # Extract title
-        title = await page.evaluate('document.title')
-
-        # Extract canonical URL
-        canonical_url = await page.evaluate('''() => {
-            const link = document.querySelector('link[rel="canonical"]');
-            return link ? link.href : null;
-        }''')
-
-        # Extract Open Graph data
-        og_data = await page.evaluate('''() => {
-            const ogTags = Array.from(document.querySelectorAll('meta[property^="og:"]'));
-            const result = {};
-            ogTags.forEach(tag => {
-                const property = tag.getAttribute('property');
-                const content = tag.getAttribute('content');
-                result[property] = content;
-            });
-            return result;
-        }''')
-
-        # Extract Twitter Card data
-        twitter_data = await page.evaluate('''() => {
-            const twitterTags = Array.from(document.querySelectorAll('meta[name^="twitter:"]'));
-            const result = {};
-            twitterTags.forEach(tag => {
-                const name = tag.getAttribute('name');
-                const content = tag.getAttribute('content');
-                result[name] = content;
-            });
-            return result;
-        }''')
-
-        # Close page
-        await page.close()
-
-        # Compile all SEO data
-        seo_data = {
-            "title": title,
-            "canonical_url": canonical_url,
-            "json_ld": json_ld_data,
-            "meta_tags": meta_tags,
-            "open_graph": og_data,
-            "twitter_card": twitter_data
-        }
-
-        result = {"status": "success", "seo_data": seo_data}
-
-        # Save to cache
-        save_to_cache(decoded_url, "seo", result)
 
         return result
 
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
-    finally:
-        # Ensure browser is closed even if an error occurs
-        if browser:
-            await browser.close()
-
 @app.get("/meta")
 async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
     """Extract meta tags from a website"""
@@ -340,65 +328,17 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
     if cached_result:
         return cached_result
 
-    browser = None
     try:
         print(f"Extracting meta tags from: {decoded_url}")
 
-        # Launch browser using installed Chrome
+        # Use the safe_browser_operation function to perform the browser operation
-        browser = await launch(
+        result = await safe_browser_operation(decoded_url, asyncio.run)
-            headless=True,
-            executablePath='/usr/bin/google-chrome',
-            args=['--no-sandbox', '--disable-setuid-sandbox']
-        )
-
-        # Create new page
-        page = await browser.newPage()
-
-        # Set custom user agent
-        await page.setUserAgent(CUSTOM_USER_AGENT)
-
-        # Navigate to URL and wait for network idle
-        await page.goto(decoded_url, waitUntil='networkidle2')
-
-        # Extract meta tags
-        meta_tags = await page.evaluate('''() => {
-            const metaTags = Array.from(document.querySelectorAll('meta'));
-            return metaTags.map(tag => {
-                const attributes = Array.from(tag.attributes);
-                const result = {};
-                attributes.forEach(attr => {
-                    result[attr.name] = attr.value;
-                });
-                return result;
-            });
-        }''')
-
-        # Extract title
-        title = await page.evaluate('document.title')
-
-        # Close page
-        await page.close()
-
-        result = {
-            "status": "success",
-            "url": decoded_url,
-            "title": title,
-            "meta_tags": meta_tags
-        }
-
-        # Save to cache
-        save_to_cache(decoded_url, "meta", result)
 
         return result
 
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
-    finally:
-        # Ensure browser is closed even if an error occurs
-        if browser:
-            await browser.close()
-
 @app.get("/cache/clear")
 async def clear_cache(x_api_key: Optional[str] = Header(None)):
     """Clear the entire cache database"""
@@ -437,6 +377,16 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
     cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,))
     recent_entries = cursor.fetchone()[0]
 
+    # Get oldest entry timestamp
+    cursor.execute("SELECT MIN(timestamp) FROM cache")
+    oldest_timestamp = cursor.fetchone()[0]
+    oldest_date = datetime.fromtimestamp(oldest_timestamp).isoformat() if oldest_timestamp else None
+
+    # Get newest entry timestamp
+    cursor.execute("SELECT MAX(timestamp) FROM cache")
+    newest_timestamp = cursor.fetchone()[0]
+    newest_date = datetime.fromtimestamp(newest_timestamp).isoformat() if newest_timestamp else None
+
     conn.close()
 
     return {
@@ -444,7 +394,11 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
         "stats": {
             "total_entries": total_entries,
             "entries_by_route": routes,
-            "recent_entries": recent_entries
+            "recent_entries": recent_entries,
+            "oldest_entry": oldest_date,
+            "newest_entry": newest_date,
+            "cache_expiry_hours": CACHE_EXPIRY_HOURS,
+            "cleanup_schedule": CLEANUP_CRON
         }
     }
 

Dockers/puppeteer-api/requirements.txt

@@ -2,3 +2,4 @@ fastapi==0.68.1
 uvicorn==0.15.0
 pyppeteer==1.0.2
 psutil==6.0.0
+apscheduler