diff --git a/Dockers/puppeteer-api/Dockerfile b/Dockers/puppeteer-api/Dockerfile index bbd39ff..d849b37 100644 --- a/Dockers/puppeteer-api/Dockerfile +++ b/Dockers/puppeteer-api/Dockerfile @@ -36,6 +36,8 @@ COPY . . # Set environment variables ENV PYTHONUNBUFFERED=1 ENV API_KEY="" +ENV CACHE_EXPIRY_HOURS=36 +ENV CLEANUP_CRON="0 3 * * *" # Create a non-root user and switch to it RUN useradd -m puppeteer && chown -R puppeteer:puppeteer /app diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index cfaae7d..8a322fa 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -9,6 +9,8 @@ import time from datetime import datetime, timedelta from typing import Optional, Dict, List, Any from urllib.parse import unquote +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger app = FastAPI() @@ -17,6 +19,12 @@ API_KEY = os.getenv('API_KEY') if not API_KEY: raise ValueError("API_KEY environment variable must be set") +# Get cache expiry time from environment variable (default: 36 hours) +CACHE_EXPIRY_HOURS = int(os.getenv('CACHE_EXPIRY_HOURS', '36')) + +# Get cleanup cron schedule from environment variable (default: every day at 3 AM) +CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *') + # Define custom user agent CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36' @@ -104,11 +112,11 @@ def init_db(): # Define the database path DB_PATH = '/db/cache.db' -# Get cached data if it exists and is not older than 36 hours +# Get cached data if it exists and is not older than the expiry time def get_cached_data(url, route): conn = sqlite3.connect(DB_PATH) cursor = conn.cursor() - cache_expiry = int(time.time()) - (36 * 60 * 60) # 36 hours in seconds + cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds cursor.execute( "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?", (url, route, cache_expiry) @@ -138,9 +146,54 @@ def save_to_cache(url, route, data): conn.close() print(f"Saved to cache: {url} on route {route}") +# Function to clean up old cache entries +def cleanup_old_cache_entries(): + try: + print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)") + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + + # Calculate the timestamp for entries older than the expiry time + expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) + + # Get count of entries to be deleted + cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,)) + count = cursor.fetchone()[0] + + # Delete old entries + cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,)) + conn.commit() + conn.close() + + print(f"Cache cleanup completed: {count} entries removed") + except Exception as e: + print(f"Error during cache cleanup: {e}") + +# Initialize scheduler for periodic cache cleanup +scheduler = BackgroundScheduler() +scheduler.add_job( + cleanup_old_cache_entries, + CronTrigger.from_crontab(CLEANUP_CRON), + id='cache_cleanup_job', + replace_existing=True +) + # Initialize database on startup init_db() +# Start the scheduler when the application starts +@app.on_event("startup") +def start_scheduler(): + scheduler.start() + print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}") + print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours") + +# Shutdown the scheduler when the application stops +@app.on_event("shutdown") +def shutdown_scheduler(): + scheduler.shutdown(wait=False) + print("Cache cleanup scheduler stopped") + async def wait_for_network_idle(page): """Wait until no network requests are in flight""" await page.waitForNetworkIdle(idleTime=500, timeout=30000) @@ -149,6 +202,43 @@ async def wait_for_network_idle(page): async def health_check(): return {"status": "ok"} +async def safe_browser_operation(url, operation_func): + """Safely perform browser operations with proper cleanup""" + browser = None + try: + browser = await launch( + headless=True, + executablePath='/usr/bin/google-chrome', + args=['--no-sandbox', '--disable-setuid-sandbox'], + handleSIGINT=False, + handleSIGTERM=False, + handleSIGHUP=False + ) + + # Create new page with timeout + page = await browser.newPage() + page.setDefaultNavigationTimeout(30000) + + # Set custom user agent + await page.setUserAgent(CUSTOM_USER_AGENT) + + # Call the operation function that uses the page + result = await operation_func(page) + + # Explicitly close the page + await page.close() + + return result + + finally: + # Ensure browser is closed properly + if browser: + try: + await browser.close() + except Exception as e: + print(f"Error closing browser: {e}") + # We don't re-raise here to avoid masking the original error + @app.get("/") async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): # Validate API key @@ -163,33 +253,30 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): if cached_result: return cached_result - browser = None try: - print(decoded_url) + print(f"Visiting URL: {decoded_url}") - # Launch browser using installed Chrome - browser = await launch( - headless=True, - executablePath='/usr/bin/google-chrome', - args=['--no-sandbox', '--disable-setuid-sandbox'] - ) + # Define the operation to perform with the browser + async def visit_operation(page): + try: + response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000) + if not response: + print(f"Warning: No response object returned for {decoded_url}") - # Create new page - page = await browser.newPage() + # Get page content + content = await page.content() + return {"status": "success", "content": content} + except Exception as e: + print(f"Error during page navigation: {e}") + # Try to get content anyway + try: + content = await page.content() + return {"status": "partial", "content": content, "error": str(e)} + except: + raise HTTPException(status_code=500, detail=f"Failed to get page content: {str(e)}") - # Set custom user agent - await page.setUserAgent(CUSTOM_USER_AGENT) - - # Navigate to URL and wait for network idle - await page.goto(decoded_url, waitUntil='networkidle2') - - # Get page content - content = await page.content() - - # Close page - await page.close() - - result = {"status": "success", "content": content} + # Perform the operation + result = await safe_browser_operation(decoded_url, visit_operation) # Save to cache save_to_cache(decoded_url, "visit", result) @@ -197,13 +284,9 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): return result except Exception as e: + print(f"Error visiting URL {decoded_url}: {e}") raise HTTPException(status_code=500, detail=str(e)) - finally: - # Ensure browser is closed even if an error occurs - if browser: - await browser.close() - @app.get("/seo") async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)): """Extract SEO information from a website""" @@ -219,112 +302,17 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)): if cached_result: return cached_result - browser = None try: print(f"Extracting SEO from: {decoded_url}") - # Launch browser using installed Chrome - browser = await launch( - headless=True, - executablePath='/usr/bin/google-chrome', - args=['--no-sandbox', '--disable-setuid-sandbox'] - ) - - # Create new page - page = await browser.newPage() - - # Set custom user agent - await page.setUserAgent(CUSTOM_USER_AGENT) - - # Navigate to URL and wait for network idle - await page.goto(decoded_url, waitUntil='networkidle2') - - # Extract JSON-LD data - json_ld_data = await page.evaluate('''() => { - const jsonLdScripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); - return jsonLdScripts.map(script => { - try { - return JSON.parse(script.textContent); - } catch (e) { - return { error: "Failed to parse JSON-LD", content: script.textContent }; - } - }); - }''') - - # Extract meta tags - meta_tags = await page.evaluate('''() => { - const metaTags = Array.from(document.querySelectorAll('meta')); - return metaTags.map(tag => { - const attributes = Array.from(tag.attributes); - const result = {}; - attributes.forEach(attr => { - result[attr.name] = attr.value; - }); - return result; - }); - }''') - - # Extract title - title = await page.evaluate('document.title') - - # Extract canonical URL - canonical_url = await page.evaluate('''() => { - const link = document.querySelector('link[rel="canonical"]'); - return link ? link.href : null; - }''') - - # Extract Open Graph data - og_data = await page.evaluate('''() => { - const ogTags = Array.from(document.querySelectorAll('meta[property^="og:"]')); - const result = {}; - ogTags.forEach(tag => { - const property = tag.getAttribute('property'); - const content = tag.getAttribute('content'); - result[property] = content; - }); - return result; - }''') - - # Extract Twitter Card data - twitter_data = await page.evaluate('''() => { - const twitterTags = Array.from(document.querySelectorAll('meta[name^="twitter:"]')); - const result = {}; - twitterTags.forEach(tag => { - const name = tag.getAttribute('name'); - const content = tag.getAttribute('content'); - result[name] = content; - }); - return result; - }''') - - # Close page - await page.close() - - # Compile all SEO data - seo_data = { - "title": title, - "canonical_url": canonical_url, - "json_ld": json_ld_data, - "meta_tags": meta_tags, - "open_graph": og_data, - "twitter_card": twitter_data - } - - result = {"status": "success", "seo_data": seo_data} - - # Save to cache - save_to_cache(decoded_url, "seo", result) + # Use the safe_browser_operation function to perform the browser operation + result = await safe_browser_operation(decoded_url, asyncio.run) return result except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - finally: - # Ensure browser is closed even if an error occurs - if browser: - await browser.close() - @app.get("/meta") async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)): """Extract meta tags from a website""" @@ -340,65 +328,17 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)): if cached_result: return cached_result - browser = None try: print(f"Extracting meta tags from: {decoded_url}") - # Launch browser using installed Chrome - browser = await launch( - headless=True, - executablePath='/usr/bin/google-chrome', - args=['--no-sandbox', '--disable-setuid-sandbox'] - ) - - # Create new page - page = await browser.newPage() - - # Set custom user agent - await page.setUserAgent(CUSTOM_USER_AGENT) - - # Navigate to URL and wait for network idle - await page.goto(decoded_url, waitUntil='networkidle2') - - # Extract meta tags - meta_tags = await page.evaluate('''() => { - const metaTags = Array.from(document.querySelectorAll('meta')); - return metaTags.map(tag => { - const attributes = Array.from(tag.attributes); - const result = {}; - attributes.forEach(attr => { - result[attr.name] = attr.value; - }); - return result; - }); - }''') - - # Extract title - title = await page.evaluate('document.title') - - # Close page - await page.close() - - result = { - "status": "success", - "url": decoded_url, - "title": title, - "meta_tags": meta_tags - } - - # Save to cache - save_to_cache(decoded_url, "meta", result) + # Use the safe_browser_operation function to perform the browser operation + result = await safe_browser_operation(decoded_url, asyncio.run) return result except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - finally: - # Ensure browser is closed even if an error occurs - if browser: - await browser.close() - @app.get("/cache/clear") async def clear_cache(x_api_key: Optional[str] = Header(None)): """Clear the entire cache database""" @@ -437,6 +377,16 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)): cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,)) recent_entries = cursor.fetchone()[0] + # Get oldest entry timestamp + cursor.execute("SELECT MIN(timestamp) FROM cache") + oldest_timestamp = cursor.fetchone()[0] + oldest_date = datetime.fromtimestamp(oldest_timestamp).isoformat() if oldest_timestamp else None + + # Get newest entry timestamp + cursor.execute("SELECT MAX(timestamp) FROM cache") + newest_timestamp = cursor.fetchone()[0] + newest_date = datetime.fromtimestamp(newest_timestamp).isoformat() if newest_timestamp else None + conn.close() return { @@ -444,7 +394,11 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)): "stats": { "total_entries": total_entries, "entries_by_route": routes, - "recent_entries": recent_entries + "recent_entries": recent_entries, + "oldest_entry": oldest_date, + "newest_entry": newest_date, + "cache_expiry_hours": CACHE_EXPIRY_HOURS, + "cleanup_schedule": CLEANUP_CRON } } diff --git a/Dockers/puppeteer-api/requirements.txt b/Dockers/puppeteer-api/requirements.txt index e782dc1..c876748 100644 --- a/Dockers/puppeteer-api/requirements.txt +++ b/Dockers/puppeteer-api/requirements.txt @@ -1,4 +1,5 @@ fastapi==0.68.1 uvicorn==0.15.0 pyppeteer==1.0.2 -psutil==6.0.0 \ No newline at end of file +psutil==6.0.0 +apscheduler \ No newline at end of file