913 lines
32 KiB
Python
913 lines
32 KiB
Python
from fastapi import FastAPI, HTTPException, Header, Request
|
|
from playwright.async_api import async_playwright
|
|
import os
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
import signal
|
|
import psutil
|
|
from datetime import datetime, timedelta
|
|
from typing import Optional, Dict, List, Any
|
|
from urllib.parse import unquote
|
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
from fastapi.middleware.base import BaseHTTPMiddleware
|
|
from app.config import BROWSER_INSTANCE_TIMEOUT_MINUTES
|
|
|
|
# Add imports for browser pool
|
|
from asyncio import Queue, Lock, Semaphore
|
|
from contextlib import asynccontextmanager
|
|
from app.utils.browser_utils import force_cleanup_old_pages
|
|
|
|
app = FastAPI()
|
|
|
|
# Get API key from environment variable
|
|
API_KEY = os.getenv('API_KEY')
|
|
if not API_KEY:
|
|
raise ValueError("API_KEY environment variable must be set")
|
|
|
|
# Get cache expiry time from environment variable (default: 36 hours)
|
|
CACHE_EXPIRY_HOURS = int(os.getenv('CACHE_EXPIRY_HOURS', '36'))
|
|
|
|
# Get cleanup cron schedule from environment variable (default: every day at 3 AM)
|
|
CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
|
|
|
|
# Define custom user agent
|
|
CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
|
|
|
|
# Browser pool configuration - Reduced for better resource management
|
|
MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '3')) # Reduced from 5 to 3
|
|
BROWSER_TTL = int(os.getenv('BROWSER_TTL', '1800')) # Reduced from 3600 to 1800 seconds (30 minutes)
|
|
MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '5')) # New: limit concurrent operations
|
|
|
|
# Browser pool management
|
|
browser_pool = Queue(maxsize=MAX_BROWSERS) # Add maxsize to prevent unbounded growth
|
|
browser_lock = Lock()
|
|
browser_creation_times = {}
|
|
page_creation_times = {} # Track page creation times for force cleanup
|
|
active_browsers = set() # Track active browsers
|
|
active_pages = set() # Track active pages for force cleanup
|
|
operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS) # Limit concurrent operations
|
|
playwright_instance = None # Global playwright instance
|
|
|
|
# Rate limiting configuration
|
|
RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60')) # requests per minute
|
|
RATE_LIMIT_WINDOW = 60 # window size in seconds
|
|
|
|
class RateLimitMiddleware(BaseHTTPMiddleware):
|
|
def __init__(self, app):
|
|
super().__init__(app)
|
|
self.requests = {}
|
|
self.lock = asyncio.Lock()
|
|
|
|
async def dispatch(self, request: Request, call_next):
|
|
# Skip rate limiting for health check
|
|
if request.url.path == "/" and request.method == "HEAD":
|
|
return await call_next(request)
|
|
|
|
api_key = request.headers.get("x-api-key")
|
|
if not api_key:
|
|
raise HTTPException(status_code=401, detail="API key required")
|
|
|
|
async with self.lock:
|
|
now = time.time()
|
|
# Clean old requests
|
|
self.requests = {k: v for k, v in self.requests.items()
|
|
if now - v[-1] < RATE_LIMIT_WINDOW}
|
|
|
|
# Get request times for this API key
|
|
requests = self.requests.get(api_key, [])
|
|
# Remove old requests outside the window
|
|
requests = [t for t in requests if now - t < RATE_LIMIT_WINDOW]
|
|
|
|
if len(requests) >= RATE_LIMIT_MINUTE:
|
|
oldest = requests[0]
|
|
wait_time = RATE_LIMIT_WINDOW - (now - oldest)
|
|
raise HTTPException(
|
|
status_code=429,
|
|
detail=f"Rate limit exceeded. Try again in {int(wait_time)} seconds"
|
|
)
|
|
|
|
requests.append(now)
|
|
self.requests[api_key] = requests
|
|
|
|
return await call_next(request)
|
|
|
|
# Add rate limiting middleware
|
|
app.add_middleware(RateLimitMiddleware)
|
|
|
|
async def create_browser():
|
|
"""Create a new browser instance with improved resource management"""
|
|
global playwright_instance
|
|
|
|
try:
|
|
if playwright_instance is None:
|
|
playwright_instance = await async_playwright().start()
|
|
|
|
browser = await playwright_instance.chromium.launch(
|
|
headless=True,
|
|
args=[
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-accelerated-2d-canvas',
|
|
'--disable-gpu',
|
|
'--disable-extensions',
|
|
'--disable-sync',
|
|
'--disable-background-networking',
|
|
'--disable-default-apps',
|
|
'--disable-translate',
|
|
'--disable-background-timer-throttling',
|
|
'--disable-backgrounding-occluded-windows',
|
|
'--disable-client-side-phishing-detection',
|
|
'--disable-features=site-per-process',
|
|
'--disable-hang-monitor',
|
|
'--disable-ipc-flooding-protection',
|
|
'--disable-popup-blocking',
|
|
'--disable-prompt-on-repost',
|
|
'--disable-renderer-backgrounding',
|
|
'--memory-pressure-off',
|
|
'--no-first-run',
|
|
'--safebrowsing-disable-auto-update',
|
|
'--max_old_space_size=512', # Limit memory usage
|
|
'--single-process', # Use single process to reduce resource usage
|
|
'--disable-web-security',
|
|
'--disable-features=VizDisplayCompositor',
|
|
],
|
|
ignore_default_args=['--enable-automation'],
|
|
)
|
|
|
|
browser_creation_times[browser] = time.time()
|
|
active_browsers.add(browser)
|
|
print(f"Created new browser instance. Total active browsers: {len(active_browsers)}")
|
|
return browser
|
|
except Exception as e:
|
|
print(f"Error creating browser: {e}")
|
|
raise
|
|
|
|
async def cleanup_browser(browser):
|
|
"""Safely cleanup a browser instance"""
|
|
try:
|
|
if browser in active_browsers:
|
|
active_browsers.remove(browser)
|
|
|
|
if browser in browser_creation_times:
|
|
del browser_creation_times[browser]
|
|
|
|
# Close all pages first and clean up page tracking
|
|
pages = browser.contexts[0].pages if browser.contexts else []
|
|
for page in pages:
|
|
try:
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await page.close()
|
|
except Exception as e:
|
|
print(f"Error closing page: {e}")
|
|
|
|
# Close browser
|
|
await browser.close()
|
|
print(f"Browser cleaned up. Total active browsers: {len(active_browsers)}")
|
|
except Exception as e:
|
|
print(f"Error during browser cleanup: {e}")
|
|
|
|
async def check_browser_health():
|
|
"""Check browser health and recycle if needed"""
|
|
while True:
|
|
try:
|
|
# Sleep for 2 minutes between checks (reduced from 5 minutes)
|
|
await asyncio.sleep(120)
|
|
|
|
# First, force cleanup old instances
|
|
await force_cleanup_old_instances()
|
|
|
|
async with browser_lock:
|
|
# Get all browsers from the pool
|
|
browsers = []
|
|
while not browser_pool.empty():
|
|
try:
|
|
browsers.append(await browser_pool.get_nowait())
|
|
except asyncio.QueueEmpty:
|
|
break
|
|
|
|
# Check each browser
|
|
for browser in browsers:
|
|
try:
|
|
# Check if browser is too old
|
|
if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
|
|
print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
|
|
await cleanup_browser(browser)
|
|
browser = await create_browser()
|
|
else:
|
|
# Quick health check
|
|
contexts = browser.contexts
|
|
if contexts:
|
|
pages = contexts[0].pages
|
|
|
|
# Put back in pool if healthy
|
|
if not browser_pool.full():
|
|
await browser_pool.put(browser)
|
|
else:
|
|
# Pool is full, cleanup this browser
|
|
await cleanup_browser(browser)
|
|
except Exception as e:
|
|
print(f"Error checking browser health: {e}")
|
|
# Cleanup the problematic browser
|
|
try:
|
|
await cleanup_browser(browser)
|
|
except:
|
|
pass
|
|
|
|
# Create new browsers if pool is empty
|
|
while browser_pool.qsize() < MAX_BROWSERS:
|
|
try:
|
|
browser = await create_browser()
|
|
await browser_pool.put(browser)
|
|
except Exception as e:
|
|
print(f"Error creating browser for pool: {e}")
|
|
break
|
|
|
|
except Exception as e:
|
|
print(f"Error in browser health check: {e}")
|
|
await asyncio.sleep(60) # Wait before retrying
|
|
|
|
async def force_cleanup_old_instances():
|
|
"""Force cleanup old browser and page instances based on timeout"""
|
|
print(f"Checking for old browser/page instances (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...")
|
|
|
|
current_time = time.time()
|
|
timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60
|
|
cleaned_browsers = 0
|
|
cleaned_pages = 0
|
|
|
|
async with browser_lock:
|
|
# Clean up old browsers
|
|
browsers_to_cleanup = []
|
|
for browser in list(active_browsers):
|
|
creation_time = browser_creation_times.get(browser, 0)
|
|
if current_time - creation_time > timeout_seconds:
|
|
browsers_to_cleanup.append(browser)
|
|
print(f"Marking browser for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)")
|
|
|
|
for browser in browsers_to_cleanup:
|
|
try:
|
|
await cleanup_browser(browser)
|
|
cleaned_browsers += 1
|
|
except Exception as e:
|
|
print(f"Error cleaning up old browser: {e}")
|
|
|
|
# Clean up old pages (this is a fallback for pages that might not be properly tracked)
|
|
for browser in list(active_browsers):
|
|
try:
|
|
if browser.contexts:
|
|
for context in browser.contexts:
|
|
for page in context.pages:
|
|
if page in page_creation_times:
|
|
creation_time = page_creation_times[page]
|
|
if current_time - creation_time > timeout_seconds:
|
|
try:
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await page.close()
|
|
cleaned_pages += 1
|
|
print(f"Force closed old page (age: {(current_time - creation_time)/60:.1f} minutes)")
|
|
except Exception as e:
|
|
print(f"Error closing old page: {e}")
|
|
except Exception as e:
|
|
print(f"Error checking pages in browser: {e}")
|
|
|
|
# Also cleanup pages from browser_utils module
|
|
await force_cleanup_old_pages()
|
|
|
|
print(f"Force cleanup completed: {cleaned_browsers} browsers, {cleaned_pages} pages cleaned")
|
|
|
|
async def force_cleanup_all_browsers():
|
|
"""Force cleanup all browsers - useful for emergency situations"""
|
|
print("Force cleaning up all browsers...")
|
|
|
|
async with browser_lock:
|
|
# Clean up browsers in pool
|
|
while not browser_pool.empty():
|
|
try:
|
|
browser = await browser_pool.get_nowait()
|
|
await cleanup_browser(browser)
|
|
except asyncio.QueueEmpty:
|
|
break
|
|
|
|
# Clean up any remaining active browsers
|
|
for browser in list(active_browsers):
|
|
try:
|
|
await cleanup_browser(browser)
|
|
except Exception as e:
|
|
print(f"Error force cleaning browser: {e}")
|
|
|
|
print("Force cleanup completed")
|
|
|
|
@app.on_event("startup")
|
|
async def init_browser_pool():
|
|
"""Initialize the browser pool on startup"""
|
|
print("Initializing browser pool...")
|
|
|
|
# Start browser health check task
|
|
asyncio.create_task(check_browser_health())
|
|
|
|
# Pre-populate pool with initial browsers
|
|
for _ in range(min(2, MAX_BROWSERS)):
|
|
try:
|
|
browser = await create_browser()
|
|
await browser_pool.put(browser)
|
|
except Exception as e:
|
|
print(f"Error creating initial browser: {e}")
|
|
|
|
print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
|
|
|
|
@app.on_event("shutdown")
|
|
async def cleanup_browser_pool():
|
|
"""Cleanup browser pool on shutdown"""
|
|
print("Cleaning up browser pool...")
|
|
await force_cleanup_all_browsers()
|
|
|
|
# Stop playwright instance
|
|
global playwright_instance
|
|
if playwright_instance:
|
|
await playwright_instance.stop()
|
|
playwright_instance = None
|
|
|
|
print("Browser pool cleanup completed")
|
|
|
|
def signal_handler(signum, frame):
|
|
"""Handle shutdown signals"""
|
|
print(f"Received signal {signum}, shutting down gracefully...")
|
|
asyncio.create_task(cleanup_browser_pool())
|
|
exit(0)
|
|
|
|
# Register signal handlers
|
|
signal.signal(signal.SIGINT, signal_handler)
|
|
signal.signal(signal.SIGTERM, signal_handler)
|
|
|
|
def init_db():
|
|
"""Initialize the SQLite database"""
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
|
|
# Create cache table
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS cache (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
url TEXT NOT NULL,
|
|
route TEXT NOT NULL,
|
|
data TEXT NOT NULL,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
UNIQUE(url, route)
|
|
)
|
|
''')
|
|
|
|
# Create index for faster lookups
|
|
cursor.execute('''
|
|
CREATE INDEX IF NOT EXISTS idx_cache_url_route
|
|
ON cache(url, route)
|
|
''')
|
|
|
|
# Create index for cleanup operations
|
|
cursor.execute('''
|
|
CREATE INDEX IF NOT EXISTS idx_cache_created_at
|
|
ON cache(created_at)
|
|
''')
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
print("Database initialized")
|
|
|
|
def get_cached_data(url, route):
|
|
"""Get cached data for a URL and route"""
|
|
try:
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute('''
|
|
SELECT data, created_at FROM cache
|
|
WHERE url = ? AND route = ?
|
|
''', (url, route))
|
|
|
|
result = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if result:
|
|
data, created_at = result
|
|
created_time = datetime.fromisoformat(created_at)
|
|
|
|
# Check if cache is still valid
|
|
if datetime.now() - created_time < timedelta(hours=CACHE_EXPIRY_HOURS):
|
|
return json.loads(data)
|
|
|
|
return None
|
|
except Exception as e:
|
|
print(f"Error getting cached data: {e}")
|
|
return None
|
|
|
|
def save_to_cache(url, route, data):
|
|
"""Save data to cache"""
|
|
try:
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute('''
|
|
INSERT OR REPLACE INTO cache (url, route, data, created_at)
|
|
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
|
|
''', (url, route, json.dumps(data)))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
except Exception as e:
|
|
print(f"Error saving to cache: {e}")
|
|
|
|
def cleanup_old_cache_entries():
|
|
"""Clean up old cache entries"""
|
|
try:
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
|
|
# Delete entries older than CACHE_EXPIRY_HOURS
|
|
cutoff_time = datetime.now() - timedelta(hours=CACHE_EXPIRY_HOURS)
|
|
|
|
cursor.execute('''
|
|
DELETE FROM cache
|
|
WHERE created_at < ?
|
|
''', (cutoff_time.isoformat(),))
|
|
|
|
deleted_count = cursor.rowcount
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
print(f"Cleaned up {deleted_count} old cache entries")
|
|
except Exception as e:
|
|
print(f"Error cleaning up cache: {e}")
|
|
|
|
# Initialize database
|
|
init_db()
|
|
|
|
# Initialize scheduler for periodic cache cleanup
|
|
scheduler = BackgroundScheduler()
|
|
scheduler.add_job(
|
|
cleanup_old_cache_entries,
|
|
CronTrigger.from_crontab(CLEANUP_CRON),
|
|
id='cache_cleanup_job',
|
|
replace_existing=True
|
|
)
|
|
|
|
@app.on_event("startup")
|
|
def start_scheduler():
|
|
scheduler.start()
|
|
print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
|
|
|
|
@app.on_event("shutdown")
|
|
def shutdown_scheduler():
|
|
scheduler.shutdown(wait=False)
|
|
print("Cache cleanup scheduler stopped")
|
|
|
|
async def wait_for_network_idle(page):
|
|
"""Wait for network to be idle"""
|
|
await page.wait_for_load_state('networkidle')
|
|
|
|
@app.head("/")
|
|
async def health_check():
|
|
return {"status": "healthy"}
|
|
|
|
async def safe_browser_operation(url, operation_func):
|
|
"""Safely perform a browser operation with proper resource management"""
|
|
async with operation_semaphore:
|
|
browser = None
|
|
context = None
|
|
page = None
|
|
|
|
try:
|
|
# Get browser from pool or create new one
|
|
try:
|
|
browser = await asyncio.wait_for(browser_pool.get(), timeout=10.0)
|
|
except asyncio.TimeoutError:
|
|
print("Timeout getting browser from pool, creating new one")
|
|
browser = await create_browser()
|
|
|
|
# Create context and page
|
|
context = await browser.new_context(
|
|
user_agent=CUSTOM_USER_AGENT,
|
|
viewport={'width': 1920, 'height': 1080},
|
|
ignore_https_errors=True,
|
|
)
|
|
|
|
page = await context.new_page()
|
|
|
|
# Track page creation time for force cleanup
|
|
page_creation_times[page] = time.time()
|
|
active_pages.add(page)
|
|
|
|
# Set up request interception for better performance
|
|
await page.route("**/*", lambda route: route.abort()
|
|
if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
|
|
else route.continue_())
|
|
|
|
# Perform the operation
|
|
result = await operation_func(page)
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error in browser operation: {e}")
|
|
raise
|
|
finally:
|
|
# Cleanup
|
|
if page:
|
|
try:
|
|
# Remove from tracking
|
|
if page in active_pages:
|
|
active_pages.remove(page)
|
|
if page in page_creation_times:
|
|
del page_creation_times[page]
|
|
await page.close()
|
|
except:
|
|
pass
|
|
if context:
|
|
try:
|
|
await context.close()
|
|
except:
|
|
pass
|
|
if browser:
|
|
try:
|
|
# Return browser to pool if it's still healthy
|
|
if not browser_pool.full():
|
|
await browser_pool.put(browser)
|
|
else:
|
|
await cleanup_browser(browser)
|
|
except:
|
|
pass
|
|
|
|
@app.get("/")
|
|
async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "visit")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
async def visit_operation(page):
|
|
try:
|
|
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
|
if not response:
|
|
print(f"Warning: No response object returned for {decoded_url}")
|
|
|
|
# Get page content
|
|
content = await page.content()
|
|
return {"status": "success", "content": content}
|
|
except Exception as e:
|
|
print(f"Error during page navigation: {e}")
|
|
# Try to get content anyway
|
|
try:
|
|
content = await page.content()
|
|
return {"status": "partial", "content": content, "error": str(e)}
|
|
except:
|
|
raise Exception(f"Failed to get page content: {str(e)}")
|
|
|
|
result = await safe_browser_operation(decoded_url, visit_operation)
|
|
save_to_cache(decoded_url, "visit", result)
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error visiting URL {decoded_url}: {e}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/seo")
|
|
async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
|
|
"""Extract SEO information from a website"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "seo")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
async def seo_operation(page):
|
|
try:
|
|
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
|
|
|
# Extract SEO information
|
|
seo_data = await page.evaluate('''() => {
|
|
const data = {
|
|
title: document.title || '',
|
|
description: '',
|
|
canonical: '',
|
|
h1: [],
|
|
h2: [],
|
|
images: 0,
|
|
links: 0
|
|
};
|
|
|
|
// Get meta description
|
|
const metaDescription = document.querySelector('meta[name="description"]');
|
|
if (metaDescription) {
|
|
data.description = metaDescription.getAttribute('content') || '';
|
|
}
|
|
|
|
// Get canonical link
|
|
const canonicalLink = document.querySelector('link[rel="canonical"]');
|
|
if (canonicalLink) {
|
|
data.canonical = canonicalLink.getAttribute('href') || '';
|
|
}
|
|
|
|
// Get h1 tags
|
|
document.querySelectorAll('h1').forEach(h1 => {
|
|
const text = h1.innerText.trim();
|
|
if (text) data.h1.push(text);
|
|
});
|
|
|
|
// Get h2 tags
|
|
document.querySelectorAll('h2').forEach(h2 => {
|
|
const text = h2.innerText.trim();
|
|
if (text) data.h2.push(text);
|
|
});
|
|
|
|
// Count images
|
|
data.images = document.querySelectorAll('img').length;
|
|
|
|
// Count links
|
|
data.links = document.querySelectorAll('a').length;
|
|
|
|
return data;
|
|
}''')
|
|
|
|
result = {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"seo": seo_data
|
|
}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during SEO extraction: {e}")
|
|
return {"status": "error", "url": decoded_url, "error": str(e)}
|
|
|
|
result = await safe_browser_operation(decoded_url, seo_operation)
|
|
save_to_cache(decoded_url, "seo", result)
|
|
return result
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/meta")
|
|
async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
|
|
"""Extract meta tags from a website"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "meta")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
async def meta_operation(page):
|
|
try:
|
|
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
|
|
|
# Extract meta tags using Playwright
|
|
meta_data = await page.evaluate('''() => {
|
|
const data = {
|
|
meta_tags: [],
|
|
open_graph: {},
|
|
twitter_card: {},
|
|
title: document.title || ''
|
|
};
|
|
|
|
// Extract all meta tags
|
|
document.querySelectorAll('meta').forEach(meta => {
|
|
const attributes = {};
|
|
for (let attr of meta.attributes) {
|
|
attributes[attr.name] = attr.value;
|
|
}
|
|
data.meta_tags.push(attributes);
|
|
});
|
|
|
|
// Extract Open Graph tags
|
|
document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
|
|
data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content');
|
|
});
|
|
|
|
// Extract Twitter card tags
|
|
document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
|
|
data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content');
|
|
});
|
|
|
|
return data;
|
|
}''')
|
|
|
|
result = {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"meta_tags": meta_data['meta_tags'],
|
|
"open_graph": meta_data['open_graph'],
|
|
"twitter_card": meta_data['twitter_card'],
|
|
"title": meta_data['title']
|
|
}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during meta tag extraction: {e}")
|
|
return {"status": "error", "url": decoded_url, "error": str(e)}
|
|
|
|
result = await safe_browser_operation(decoded_url, meta_operation)
|
|
save_to_cache(decoded_url, "meta", result)
|
|
return result
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/cache/clear")
|
|
async def clear_cache(x_api_key: Optional[str] = Header(None)):
|
|
"""Clear all cached data"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
try:
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
cursor.execute('DELETE FROM cache')
|
|
deleted_count = cursor.rowcount
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {"status": "success", "message": f"Cleared {deleted_count} cache entries"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/cache/stats")
|
|
async def cache_stats(x_api_key: Optional[str] = Header(None)):
|
|
"""Get cache statistics"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
try:
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
|
|
# Get total count
|
|
cursor.execute('SELECT COUNT(*) FROM cache')
|
|
total_count = cursor.fetchone()[0]
|
|
|
|
# Get count by route
|
|
cursor.execute('''
|
|
SELECT route, COUNT(*) as count
|
|
FROM cache
|
|
GROUP BY route
|
|
''')
|
|
route_counts = dict(cursor.fetchall())
|
|
|
|
# Get oldest and newest entries
|
|
cursor.execute('''
|
|
SELECT MIN(created_at), MAX(created_at)
|
|
FROM cache
|
|
''')
|
|
oldest, newest = cursor.fetchone()
|
|
|
|
# Get database size
|
|
cursor.execute('PRAGMA page_count')
|
|
page_count = cursor.fetchone()[0]
|
|
cursor.execute('PRAGMA page_size')
|
|
page_size = cursor.fetchone()[0]
|
|
db_size = page_count * page_size
|
|
|
|
conn.close()
|
|
|
|
return {
|
|
"status": "success",
|
|
"total_entries": total_count,
|
|
"route_counts": route_counts,
|
|
"oldest_entry": oldest,
|
|
"newest_entry": newest,
|
|
"database_size_bytes": db_size,
|
|
"cache_expiry_hours": CACHE_EXPIRY_HOURS
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/status")
|
|
async def system_status(x_api_key: Optional[str] = Header(None)):
|
|
"""Get system status and health information"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
try:
|
|
# Get system information
|
|
cpu_percent = psutil.cpu_percent(interval=1)
|
|
memory = psutil.virtual_memory()
|
|
disk = psutil.disk_usage('/')
|
|
|
|
# Get browser pool information
|
|
pool_size = browser_pool.qsize()
|
|
active_browser_count = len(active_browsers)
|
|
active_page_count = len(active_pages)
|
|
|
|
# Get cache statistics
|
|
conn = sqlite3.connect('/db/cache.db')
|
|
cursor = conn.cursor()
|
|
cursor.execute('SELECT COUNT(*) FROM cache')
|
|
cache_count = cursor.fetchone()[0]
|
|
conn.close()
|
|
|
|
return {
|
|
"status": "success",
|
|
"system": {
|
|
"cpu_percent": cpu_percent,
|
|
"memory_percent": memory.percent,
|
|
"memory_available_gb": round(memory.available / (1024**3), 2),
|
|
"disk_percent": disk.percent,
|
|
"disk_free_gb": round(disk.free / (1024**3), 2)
|
|
},
|
|
"browser_pool": {
|
|
"pool_size": pool_size,
|
|
"max_browsers": MAX_BROWSERS,
|
|
"active_browsers": active_browser_count,
|
|
"active_pages": active_page_count,
|
|
"browser_ttl_seconds": BROWSER_TTL,
|
|
"instance_timeout_minutes": BROWSER_INSTANCE_TIMEOUT_MINUTES
|
|
},
|
|
"cache": {
|
|
"total_entries": cache_count,
|
|
"expiry_hours": CACHE_EXPIRY_HOURS
|
|
},
|
|
"rate_limiting": {
|
|
"requests_per_minute": RATE_LIMIT_MINUTE,
|
|
"window_seconds": RATE_LIMIT_WINDOW
|
|
}
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.post("/emergency-cleanup")
|
|
async def emergency_cleanup(x_api_key: Optional[str] = Header(None)):
|
|
"""Emergency cleanup endpoint to force cleanup all browsers"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
try:
|
|
await force_cleanup_all_browsers()
|
|
return {"status": "success", "message": "Emergency cleanup completed"}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.post("/force-cleanup-old")
|
|
async def force_cleanup_old(x_api_key: Optional[str] = Header(None)):
|
|
"""Force cleanup old browser and page instances based on timeout"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
try:
|
|
await force_cleanup_old_instances()
|
|
return {
|
|
"status": "success",
|
|
"message": f"Force cleanup of old instances completed (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)"
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@asynccontextmanager
|
|
async def get_browser():
|
|
"""Context manager for getting a browser from the pool"""
|
|
browser = None
|
|
try:
|
|
browser = await browser_pool.get()
|
|
yield browser
|
|
finally:
|
|
if browser:
|
|
await browser_pool.put(browser)
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=8000)
|