This commit is contained in:
@@ -36,6 +36,8 @@ COPY . .
|
||||
# Set environment variables
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV API_KEY=""
|
||||
ENV CACHE_EXPIRY_HOURS=36
|
||||
ENV CLEANUP_CRON="0 3 * * *"
|
||||
|
||||
# Create a non-root user and switch to it
|
||||
RUN useradd -m puppeteer && chown -R puppeteer:puppeteer /app
|
||||
|
||||
+132
-178
@@ -9,6 +9,8 @@ import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, Dict, List, Any
|
||||
from urllib.parse import unquote
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@@ -17,6 +19,12 @@ API_KEY = os.getenv('API_KEY')
|
||||
if not API_KEY:
|
||||
raise ValueError("API_KEY environment variable must be set")
|
||||
|
||||
# Get cache expiry time from environment variable (default: 36 hours)
|
||||
CACHE_EXPIRY_HOURS = int(os.getenv('CACHE_EXPIRY_HOURS', '36'))
|
||||
|
||||
# Get cleanup cron schedule from environment variable (default: every day at 3 AM)
|
||||
CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
|
||||
|
||||
# Define custom user agent
|
||||
CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
|
||||
|
||||
@@ -104,11 +112,11 @@ def init_db():
|
||||
# Define the database path
|
||||
DB_PATH = '/db/cache.db'
|
||||
|
||||
# Get cached data if it exists and is not older than 36 hours
|
||||
# Get cached data if it exists and is not older than the expiry time
|
||||
def get_cached_data(url, route):
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cursor = conn.cursor()
|
||||
cache_expiry = int(time.time()) - (36 * 60 * 60) # 36 hours in seconds
|
||||
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds
|
||||
cursor.execute(
|
||||
"SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
|
||||
(url, route, cache_expiry)
|
||||
@@ -138,9 +146,54 @@ def save_to_cache(url, route, data):
|
||||
conn.close()
|
||||
print(f"Saved to cache: {url} on route {route}")
|
||||
|
||||
# Function to clean up old cache entries
|
||||
def cleanup_old_cache_entries():
|
||||
try:
|
||||
print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours)")
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Calculate the timestamp for entries older than the expiry time
|
||||
expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
|
||||
|
||||
# Get count of entries to be deleted
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp < ?", (expiry_timestamp,))
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
# Delete old entries
|
||||
cursor.execute("DELETE FROM cache WHERE timestamp < ?", (expiry_timestamp,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(f"Cache cleanup completed: {count} entries removed")
|
||||
except Exception as e:
|
||||
print(f"Error during cache cleanup: {e}")
|
||||
|
||||
# Initialize scheduler for periodic cache cleanup
|
||||
scheduler = BackgroundScheduler()
|
||||
scheduler.add_job(
|
||||
cleanup_old_cache_entries,
|
||||
CronTrigger.from_crontab(CLEANUP_CRON),
|
||||
id='cache_cleanup_job',
|
||||
replace_existing=True
|
||||
)
|
||||
|
||||
# Initialize database on startup
|
||||
init_db()
|
||||
|
||||
# Start the scheduler when the application starts
|
||||
@app.on_event("startup")
|
||||
def start_scheduler():
|
||||
scheduler.start()
|
||||
print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
|
||||
print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours")
|
||||
|
||||
# Shutdown the scheduler when the application stops
|
||||
@app.on_event("shutdown")
|
||||
def shutdown_scheduler():
|
||||
scheduler.shutdown(wait=False)
|
||||
print("Cache cleanup scheduler stopped")
|
||||
|
||||
async def wait_for_network_idle(page):
|
||||
"""Wait until no network requests are in flight"""
|
||||
await page.waitForNetworkIdle(idleTime=500, timeout=30000)
|
||||
@@ -149,6 +202,43 @@ async def wait_for_network_idle(page):
|
||||
async def health_check():
|
||||
return {"status": "ok"}
|
||||
|
||||
async def safe_browser_operation(url, operation_func):
|
||||
"""Safely perform browser operations with proper cleanup"""
|
||||
browser = None
|
||||
try:
|
||||
browser = await launch(
|
||||
headless=True,
|
||||
executablePath='/usr/bin/google-chrome',
|
||||
args=['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
handleSIGINT=False,
|
||||
handleSIGTERM=False,
|
||||
handleSIGHUP=False
|
||||
)
|
||||
|
||||
# Create new page with timeout
|
||||
page = await browser.newPage()
|
||||
page.setDefaultNavigationTimeout(30000)
|
||||
|
||||
# Set custom user agent
|
||||
await page.setUserAgent(CUSTOM_USER_AGENT)
|
||||
|
||||
# Call the operation function that uses the page
|
||||
result = await operation_func(page)
|
||||
|
||||
# Explicitly close the page
|
||||
await page.close()
|
||||
|
||||
return result
|
||||
|
||||
finally:
|
||||
# Ensure browser is closed properly
|
||||
if browser:
|
||||
try:
|
||||
await browser.close()
|
||||
except Exception as e:
|
||||
print(f"Error closing browser: {e}")
|
||||
# We don't re-raise here to avoid masking the original error
|
||||
|
||||
@app.get("/")
|
||||
async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
# Validate API key
|
||||
@@ -163,33 +253,30 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
browser = None
|
||||
try:
|
||||
print(decoded_url)
|
||||
print(f"Visiting URL: {decoded_url}")
|
||||
|
||||
# Launch browser using installed Chrome
|
||||
browser = await launch(
|
||||
headless=True,
|
||||
executablePath='/usr/bin/google-chrome',
|
||||
args=['--no-sandbox', '--disable-setuid-sandbox']
|
||||
)
|
||||
# Define the operation to perform with the browser
|
||||
async def visit_operation(page):
|
||||
try:
|
||||
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||
if not response:
|
||||
print(f"Warning: No response object returned for {decoded_url}")
|
||||
|
||||
# Create new page
|
||||
page = await browser.newPage()
|
||||
# Get page content
|
||||
content = await page.content()
|
||||
return {"status": "success", "content": content}
|
||||
except Exception as e:
|
||||
print(f"Error during page navigation: {e}")
|
||||
# Try to get content anyway
|
||||
try:
|
||||
content = await page.content()
|
||||
return {"status": "partial", "content": content, "error": str(e)}
|
||||
except:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to get page content: {str(e)}")
|
||||
|
||||
# Set custom user agent
|
||||
await page.setUserAgent(CUSTOM_USER_AGENT)
|
||||
|
||||
# Navigate to URL and wait for network idle
|
||||
await page.goto(decoded_url, waitUntil='networkidle2')
|
||||
|
||||
# Get page content
|
||||
content = await page.content()
|
||||
|
||||
# Close page
|
||||
await page.close()
|
||||
|
||||
result = {"status": "success", "content": content}
|
||||
# Perform the operation
|
||||
result = await safe_browser_operation(decoded_url, visit_operation)
|
||||
|
||||
# Save to cache
|
||||
save_to_cache(decoded_url, "visit", result)
|
||||
@@ -197,13 +284,9 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error visiting URL {decoded_url}: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
finally:
|
||||
# Ensure browser is closed even if an error occurs
|
||||
if browser:
|
||||
await browser.close()
|
||||
|
||||
@app.get("/seo")
|
||||
async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
"""Extract SEO information from a website"""
|
||||
@@ -219,112 +302,17 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
browser = None
|
||||
try:
|
||||
print(f"Extracting SEO from: {decoded_url}")
|
||||
|
||||
# Launch browser using installed Chrome
|
||||
browser = await launch(
|
||||
headless=True,
|
||||
executablePath='/usr/bin/google-chrome',
|
||||
args=['--no-sandbox', '--disable-setuid-sandbox']
|
||||
)
|
||||
|
||||
# Create new page
|
||||
page = await browser.newPage()
|
||||
|
||||
# Set custom user agent
|
||||
await page.setUserAgent(CUSTOM_USER_AGENT)
|
||||
|
||||
# Navigate to URL and wait for network idle
|
||||
await page.goto(decoded_url, waitUntil='networkidle2')
|
||||
|
||||
# Extract JSON-LD data
|
||||
json_ld_data = await page.evaluate('''() => {
|
||||
const jsonLdScripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||||
return jsonLdScripts.map(script => {
|
||||
try {
|
||||
return JSON.parse(script.textContent);
|
||||
} catch (e) {
|
||||
return { error: "Failed to parse JSON-LD", content: script.textContent };
|
||||
}
|
||||
});
|
||||
}''')
|
||||
|
||||
# Extract meta tags
|
||||
meta_tags = await page.evaluate('''() => {
|
||||
const metaTags = Array.from(document.querySelectorAll('meta'));
|
||||
return metaTags.map(tag => {
|
||||
const attributes = Array.from(tag.attributes);
|
||||
const result = {};
|
||||
attributes.forEach(attr => {
|
||||
result[attr.name] = attr.value;
|
||||
});
|
||||
return result;
|
||||
});
|
||||
}''')
|
||||
|
||||
# Extract title
|
||||
title = await page.evaluate('document.title')
|
||||
|
||||
# Extract canonical URL
|
||||
canonical_url = await page.evaluate('''() => {
|
||||
const link = document.querySelector('link[rel="canonical"]');
|
||||
return link ? link.href : null;
|
||||
}''')
|
||||
|
||||
# Extract Open Graph data
|
||||
og_data = await page.evaluate('''() => {
|
||||
const ogTags = Array.from(document.querySelectorAll('meta[property^="og:"]'));
|
||||
const result = {};
|
||||
ogTags.forEach(tag => {
|
||||
const property = tag.getAttribute('property');
|
||||
const content = tag.getAttribute('content');
|
||||
result[property] = content;
|
||||
});
|
||||
return result;
|
||||
}''')
|
||||
|
||||
# Extract Twitter Card data
|
||||
twitter_data = await page.evaluate('''() => {
|
||||
const twitterTags = Array.from(document.querySelectorAll('meta[name^="twitter:"]'));
|
||||
const result = {};
|
||||
twitterTags.forEach(tag => {
|
||||
const name = tag.getAttribute('name');
|
||||
const content = tag.getAttribute('content');
|
||||
result[name] = content;
|
||||
});
|
||||
return result;
|
||||
}''')
|
||||
|
||||
# Close page
|
||||
await page.close()
|
||||
|
||||
# Compile all SEO data
|
||||
seo_data = {
|
||||
"title": title,
|
||||
"canonical_url": canonical_url,
|
||||
"json_ld": json_ld_data,
|
||||
"meta_tags": meta_tags,
|
||||
"open_graph": og_data,
|
||||
"twitter_card": twitter_data
|
||||
}
|
||||
|
||||
result = {"status": "success", "seo_data": seo_data}
|
||||
|
||||
# Save to cache
|
||||
save_to_cache(decoded_url, "seo", result)
|
||||
# Use the safe_browser_operation function to perform the browser operation
|
||||
result = await safe_browser_operation(decoded_url, asyncio.run)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
finally:
|
||||
# Ensure browser is closed even if an error occurs
|
||||
if browser:
|
||||
await browser.close()
|
||||
|
||||
@app.get("/meta")
|
||||
async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
"""Extract meta tags from a website"""
|
||||
@@ -340,65 +328,17 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
browser = None
|
||||
try:
|
||||
print(f"Extracting meta tags from: {decoded_url}")
|
||||
|
||||
# Launch browser using installed Chrome
|
||||
browser = await launch(
|
||||
headless=True,
|
||||
executablePath='/usr/bin/google-chrome',
|
||||
args=['--no-sandbox', '--disable-setuid-sandbox']
|
||||
)
|
||||
|
||||
# Create new page
|
||||
page = await browser.newPage()
|
||||
|
||||
# Set custom user agent
|
||||
await page.setUserAgent(CUSTOM_USER_AGENT)
|
||||
|
||||
# Navigate to URL and wait for network idle
|
||||
await page.goto(decoded_url, waitUntil='networkidle2')
|
||||
|
||||
# Extract meta tags
|
||||
meta_tags = await page.evaluate('''() => {
|
||||
const metaTags = Array.from(document.querySelectorAll('meta'));
|
||||
return metaTags.map(tag => {
|
||||
const attributes = Array.from(tag.attributes);
|
||||
const result = {};
|
||||
attributes.forEach(attr => {
|
||||
result[attr.name] = attr.value;
|
||||
});
|
||||
return result;
|
||||
});
|
||||
}''')
|
||||
|
||||
# Extract title
|
||||
title = await page.evaluate('document.title')
|
||||
|
||||
# Close page
|
||||
await page.close()
|
||||
|
||||
result = {
|
||||
"status": "success",
|
||||
"url": decoded_url,
|
||||
"title": title,
|
||||
"meta_tags": meta_tags
|
||||
}
|
||||
|
||||
# Save to cache
|
||||
save_to_cache(decoded_url, "meta", result)
|
||||
# Use the safe_browser_operation function to perform the browser operation
|
||||
result = await safe_browser_operation(decoded_url, asyncio.run)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
finally:
|
||||
# Ensure browser is closed even if an error occurs
|
||||
if browser:
|
||||
await browser.close()
|
||||
|
||||
@app.get("/cache/clear")
|
||||
async def clear_cache(x_api_key: Optional[str] = Header(None)):
|
||||
"""Clear the entire cache database"""
|
||||
@@ -437,6 +377,16 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,))
|
||||
recent_entries = cursor.fetchone()[0]
|
||||
|
||||
# Get oldest entry timestamp
|
||||
cursor.execute("SELECT MIN(timestamp) FROM cache")
|
||||
oldest_timestamp = cursor.fetchone()[0]
|
||||
oldest_date = datetime.fromtimestamp(oldest_timestamp).isoformat() if oldest_timestamp else None
|
||||
|
||||
# Get newest entry timestamp
|
||||
cursor.execute("SELECT MAX(timestamp) FROM cache")
|
||||
newest_timestamp = cursor.fetchone()[0]
|
||||
newest_date = datetime.fromtimestamp(newest_timestamp).isoformat() if newest_timestamp else None
|
||||
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
@@ -444,7 +394,11 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
|
||||
"stats": {
|
||||
"total_entries": total_entries,
|
||||
"entries_by_route": routes,
|
||||
"recent_entries": recent_entries
|
||||
"recent_entries": recent_entries,
|
||||
"oldest_entry": oldest_date,
|
||||
"newest_entry": newest_date,
|
||||
"cache_expiry_hours": CACHE_EXPIRY_HOURS,
|
||||
"cleanup_schedule": CLEANUP_CRON
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
fastapi==0.68.1
|
||||
uvicorn==0.15.0
|
||||
pyppeteer==1.0.2
|
||||
psutil==6.0.0
|
||||
psutil==6.0.0
|
||||
apscheduler
|
||||
Reference in New Issue
Block a user