1077 lines
42 KiB
Python
1077 lines
42 KiB
Python
from fastapi import FastAPI, HTTPException, Header, Request
|
|
from pyppeteer import launch
|
|
import os
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from typing import Optional, Dict, List, Any
|
|
from urllib.parse import unquote
|
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
from fastapi.middleware.base import BaseHTTPMiddleware
|
|
|
|
# Add imports for browser pool
|
|
from asyncio import Queue, Lock
|
|
from contextlib import asynccontextmanager
|
|
|
|
app = FastAPI()
|
|
|
|
# Get API key from environment variable
|
|
API_KEY = os.getenv('API_KEY')
|
|
if not API_KEY:
|
|
raise ValueError("API_KEY environment variable must be set")
|
|
|
|
# Get cache expiry time from environment variable (default: 36 hours)
|
|
CACHE_EXPIRY_HOURS = int(os.getenv('CACHE_EXPIRY_HOURS', '36'))
|
|
|
|
# Get cleanup cron schedule from environment variable (default: every day at 3 AM)
|
|
CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
|
|
|
|
# Define custom user agent
|
|
CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
|
|
|
|
# Browser pool configuration
|
|
MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '5')) # Maximum number of browser instances
|
|
browser_pool = Queue()
|
|
browser_lock = Lock()
|
|
|
|
# Rate limiting configuration
|
|
RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60')) # requests per minute
|
|
RATE_LIMIT_WINDOW = 60 # window size in seconds
|
|
|
|
class RateLimitMiddleware(BaseHTTPMiddleware):
|
|
def __init__(self, app):
|
|
super().__init__(app)
|
|
self.requests = {}
|
|
self.lock = asyncio.Lock()
|
|
|
|
async def dispatch(self, request: Request, call_next):
|
|
# Skip rate limiting for health check
|
|
if request.url.path == "/" and request.method == "HEAD":
|
|
return await call_next(request)
|
|
|
|
api_key = request.headers.get("x-api-key")
|
|
if not api_key:
|
|
raise HTTPException(status_code=401, detail="API key required")
|
|
|
|
async with self.lock:
|
|
now = time.time()
|
|
# Clean old requests
|
|
self.requests = {k: v for k, v in self.requests.items()
|
|
if now - v[-1] < RATE_LIMIT_WINDOW}
|
|
|
|
# Get request times for this API key
|
|
requests = self.requests.get(api_key, [])
|
|
# Remove old requests outside the window
|
|
requests = [t for t in requests if now - t < RATE_LIMIT_WINDOW]
|
|
|
|
if len(requests) >= RATE_LIMIT_MINUTE:
|
|
oldest = requests[0]
|
|
wait_time = RATE_LIMIT_WINDOW - (now - oldest)
|
|
raise HTTPException(
|
|
status_code=429,
|
|
detail=f"Rate limit exceeded. Try again in {int(wait_time)} seconds"
|
|
)
|
|
|
|
requests.append(now)
|
|
self.requests[api_key] = requests
|
|
|
|
return await call_next(request)
|
|
|
|
# Add rate limiting middleware
|
|
app.add_middleware(RateLimitMiddleware)
|
|
|
|
async def create_browser():
|
|
"""Create a new browser instance"""
|
|
return await launch(
|
|
headless=True,
|
|
executablePath='/usr/bin/google-chrome',
|
|
args=['--no-sandbox', '--disable-setuid-sandbox'],
|
|
handleSIGINT=False,
|
|
handleSIGTERM=False,
|
|
handleSIGHUP=False
|
|
)
|
|
|
|
@asynccontextmanager
|
|
async def get_browser():
|
|
"""Get a browser from the pool or create a new one if needed"""
|
|
browser = None
|
|
try:
|
|
# Try to get a browser from the pool
|
|
try:
|
|
browser = await browser_pool.get_nowait()
|
|
except asyncio.QueueEmpty:
|
|
# If pool is empty, create a new browser if under the limit
|
|
async with browser_lock:
|
|
if browser_pool.qsize() + 1 <= MAX_BROWSERS:
|
|
browser = await create_browser()
|
|
else:
|
|
# If at limit, wait for a browser to become available
|
|
browser = await browser_pool.get()
|
|
|
|
yield browser
|
|
finally:
|
|
# Return browser to pool if it's still viable
|
|
if browser:
|
|
try:
|
|
# Quick check if browser is still usable
|
|
await browser.pages()
|
|
await browser_pool.put(browser)
|
|
except Exception:
|
|
# If browser is not usable, close it and create a new one
|
|
try:
|
|
await browser.close()
|
|
except:
|
|
pass
|
|
browser = await create_browser()
|
|
await browser_pool.put(browser)
|
|
|
|
# Initialize browser pool
|
|
@app.on_event("startup")
|
|
async def init_browser_pool():
|
|
"""Initialize the browser pool with some browsers"""
|
|
for _ in range(min(3, MAX_BROWSERS)): # Start with 3 browsers or MAX_BROWSERS, whichever is smaller
|
|
browser = await create_browser()
|
|
await browser_pool.put(browser)
|
|
print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
|
|
|
|
@app.on_event("shutdown")
|
|
async def cleanup_browser_pool():
|
|
"""Clean up all browsers in the pool"""
|
|
while not browser_pool.empty():
|
|
try:
|
|
browser = await browser_pool.get_nowait()
|
|
await browser.close()
|
|
except:
|
|
pass
|
|
|
|
# Initialize SQLite database
|
|
def init_db():
|
|
global DB_PATH
|
|
|
|
# Try to use the mounted volume first
|
|
db_path = '/db/cache.db'
|
|
db_dir = os.path.dirname(db_path)
|
|
|
|
# Check if directory exists and is writable
|
|
dir_writable = False
|
|
if os.path.exists(db_dir):
|
|
try:
|
|
test_file = os.path.join(db_dir, '.write_test')
|
|
with open(test_file, 'w') as f:
|
|
f.write('test')
|
|
os.remove(test_file)
|
|
dir_writable = True
|
|
except (IOError, PermissionError):
|
|
print(f"Directory {db_dir} exists but is not writable")
|
|
dir_writable = False
|
|
|
|
# If directory doesn't exist or isn't writable, try to create it
|
|
if not os.path.exists(db_dir) or not dir_writable:
|
|
try:
|
|
os.makedirs(db_dir, exist_ok=True)
|
|
# Test if we can write to the directory
|
|
test_file = os.path.join(db_dir, '.write_test')
|
|
with open(test_file, 'w') as f:
|
|
f.write('test')
|
|
os.remove(test_file)
|
|
print(f"Created directory: {db_dir}")
|
|
dir_writable = True
|
|
except Exception as e:
|
|
print(f"Warning: Could not create or write to directory {db_dir}: {e}")
|
|
# Fallback to using a local database file
|
|
db_path = 'cache.db'
|
|
print(f"Using local database file: {db_path}")
|
|
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS cache (
|
|
url TEXT,
|
|
route TEXT,
|
|
data TEXT,
|
|
timestamp INTEGER,
|
|
PRIMARY KEY (url, route)
|
|
)
|
|
''')
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"Database initialized at {db_path}")
|
|
# Update the global DB_PATH
|
|
DB_PATH = db_path
|
|
except sqlite3.OperationalError as e:
|
|
print(f"Error initializing database at {db_path}: {e}")
|
|
# Fallback to using a local database file if the mounted volume has permission issues
|
|
db_path = 'cache.db'
|
|
print(f"Falling back to local database file: {db_path}")
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS cache (
|
|
url TEXT,
|
|
route TEXT,
|
|
data TEXT,
|
|
timestamp INTEGER,
|
|
PRIMARY KEY (url, route)
|
|
)
|
|
''')
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"Local database initialized at {db_path}")
|
|
# Update the global DB_PATH
|
|
DB_PATH = db_path
|
|
except sqlite3.OperationalError as e2:
|
|
print(f"Error initializing local database: {e2}")
|
|
raise
|
|
|
|
# Define the database path
|
|
DB_PATH = '/db/cache.db'
|
|
|
|
# Get cached data if it exists and is not older than the expiry time
|
|
def get_cached_data(url, route):
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds
|
|
cursor.execute(
|
|
"SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
|
|
(url, route, cache_expiry)
|
|
)
|
|
result = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if result:
|
|
print(f"Cache hit for {url} on route {route}")
|
|
return json.loads(result[0])
|
|
return None
|
|
|
|
# Save data to cache
|
|
def save_to_cache(url, route, data):
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
timestamp = int(time.time())
|
|
|
|
# Convert data to JSON string
|
|
data_json = json.dumps(data)
|
|
|
|
cursor.execute(
|
|
"INSERT OR REPLACE INTO cache (url, route, data, timestamp) VALUES (?, ?, ?, ?)",
|
|
(url, route, data_json, timestamp)
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"Saved to cache: {url} on route {route}")
|
|
|
|
# Function to clean up old cache entries
|
|
def cleanup_old_cache_entries():
|
|
try:
|
|
print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
|
|
# Calculate the timestamp for entries older than the expiry time
|
|
expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
|
|
pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
|
|
|
|
# Get count of entries to be deleted (non-pagination)
|
|
cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
|
|
count_non_pagination = cursor.fetchone()[0]
|
|
|
|
# Get count of pagination entries to be deleted
|
|
cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
|
|
count_pagination = cursor.fetchone()[0]
|
|
|
|
# Delete old non-pagination entries
|
|
cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
|
|
|
|
# Delete old pagination entries
|
|
cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
|
|
except Exception as e:
|
|
print(f"Error during cache cleanup: {e}")
|
|
|
|
# Initialize scheduler for periodic cache cleanup
|
|
scheduler = BackgroundScheduler()
|
|
scheduler.add_job(
|
|
cleanup_old_cache_entries,
|
|
CronTrigger.from_crontab(CLEANUP_CRON),
|
|
id='cache_cleanup_job',
|
|
replace_existing=True
|
|
)
|
|
|
|
# Initialize database on startup
|
|
init_db()
|
|
|
|
# Start the scheduler when the application starts
|
|
@app.on_event("startup")
|
|
def start_scheduler():
|
|
scheduler.start()
|
|
print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
|
|
print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours")
|
|
|
|
# Shutdown the scheduler when the application stops
|
|
@app.on_event("shutdown")
|
|
def shutdown_scheduler():
|
|
scheduler.shutdown(wait=False)
|
|
print("Cache cleanup scheduler stopped")
|
|
|
|
async def wait_for_network_idle(page):
|
|
"""Wait until no network requests are in flight"""
|
|
await page.waitForNetworkIdle(idleTime=500, timeout=30000)
|
|
|
|
@app.head("/")
|
|
async def health_check():
|
|
return {"status": "ok"}
|
|
|
|
async def safe_browser_operation(url, operation_func):
|
|
"""Safely perform browser operations with proper cleanup"""
|
|
async with get_browser() as browser:
|
|
try:
|
|
# Create new page with timeout
|
|
page = await browser.newPage()
|
|
page.setDefaultNavigationTimeout(30000)
|
|
|
|
# Set custom user agent
|
|
await page.setUserAgent(CUSTOM_USER_AGENT)
|
|
|
|
# Call the operation function that uses the page
|
|
result = await operation_func(page)
|
|
|
|
# Explicitly close the page
|
|
await page.close()
|
|
|
|
return result
|
|
except Exception as e:
|
|
# If there's an error, close the page and re-raise
|
|
try:
|
|
await page.close()
|
|
except:
|
|
pass
|
|
raise e
|
|
|
|
@app.get("/")
|
|
async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "visit")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
print(f"Visiting URL: {decoded_url}")
|
|
|
|
# Define the operation to perform with the browser
|
|
async def visit_operation(page):
|
|
try:
|
|
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
|
if not response:
|
|
print(f"Warning: No response object returned for {decoded_url}")
|
|
|
|
# Get page content
|
|
content = await page.content()
|
|
return {"status": "success", "content": content}
|
|
except Exception as e:
|
|
print(f"Error during page navigation: {e}")
|
|
# Try to get content anyway
|
|
try:
|
|
content = await page.content()
|
|
return {"status": "partial", "content": content, "error": str(e)}
|
|
except:
|
|
raise HTTPException(status_code=500, detail=f"Failed to get page content: {str(e)}")
|
|
|
|
# Perform the operation
|
|
result = await safe_browser_operation(decoded_url, visit_operation)
|
|
|
|
# Save to cache
|
|
save_to_cache(decoded_url, "visit", result)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error visiting URL {decoded_url}: {e}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/seo")
|
|
async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
|
|
"""Extract SEO information from a website"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "seo")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
print(f"Extracting SEO from: {decoded_url}")
|
|
|
|
# Define the operation to perform with the browser
|
|
async def seo_operation(page):
|
|
try:
|
|
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
|
|
|
# Extract SEO information
|
|
seo_data = await page.evaluate('''() => {
|
|
const data = {
|
|
title: document.title || '',
|
|
description: '',
|
|
canonical: '',
|
|
h1: [],
|
|
h2: [],
|
|
images: 0,
|
|
links: 0
|
|
};
|
|
|
|
// Get meta description
|
|
const metaDescription = document.querySelector('meta[name="description"]');
|
|
if (metaDescription) {
|
|
data.description = metaDescription.getAttribute('content') || '';
|
|
}
|
|
|
|
// Get canonical link
|
|
const canonicalLink = document.querySelector('link[rel="canonical"]');
|
|
if (canonicalLink) {
|
|
data.canonical = canonicalLink.getAttribute('href') || '';
|
|
}
|
|
|
|
// Get h1 tags
|
|
document.querySelectorAll('h1').forEach(h1 => {
|
|
const text = h1.innerText.trim();
|
|
if (text) data.h1.push(text);
|
|
});
|
|
|
|
// Get h2 tags
|
|
document.querySelectorAll('h2').forEach(h2 => {
|
|
const text = h2.innerText.trim();
|
|
if (text) data.h2.push(text);
|
|
});
|
|
|
|
// Count images
|
|
data.images = document.querySelectorAll('img').length;
|
|
|
|
// Count links
|
|
data.links = document.querySelectorAll('a').length;
|
|
|
|
return data;
|
|
}''')
|
|
|
|
result = {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"seo": seo_data
|
|
}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during SEO extraction: {e}")
|
|
return {"status": "error", "url": decoded_url, "error": str(e)}
|
|
|
|
# Perform the operation
|
|
result = await safe_browser_operation(decoded_url, seo_operation)
|
|
|
|
# Save to cache
|
|
save_to_cache(decoded_url, "seo", result)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/meta")
|
|
async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
|
|
"""Extract meta tags from a website"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "meta")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
print(f"Extracting meta tags from: {decoded_url}")
|
|
|
|
# Define the operation to perform with the browser
|
|
async def meta_operation(page):
|
|
try:
|
|
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
|
|
|
# Extract all meta tags
|
|
meta_tags = await page.evaluate('''() => {
|
|
const metas = Array.from(document.querySelectorAll('meta'));
|
|
return metas.map(meta => {
|
|
const attributes = {};
|
|
Array.from(meta.attributes).forEach(attr => {
|
|
attributes[attr.name] = attr.value;
|
|
});
|
|
return attributes;
|
|
});
|
|
}''')
|
|
|
|
# Extract Open Graph tags
|
|
og_tags = await page.evaluate('''() => {
|
|
const ogTags = {};
|
|
document.querySelectorAll('meta[property^="og:"]').forEach(tag => {
|
|
const property = tag.getAttribute('property');
|
|
ogTags[property] = tag.getAttribute('content');
|
|
});
|
|
return ogTags;
|
|
}''')
|
|
|
|
# Extract Twitter card tags
|
|
twitter_tags = await page.evaluate('''() => {
|
|
const twitterTags = {};
|
|
document.querySelectorAll('meta[name^="twitter:"]').forEach(tag => {
|
|
const name = tag.getAttribute('name');
|
|
twitterTags[name] = tag.getAttribute('content');
|
|
});
|
|
return twitterTags;
|
|
}''')
|
|
|
|
result = {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"meta_tags": meta_tags,
|
|
"open_graph": og_tags,
|
|
"twitter_card": twitter_tags,
|
|
"title": await page.title()
|
|
}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during meta tag extraction: {e}")
|
|
return {"status": "error", "url": decoded_url, "error": str(e)}
|
|
|
|
# Perform the operation
|
|
result = await safe_browser_operation(decoded_url, meta_operation)
|
|
|
|
# Save to cache
|
|
save_to_cache(decoded_url, "meta", result)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@app.get("/cache/clear")
|
|
async def clear_cache(x_api_key: Optional[str] = Header(None)):
|
|
"""Clear the entire cache database"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
cursor.execute("DELETE FROM cache")
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {"status": "success", "message": "Cache cleared successfully"}
|
|
|
|
@app.get("/cache/stats")
|
|
async def cache_stats(x_api_key: Optional[str] = Header(None)):
|
|
"""Get cache statistics"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
cursor = conn.cursor()
|
|
|
|
# Get total entries
|
|
cursor.execute("SELECT COUNT(*) FROM cache")
|
|
total_entries = cursor.fetchone()[0]
|
|
|
|
# Get entries by route
|
|
cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route")
|
|
routes = {route: count for route, count in cursor.fetchall()}
|
|
|
|
# Get recent entries (last 24 hours)
|
|
recent_timestamp = int(time.time()) - (24 * 60 * 60)
|
|
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,))
|
|
recent_entries = cursor.fetchone()[0]
|
|
|
|
# Get oldest entry timestamp
|
|
cursor.execute("SELECT MIN(timestamp) FROM cache")
|
|
oldest_timestamp = cursor.fetchone()[0]
|
|
oldest_date = datetime.fromtimestamp(oldest_timestamp).isoformat() if oldest_timestamp else None
|
|
|
|
# Get newest entry timestamp
|
|
cursor.execute("SELECT MAX(timestamp) FROM cache")
|
|
newest_timestamp = cursor.fetchone()[0]
|
|
newest_date = datetime.fromtimestamp(newest_timestamp).isoformat() if newest_timestamp else None
|
|
|
|
conn.close()
|
|
|
|
return {
|
|
"status": "success",
|
|
"stats": {
|
|
"total_entries": total_entries,
|
|
"entries_by_route": routes,
|
|
"recent_entries": recent_entries,
|
|
"oldest_entry": oldest_date,
|
|
"newest_entry": newest_date,
|
|
"cache_expiry_hours": CACHE_EXPIRY_HOURS,
|
|
"cleanup_schedule": CLEANUP_CRON
|
|
}
|
|
}
|
|
|
|
@app.get("/pagination")
|
|
async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)):
|
|
"""Detect pagination on a website and determine the pagination pattern"""
|
|
# Validate API key
|
|
if not x_api_key or x_api_key != API_KEY:
|
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
|
|
# Decode URL if it's encoded
|
|
decoded_url = unquote(url)
|
|
|
|
# Check cache first
|
|
cached_result = get_cached_data(decoded_url, "pagination")
|
|
if cached_result:
|
|
return cached_result
|
|
|
|
try:
|
|
print(f"Detecting pagination on: {decoded_url}")
|
|
|
|
# Define the operation to perform with the browser
|
|
async def pagination_operation(page):
|
|
try:
|
|
# Navigate to the URL
|
|
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
|
original_url = page.url
|
|
|
|
print(f"Successfully loaded page: {original_url}")
|
|
|
|
# Analyze the page for pagination information
|
|
pagination_info = await page.evaluate('''() => {
|
|
// Find the last page number if available
|
|
const findLastPageNumber = () => {
|
|
// Get all links on the page
|
|
const links = Array.from(document.querySelectorAll('a'));
|
|
|
|
// Strategy 1: Find numeric links (page numbers)
|
|
const numericLinks = links.filter(link => {
|
|
const text = link.innerText.trim();
|
|
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
|
});
|
|
|
|
if (numericLinks.length > 0) {
|
|
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
|
return Math.max(...numericValues);
|
|
}
|
|
|
|
// Strategy 2: Look for "last page" link
|
|
const lastLinks = links.filter(link => {
|
|
const text = link.innerText.trim().toLowerCase();
|
|
const classes = (link.className || '').toLowerCase();
|
|
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
|
|
|
|
return (text === 'last' ||
|
|
classes.includes('last') ||
|
|
ariaLabel.includes('last') ||
|
|
link.getAttribute('rel') === 'last');
|
|
});
|
|
|
|
if (lastLinks.length > 0) {
|
|
const lastLink = lastLinks[0];
|
|
const href = lastLink.href;
|
|
|
|
// Common patterns: page=X, /page/X, etc.
|
|
const pagePatterns = [
|
|
/[?&]page=(\d+)/,
|
|
/[?&]p=(\d+)/,
|
|
/[?&]pg=(\d+)/,
|
|
/\/page\/(\d+)/,
|
|
/\/p\/(\d+)/,
|
|
/\/paged\/(\d+)/
|
|
];
|
|
|
|
for (const pattern of pagePatterns) {
|
|
const match = href.match(pattern);
|
|
if (match && match[1]) {
|
|
return parseInt(match[1]);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Strategy 3: Analyze all URLs for page numbers
|
|
const pageNumbersFromUrls = [];
|
|
links.forEach(link => {
|
|
if (!link.href || link.href === '#') return;
|
|
|
|
// Check for common pagination URL patterns
|
|
const patterns = [
|
|
/[?&]page=(\d+)/,
|
|
/[?&]p=(\d+)/,
|
|
/[?&]pg=(\d+)/,
|
|
/\/page\/(\d+)/,
|
|
/\/p\/(\d+)/,
|
|
/\/paged\/(\d+)/,
|
|
/\/pages\/(\d+)/
|
|
];
|
|
|
|
for (const pattern of patterns) {
|
|
const match = link.href.match(pattern);
|
|
if (match && match[1]) {
|
|
pageNumbersFromUrls.push(parseInt(match[1]));
|
|
}
|
|
}
|
|
});
|
|
|
|
if (pageNumbersFromUrls.length > 0) {
|
|
return Math.max(...pageNumbersFromUrls);
|
|
}
|
|
|
|
return null;
|
|
};
|
|
|
|
// Find a pagination link to click
|
|
const findPaginationLink = () => {
|
|
const links = Array.from(document.querySelectorAll('a'));
|
|
|
|
// Try to find a page "2" link first (most reliable)
|
|
const page2Link = links.find(link => {
|
|
const text = link.innerText.trim();
|
|
return text === '2' && link.href && link.href !== '#';
|
|
});
|
|
|
|
if (page2Link) {
|
|
return { element: page2Link, href: page2Link.href, type: 'numeric' };
|
|
}
|
|
|
|
// Try common "next page" selectors
|
|
const nextSelectors = [
|
|
'a.next',
|
|
'a.page-next',
|
|
'a[rel="next"]',
|
|
'a[aria-label="Next page"]',
|
|
'a[aria-label="next"]'
|
|
];
|
|
|
|
for (const selector of nextSelectors) {
|
|
const element = document.querySelector(selector);
|
|
if (element && element.href && element.href !== '#') {
|
|
return { element, href: element.href, type: 'next' };
|
|
}
|
|
}
|
|
|
|
// Look for any link that might be pagination
|
|
const paginationLinks = links.filter(link => {
|
|
if (!link.href || link.href === '#') return false;
|
|
|
|
const text = link.innerText.trim();
|
|
const href = link.href;
|
|
|
|
// Check for numeric text or next/prev indicators
|
|
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
|
|
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
|
|
/[»«‹›<>]/.test(text);
|
|
|
|
// Check for page parameter in URL
|
|
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
|
|
|
|
return (isNumeric || isNextPrev || hasPageParam);
|
|
});
|
|
|
|
if (paginationLinks.length > 0) {
|
|
const link = paginationLinks[0];
|
|
return { element: link, href: link.href, type: 'other' };
|
|
}
|
|
|
|
return null;
|
|
};
|
|
|
|
const lastPage = findLastPageNumber();
|
|
const paginationLink = findPaginationLink();
|
|
|
|
if (paginationLink) {
|
|
// Click the link
|
|
paginationLink.element.click();
|
|
return {
|
|
clicked: true,
|
|
href: paginationLink.href,
|
|
type: paginationLink.type,
|
|
lastPage
|
|
};
|
|
}
|
|
|
|
return { clicked: false, lastPage };
|
|
}''')
|
|
|
|
# If no pagination was found or clicked
|
|
if not pagination_info.get('clicked', False):
|
|
return {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"hasPagination": False,
|
|
"urlTemplate": None,
|
|
"lastPage": pagination_info.get('lastPage')
|
|
}
|
|
|
|
# Wait for navigation to complete after the click
|
|
try:
|
|
await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'})
|
|
except Exception as e:
|
|
print(f"Navigation timeout: {e}")
|
|
|
|
# Get the new URL after clicking
|
|
next_page_url = page.url
|
|
|
|
# If URL didn't change, pagination might be handled by AJAX
|
|
if next_page_url == original_url:
|
|
return {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"hasPagination": True,
|
|
"urlTemplate": "AJAX pagination (URL doesn't change)",
|
|
"lastPage": pagination_info.get('lastPage')
|
|
}
|
|
|
|
print(f"Navigation successful: {original_url} -> {next_page_url}")
|
|
|
|
# Analyze the URL structure to determine pagination pattern
|
|
url_template = await page.evaluate('''(originalUrl, nextPageUrl) => {
|
|
// Helper function to parse URL query parameters
|
|
const parseQueryParams = (url) => {
|
|
const params = {};
|
|
if (url.includes('?')) {
|
|
const queryString = url.split('?')[1].split('#')[0];
|
|
queryString.split('&').forEach(param => {
|
|
if (param.includes('=')) {
|
|
const [key, value] = param.split('=', 2);
|
|
params[key] = value;
|
|
}
|
|
});
|
|
}
|
|
return params;
|
|
};
|
|
|
|
// Check for query parameter based pagination
|
|
if (nextPageUrl.includes('?')) {
|
|
const originalParams = parseQueryParams(originalUrl);
|
|
const nextParams = parseQueryParams(nextPageUrl);
|
|
|
|
// Find parameters that changed or were added
|
|
let paginationParam = null;
|
|
|
|
// First check for common pagination parameter names
|
|
const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber'];
|
|
|
|
for (const key of commonPaginationParams) {
|
|
if (key in nextParams &&
|
|
(!(key in originalParams) || originalParams[key] !== nextParams[key])) {
|
|
if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) {
|
|
paginationParam = key;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If no common parameter found, check all parameters
|
|
if (!paginationParam) {
|
|
for (const [key, value] of Object.entries(nextParams)) {
|
|
// Check if parameter is new or changed
|
|
if (!(key in originalParams) || originalParams[key] !== value) {
|
|
// Check if the value is numeric and could be a page number
|
|
if (/^\d+$/.test(value) && parseInt(value) > 1) {
|
|
paginationParam = key;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we found a pagination parameter
|
|
if (paginationParam) {
|
|
const baseUrl = nextPageUrl.split('?')[0];
|
|
|
|
// Reconstruct the URL template with all parameters
|
|
const queryParts = [];
|
|
for (const [key, value] of Object.entries(nextParams)) {
|
|
if (key === paginationParam) {
|
|
queryParts.push(`${key}={PAGE_NUMBER}`);
|
|
} else {
|
|
queryParts.push(`${key}=${value}`);
|
|
}
|
|
}
|
|
|
|
return `${baseUrl}?${queryParts.join('&')}`;
|
|
}
|
|
}
|
|
|
|
// Check for path-based pagination
|
|
const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/'];
|
|
for (const pattern of pathPatterns) {
|
|
if (nextPageUrl.includes(pattern)) {
|
|
const parts = nextPageUrl.split(pattern);
|
|
let template = `${parts[0]}${pattern}{PAGE_NUMBER}`;
|
|
|
|
// Add any suffix after the page number
|
|
if (parts.length > 1 && parts[1].includes('/')) {
|
|
const suffix = parts[1].split('/', 1)[1];
|
|
if (suffix) {
|
|
template += `/${suffix}`;
|
|
}
|
|
}
|
|
|
|
return template;
|
|
}
|
|
}
|
|
|
|
// If we couldn't determine the pattern, try to make an educated guess
|
|
// For the specific case where a parameter like current_page=2 is added
|
|
const originalUrlObj = new URL(originalUrl);
|
|
const nextUrlObj = new URL(nextPageUrl);
|
|
|
|
// Check if the paths are the same but query params differ
|
|
if (originalUrlObj.pathname === nextUrlObj.pathname) {
|
|
const originalParams = parseQueryParams(originalUrl);
|
|
const nextParams = parseQueryParams(nextPageUrl);
|
|
|
|
// Find parameters that exist in next but not in original
|
|
const newParams = Object.keys(nextParams).filter(key => !(key in originalParams));
|
|
|
|
// If there's exactly one new parameter and it has a numeric value
|
|
if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) {
|
|
const paginationParam = newParams[0];
|
|
const baseUrl = nextPageUrl.split('?')[0];
|
|
|
|
// Reconstruct the URL template
|
|
const queryParts = [];
|
|
for (const [key, value] of Object.entries(nextParams)) {
|
|
if (key === paginationParam) {
|
|
queryParts.push(`${key}={PAGE_NUMBER}`);
|
|
} else {
|
|
queryParts.push(`${key}=${value}`);
|
|
}
|
|
}
|
|
|
|
return `${baseUrl}?${queryParts.join('&')}`;
|
|
}
|
|
}
|
|
|
|
// If we still couldn't determine the pattern, return both URLs as examples
|
|
return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`;
|
|
}''', original_url, next_page_url)
|
|
|
|
# Try to extract last page number from the next page if we didn't find it on the first page
|
|
if not pagination_info.get('lastPage'):
|
|
last_page_from_next = await page.evaluate('''() => {
|
|
// Get all links on the page
|
|
const links = Array.from(document.querySelectorAll('a'));
|
|
|
|
// Strategy 1: Find numeric links (page numbers)
|
|
const numericLinks = links.filter(link => {
|
|
const text = link.innerText.trim();
|
|
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
|
});
|
|
|
|
if (numericLinks.length > 0) {
|
|
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
|
return Math.max(...numericValues);
|
|
}
|
|
|
|
// Strategy 2: Analyze all URLs for page numbers
|
|
const pageNumbersFromUrls = [];
|
|
links.forEach(link => {
|
|
if (!link.href || link.href === '#') return;
|
|
|
|
// Check for common pagination URL patterns
|
|
const patterns = [
|
|
/[?&]page=(\d+)/,
|
|
/[?&]p=(\d+)/,
|
|
/[?&]pg=(\d+)/,
|
|
/\/page\/(\d+)/,
|
|
/\/p\/(\d+)/,
|
|
/\/paged\/(\d+)/,
|
|
/\/pages\/(\d+)/
|
|
];
|
|
|
|
for (const pattern of patterns) {
|
|
const match = link.href.match(pattern);
|
|
if (match && match[1]) {
|
|
pageNumbersFromUrls.push(parseInt(match[1]));
|
|
}
|
|
}
|
|
});
|
|
|
|
if (pageNumbersFromUrls.length > 0) {
|
|
return Math.max(...pageNumbersFromUrls);
|
|
}
|
|
|
|
return null;
|
|
}''')
|
|
|
|
if last_page_from_next:
|
|
pagination_info['lastPage'] = last_page_from_next
|
|
|
|
# Check if the pattern is unclear
|
|
has_pagination = True
|
|
url_string_template = str(url_template)
|
|
if url_string_template and url_string_template.startswith("Pattern unclear"):
|
|
has_pagination = False
|
|
url_template = None
|
|
|
|
result = {
|
|
"status": "success",
|
|
"url": decoded_url,
|
|
"hasPagination": has_pagination,
|
|
"urlTemplate": url_template,
|
|
"lastPage": pagination_info.get('lastPage'),
|
|
"originalUrl": original_url,
|
|
"nextPageUrl": next_page_url
|
|
}
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
print(f"Error during pagination detection: {e}")
|
|
return {
|
|
"status": "error",
|
|
"url": decoded_url,
|
|
"error": str(e),
|
|
"hasPagination": False,
|
|
"urlTemplate": None,
|
|
"lastPage": None
|
|
}
|
|
|
|
# Perform the operation
|
|
result = await safe_browser_operation(decoded_url, pagination_operation)
|
|
|
|
# Save to cache
|
|
save_to_cache(decoded_url, "pagination", result)
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=8000)
|