This commit is contained in:
@@ -17,5 +17,16 @@ BROWSER_INSTANCE_TIMEOUT_MINUTES = int(os.getenv('BROWSER_INSTANCE_TIMEOUT_MINUT
|
||||
# Define custom user agent
|
||||
CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
|
||||
|
||||
# Database path
|
||||
# Database configuration
|
||||
# Use PostgreSQL if credentials are provided, otherwise use SQLite
|
||||
POSTGRES_HOST = os.getenv('POSTGRES_HOST')
|
||||
POSTGRES_PORT = os.getenv('POSTGRES_PORT', '5432')
|
||||
POSTGRES_DB = os.getenv('POSTGRES_DB')
|
||||
POSTGRES_USER = os.getenv('POSTGRES_USER')
|
||||
POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD')
|
||||
|
||||
# Determine database type
|
||||
USE_POSTGRES = all([POSTGRES_HOST, POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD])
|
||||
|
||||
# Database path (for SQLite)
|
||||
DB_PATH = '/db/cache.db'
|
||||
@@ -3,70 +3,114 @@ import sqlite3
|
||||
import time
|
||||
import json
|
||||
from datetime import datetime
|
||||
from app.config import DB_PATH, CACHE_EXPIRY_HOURS
|
||||
from app.config import (
|
||||
DB_PATH, CACHE_EXPIRY_HOURS, USE_POSTGRES,
|
||||
POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD
|
||||
)
|
||||
|
||||
# Initialize SQLite database
|
||||
def init_db():
|
||||
global DB_PATH
|
||||
# Import PostgreSQL dependencies only if needed
|
||||
if USE_POSTGRES:
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
# Try to use the mounted volume first
|
||||
db_path = '/db/cache.db'
|
||||
db_dir = os.path.dirname(db_path)
|
||||
class DatabaseManager:
|
||||
def __init__(self):
|
||||
self.db_type = "postgresql" if USE_POSTGRES else "sqlite"
|
||||
self.connection_params = None
|
||||
|
||||
# Check if directory exists and is writable
|
||||
dir_writable = False
|
||||
if os.path.exists(db_dir):
|
||||
try:
|
||||
test_file = os.path.join(db_dir, '.write_test')
|
||||
with open(test_file, 'w') as f:
|
||||
f.write('test')
|
||||
os.remove(test_file)
|
||||
dir_writable = True
|
||||
except (IOError, PermissionError):
|
||||
print(f"Directory {db_dir} exists but is not writable")
|
||||
dir_writable = False
|
||||
if USE_POSTGRES:
|
||||
self.connection_params = {
|
||||
'host': POSTGRES_HOST,
|
||||
'port': POSTGRES_PORT,
|
||||
'database': POSTGRES_DB,
|
||||
'user': POSTGRES_USER,
|
||||
'password': POSTGRES_PASSWORD
|
||||
}
|
||||
else:
|
||||
self._init_sqlite_path()
|
||||
|
||||
# If directory doesn't exist or isn't writable, try to create it
|
||||
if not os.path.exists(db_dir) or not dir_writable:
|
||||
try:
|
||||
os.makedirs(db_dir, exist_ok=True)
|
||||
# Test if we can write to the directory
|
||||
test_file = os.path.join(db_dir, '.write_test')
|
||||
with open(test_file, 'w') as f:
|
||||
f.write('test')
|
||||
os.remove(test_file)
|
||||
print(f"Created directory: {db_dir}")
|
||||
dir_writable = True
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not create or write to directory {db_dir}: {e}")
|
||||
# Fallback to using a local database file
|
||||
db_path = 'cache.db'
|
||||
print(f"Using local database file: {db_path}")
|
||||
def _init_sqlite_path(self):
|
||||
"""Initialize SQLite database path with fallback logic"""
|
||||
global DB_PATH
|
||||
|
||||
# Try to use the mounted volume first
|
||||
db_path = '/db/cache.db'
|
||||
db_dir = os.path.dirname(db_path)
|
||||
|
||||
# Check if directory exists and is writable
|
||||
dir_writable = False
|
||||
if os.path.exists(db_dir):
|
||||
try:
|
||||
test_file = os.path.join(db_dir, '.write_test')
|
||||
with open(test_file, 'w') as f:
|
||||
f.write('test')
|
||||
os.remove(test_file)
|
||||
dir_writable = True
|
||||
except (IOError, PermissionError):
|
||||
print(f"Directory {db_dir} exists but is not writable")
|
||||
dir_writable = False
|
||||
|
||||
# If directory doesn't exist or isn't writable, try to create it
|
||||
if not os.path.exists(db_dir) or not dir_writable:
|
||||
try:
|
||||
os.makedirs(db_dir, exist_ok=True)
|
||||
# Test if we can write to the directory
|
||||
test_file = os.path.join(db_dir, '.write_test')
|
||||
with open(test_file, 'w') as f:
|
||||
f.write('test')
|
||||
os.remove(test_file)
|
||||
print(f"Created directory: {db_dir}")
|
||||
dir_writable = True
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not create or write to directory {db_dir}: {e}")
|
||||
# Fallback to using a local database file
|
||||
db_path = 'cache.db'
|
||||
print(f"Using local database file: {db_path}")
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS cache (
|
||||
url TEXT,
|
||||
route TEXT,
|
||||
data TEXT,
|
||||
timestamp INTEGER,
|
||||
PRIMARY KEY (url, route)
|
||||
)
|
||||
''')
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Database initialized at {db_path}")
|
||||
# Update the global DB_PATH
|
||||
DB_PATH = db_path
|
||||
except sqlite3.OperationalError as e:
|
||||
print(f"Error initializing database at {db_path}: {e}")
|
||||
# Fallback to using a local database file if the mounted volume has permission issues
|
||||
db_path = 'cache.db'
|
||||
print(f"Falling back to local database file: {db_path}")
|
||||
|
||||
def get_connection(self):
|
||||
"""Get database connection based on configured database type"""
|
||||
if USE_POSTGRES:
|
||||
return psycopg2.connect(**self.connection_params)
|
||||
else:
|
||||
return sqlite3.connect(DB_PATH)
|
||||
|
||||
def init_db(self):
|
||||
"""Initialize database and create tables"""
|
||||
if USE_POSTGRES:
|
||||
self._init_postgres_db()
|
||||
else:
|
||||
self._init_sqlite_db()
|
||||
|
||||
def _init_postgres_db(self):
|
||||
"""Initialize PostgreSQL database"""
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Create cache table
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS cache (
|
||||
url TEXT,
|
||||
route TEXT,
|
||||
data TEXT,
|
||||
timestamp BIGINT,
|
||||
PRIMARY KEY (url, route)
|
||||
)
|
||||
''')
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"PostgreSQL database initialized at {POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}")
|
||||
except Exception as e:
|
||||
print(f"Error initializing PostgreSQL database: {e}")
|
||||
raise
|
||||
|
||||
def _init_sqlite_db(self):
|
||||
"""Initialize SQLite database"""
|
||||
try:
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS cache (
|
||||
@@ -79,82 +123,128 @@ def init_db():
|
||||
''')
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Local database initialized at {db_path}")
|
||||
# Update the global DB_PATH
|
||||
DB_PATH = db_path
|
||||
except sqlite3.OperationalError as e2:
|
||||
print(f"Error initializing local database: {e2}")
|
||||
raise
|
||||
print(f"SQLite database initialized at {DB_PATH}")
|
||||
except sqlite3.OperationalError as e:
|
||||
print(f"Error initializing database at {DB_PATH}: {e}")
|
||||
# Fallback to using a local database file if the mounted volume has permission issues
|
||||
global DB_PATH
|
||||
DB_PATH = 'cache.db'
|
||||
print(f"Falling back to local database file: {DB_PATH}")
|
||||
try:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS cache (
|
||||
url TEXT,
|
||||
route TEXT,
|
||||
data TEXT,
|
||||
timestamp INTEGER,
|
||||
PRIMARY KEY (url, route)
|
||||
)
|
||||
''')
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Local database initialized at {DB_PATH}")
|
||||
except sqlite3.OperationalError as e2:
|
||||
print(f"Error initializing local database: {e2}")
|
||||
raise
|
||||
|
||||
# Get cached data if it exists and is not older than the expiry time
|
||||
def get_cached_data(url, route):
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Calculate cache expiry time
|
||||
if route == 'pagination':
|
||||
# Pagination cache expires after 31 times the normal cache expiry
|
||||
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
|
||||
else:
|
||||
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds
|
||||
|
||||
cursor.execute(
|
||||
"SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
|
||||
(url, route, cache_expiry)
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if result:
|
||||
print(f"Cache hit for {url} on route {route}")
|
||||
return json.loads(result[0])
|
||||
return None
|
||||
|
||||
# Save data to cache
|
||||
def save_to_cache(url, route, data):
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
cursor = conn.cursor()
|
||||
timestamp = int(time.time())
|
||||
|
||||
# Convert data to JSON string
|
||||
data_json = json.dumps(data)
|
||||
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO cache (url, route, data, timestamp) VALUES (?, ?, ?, ?)",
|
||||
(url, route, data_json, timestamp)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Saved to cache: {url} on route {route}")
|
||||
|
||||
# Function to clean up old cache entries
|
||||
def cleanup_old_cache_entries():
|
||||
try:
|
||||
print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
def get_cached_data(self, url, route):
|
||||
"""Get cached data if it exists and is not older than the expiry time"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Calculate the timestamp for entries older than the expiry time
|
||||
expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
|
||||
pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
|
||||
# Calculate cache expiry time
|
||||
if route == 'pagination':
|
||||
# Pagination cache expires after 31 times the normal cache expiry
|
||||
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
|
||||
else:
|
||||
cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60) # Convert hours to seconds
|
||||
|
||||
# Get count of entries to be deleted (non-pagination)
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
|
||||
count_non_pagination = cursor.fetchone()[0]
|
||||
cursor.execute(
|
||||
"SELECT data FROM cache WHERE url = %s AND route = %s AND timestamp > %s",
|
||||
(url, route, cache_expiry)
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
# Get count of pagination entries to be deleted
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
|
||||
count_pagination = cursor.fetchone()[0]
|
||||
if result:
|
||||
print(f"Cache hit for {url} on route {route}")
|
||||
return json.loads(result[0])
|
||||
return None
|
||||
|
||||
# Delete old non-pagination entries
|
||||
cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
|
||||
def save_to_cache(self, url, route, data):
|
||||
"""Save data to cache"""
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
timestamp = int(time.time())
|
||||
|
||||
# Delete old pagination entries
|
||||
cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
|
||||
# Convert data to JSON string
|
||||
data_json = json.dumps(data)
|
||||
|
||||
if USE_POSTGRES:
|
||||
cursor.execute(
|
||||
"INSERT INTO cache (url, route, data, timestamp) VALUES (%s, %s, %s, %s) ON CONFLICT (url, route) DO UPDATE SET data = %s, timestamp = %s",
|
||||
(url, route, data_json, timestamp, data_json, timestamp)
|
||||
)
|
||||
else:
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO cache (url, route, data, timestamp) VALUES (?, ?, ?, ?)",
|
||||
(url, route, data_json, timestamp)
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Saved to cache: {url} on route {route}")
|
||||
|
||||
print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
|
||||
except Exception as e:
|
||||
print(f"Error during cache cleanup: {e}")
|
||||
def cleanup_old_cache_entries(self):
|
||||
"""Clean up old cache entries"""
|
||||
try:
|
||||
print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Calculate the timestamp for entries older than the expiry time
|
||||
expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
|
||||
pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
|
||||
|
||||
# Get count of entries to be deleted (non-pagination)
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE route != %s AND timestamp < %s", ('pagination', expiry_timestamp))
|
||||
count_non_pagination = cursor.fetchone()[0]
|
||||
|
||||
# Get count of pagination entries to be deleted
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE route = %s AND timestamp < %s", ('pagination', pagination_expiry_timestamp))
|
||||
count_pagination = cursor.fetchone()[0]
|
||||
|
||||
# Delete old non-pagination entries
|
||||
cursor.execute("DELETE FROM cache WHERE route != %s AND timestamp < %s", ('pagination', expiry_timestamp))
|
||||
|
||||
# Delete old pagination entries
|
||||
cursor.execute("DELETE FROM cache WHERE route = %s AND timestamp < %s", ('pagination', pagination_expiry_timestamp))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
|
||||
except Exception as e:
|
||||
print(f"Error during cache cleanup: {e}")
|
||||
|
||||
# Create global database manager instance
|
||||
db_manager = DatabaseManager()
|
||||
|
||||
# Backward compatibility functions
|
||||
def init_db():
|
||||
"""Initialize database (backward compatibility)"""
|
||||
db_manager.init_db()
|
||||
|
||||
def get_cached_data(url, route):
|
||||
"""Get cached data (backward compatibility)"""
|
||||
return db_manager.get_cached_data(url, route)
|
||||
|
||||
def save_to_cache(url, route, data):
|
||||
"""Save data to cache (backward compatibility)"""
|
||||
db_manager.save_to_cache(url, route, data)
|
||||
|
||||
def cleanup_old_cache_entries():
|
||||
"""Clean up old cache entries (backward compatibility)"""
|
||||
db_manager.cleanup_old_cache_entries()
|
||||
@@ -7,7 +7,6 @@ from app.services.browser import (
|
||||
visit_url_service,
|
||||
extract_seo_service,
|
||||
extract_meta_tags_service,
|
||||
detect_pagination_service,
|
||||
capture_outgoing_calls_service,
|
||||
get_resulting_url_service
|
||||
)
|
||||
@@ -98,33 +97,6 @@ async def extract_meta_tags(url: str, skipCache: bool = False, x_api_key: Option
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.get("/pagination")
|
||||
async def detect_pagination(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
|
||||
"""Detect pagination on a website and determine the pagination pattern"""
|
||||
# Validate API key
|
||||
if not x_api_key or x_api_key != API_KEY:
|
||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||
|
||||
# Decode URL if it's encoded
|
||||
decoded_url = unquote(url)
|
||||
|
||||
# Check cache first (unless skipCache is True)
|
||||
if not skipCache:
|
||||
cached_result = get_cached_data(decoded_url, "pagination")
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
try:
|
||||
# Call the service function
|
||||
result = await detect_pagination_service(decoded_url)
|
||||
|
||||
# Save to cache
|
||||
if(result["status"] == "success"):
|
||||
save_to_cache(decoded_url, "pagination", result)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.get("/outgoing-calls")
|
||||
async def capture_outgoing_calls(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
|
||||
|
||||
@@ -146,638 +146,6 @@ async def extract_meta_tags_service(decoded_url):
|
||||
print(f"Error during meta tag extraction: {e}")
|
||||
return {"status": "error", "url": decoded_url, "error": str(e)}
|
||||
|
||||
async def detect_pagination_service(decoded_url):
|
||||
"""Service function to detect pagination on a website"""
|
||||
print(f"Detecting pagination on: {decoded_url}")
|
||||
|
||||
# Define the operation to perform with the browser
|
||||
async def pagination_operation(page):
|
||||
try:
|
||||
# Navigate to the URL
|
||||
try:
|
||||
await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
||||
except Exception as e:
|
||||
print(f"Error navigating to URL: {e}")
|
||||
# Try to get the current URL even if navigation failed
|
||||
try:
|
||||
original_url = page.url
|
||||
except:
|
||||
original_url = decoded_url
|
||||
else:
|
||||
original_url = page.url
|
||||
|
||||
# Check for common pagination indicators
|
||||
try:
|
||||
pagination_data = await page.evaluate('''() => {
|
||||
const data = {
|
||||
hasPagination: false,
|
||||
paginationType: null,
|
||||
paginationElements: [],
|
||||
detectedParameter: null,
|
||||
lastPageNumber: null
|
||||
};
|
||||
|
||||
// Look for numbered pagination links (1, 2, 3...)
|
||||
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
|
||||
.filter(el => {
|
||||
const text = el.innerText.trim();
|
||||
|
||||
// check for data-page attribute
|
||||
const dataPage = el.getAttribute('data-page');
|
||||
if (dataPage) {
|
||||
return /^[0-9]+$/.test(dataPage);
|
||||
}
|
||||
|
||||
return /^[0-9]+$/.test(text) &&
|
||||
(el.tagName === 'A' || el.onclick ||
|
||||
el.closest('button, [role="button"]'));
|
||||
});
|
||||
|
||||
if(numberedLinks.length <= 1) {
|
||||
return data;
|
||||
}
|
||||
|
||||
// Look for next/prev buttons
|
||||
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||
.filter(el => {
|
||||
const text = el.innerText.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||
|
||||
return text.includes('next') ||
|
||||
text.includes('›') ||
|
||||
text.includes('»') ||
|
||||
text.includes('→') ||
|
||||
ariaLabel.includes('next') ||
|
||||
hasNextIcon;
|
||||
});
|
||||
|
||||
// Check for pagination containers
|
||||
const paginationContainers = Array.from(document.querySelectorAll(
|
||||
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
|
||||
));
|
||||
|
||||
// Collect all potential pagination elements
|
||||
if (numberedLinks.length > 0) {
|
||||
data.hasPagination = true;
|
||||
data.paginationType = 'numbered';
|
||||
|
||||
// Get href attributes or other identifiers from numbered links
|
||||
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
|
||||
return {
|
||||
text: el.innerText.trim(),
|
||||
dataPage: el.getAttribute('data-page'),
|
||||
href: el.tagName === 'A' ? el.href : null,
|
||||
classes: el.className,
|
||||
id: el.id
|
||||
};
|
||||
});
|
||||
|
||||
// Try to find the last page number
|
||||
const pageNumbers = numberedLinks
|
||||
.map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
|
||||
.filter(num => !isNaN(num));
|
||||
console.log(pageNumbers);
|
||||
if (pageNumbers.length > 0) {
|
||||
data.lastPageNumber = Math.max(...pageNumbers);
|
||||
}
|
||||
|
||||
// Also look for a "last page" element that might have text like "Last" or "»"
|
||||
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
|
||||
.find(el => {
|
||||
const text = el.innerText.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
return text.includes('last') ||
|
||||
text === '»' ||
|
||||
ariaLabel.includes('last page');
|
||||
});
|
||||
|
||||
if (lastPageElement && lastPageElement.href) {
|
||||
// Try to extract page number from the URL
|
||||
try {
|
||||
const url = new URL(lastPageElement.href);
|
||||
// Check common pagination parameters
|
||||
['page', 'p', 'pg'].forEach(param => {
|
||||
if (url.searchParams.has(param)) {
|
||||
const value = parseInt(url.searchParams.get(param));
|
||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||
console.log("Last page number: " + value);
|
||||
data.lastPageNumber = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Check for path-based pagination (like /page/10)
|
||||
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
|
||||
if (pathMatch && pathMatch[1]) {
|
||||
const value = parseInt(pathMatch[1]);
|
||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||
console.log("Last page number: " + value);
|
||||
data.lastPageNumber = value;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Error parsing last page URL:", e);
|
||||
}
|
||||
}
|
||||
} else if (nextButtons.length > 0) {
|
||||
data.hasPagination = true;
|
||||
data.paginationType = 'next-prev';
|
||||
|
||||
// Get information about next buttons
|
||||
data.paginationElements = nextButtons.slice(0, 3).map(el => {
|
||||
return {
|
||||
text: el.innerText.trim(),
|
||||
href: el.tagName === 'A' ? el.href : null,
|
||||
classes: el.className,
|
||||
id: el.id
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// Check for URL parameters that might indicate pagination
|
||||
const currentUrl = window.location.href;
|
||||
const urlParams = new URL(currentUrl).searchParams;
|
||||
|
||||
// Common pagination parameters
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
|
||||
|
||||
for (const param of paginationParams) {
|
||||
if (urlParams.has(param)) {
|
||||
data.detectedParameter = {
|
||||
name: param,
|
||||
value: urlParams.get(param)
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}''')
|
||||
except Exception as e:
|
||||
print(f"Error during JavaScript evaluation for pagination detection: {e}")
|
||||
# Return a safe default if JavaScript evaluation fails
|
||||
pagination_data = {
|
||||
'hasPagination': False,
|
||||
'paginationType': None,
|
||||
'paginationElements': [],
|
||||
'detectedParameter': None,
|
||||
'lastPageNumber': None
|
||||
}
|
||||
|
||||
print(f"Pagination data: {pagination_data}")
|
||||
# If pagination is detected, try to navigate to the next page by clicking
|
||||
next_page_url = None
|
||||
pagination_parameter = None
|
||||
url_template = None
|
||||
step_size = None
|
||||
original_parsed = None # Initialize the variable
|
||||
|
||||
if pagination_data['hasPagination']:
|
||||
print("Pagination detected, attempting to click on a pagination element")
|
||||
|
||||
# Capture the original URL before any navigation
|
||||
original_url_before_navigation = page.url
|
||||
|
||||
# Always try to click on a pagination element, regardless of type
|
||||
clicked = False
|
||||
|
||||
# First try to click on a numbered link (preferably "2" if we're on page 1)
|
||||
try:
|
||||
clicked = await page.evaluate('''() => {
|
||||
try {
|
||||
// First try to find and click on a "2" link or button
|
||||
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||
.filter(el => {
|
||||
// Check for text content "2"
|
||||
if (el.innerText.trim() === '2') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for href with page=2 or similar (for anchor elements)
|
||||
if (el.tagName === 'A' && el.href) {
|
||||
try {
|
||||
const url = new URL(el.href, window.location.origin);
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
|
||||
'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||
|
||||
for (const param of paginationParams) {
|
||||
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for path-based pagination like /page/2/ or /vacatures/page/2
|
||||
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
||||
if (pathMatch) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for path-based pagination where /page/2 is appended to the current path
|
||||
const currentPath = window.location.pathname;
|
||||
const expectedPath = currentPath.replace(/\/$/, '') + '/page/2';
|
||||
if (url.pathname === expectedPath) {
|
||||
return true;
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
// Check for data attributes that might indicate pagination
|
||||
if (el.getAttribute('data-page') === '2' ||
|
||||
el.getAttribute('data-pagenumber') === '2' ||
|
||||
el.getAttribute('data-page-number') === '2') {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
|
||||
if (page2Elements.length > 0) {
|
||||
console.log("Clicking on page 2 element");
|
||||
page2Elements[0].click();
|
||||
return true;
|
||||
}
|
||||
|
||||
// If no "2" link found, try any numbered link or button
|
||||
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||
.filter(el => /^\d+$/.test(el.innerText.trim()));
|
||||
|
||||
if (numberedElements.length > 0) {
|
||||
// Sort by number and get the second one (likely page 2)
|
||||
const sorted = numberedElements.sort((a, b) => {
|
||||
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
|
||||
});
|
||||
|
||||
// Get the second element if available (page 2), otherwise the first one
|
||||
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
|
||||
console.log("Clicking on numbered element: " + elementToClick.innerText);
|
||||
elementToClick.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
// If no numbered links, try next button
|
||||
const nextTexts = ['next', '›', '»', '→'];
|
||||
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||
.filter(el => {
|
||||
const text = el.textContent.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
return nextTexts.some(t => text.includes(t)) ||
|
||||
ariaLabel.includes('next') ||
|
||||
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||
});
|
||||
|
||||
if (nextElements.length > 0) {
|
||||
console.log("Clicking on next button");
|
||||
nextElements[0].click();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.error("Error during pagination click operation:", error);
|
||||
return false;
|
||||
}
|
||||
}''')
|
||||
|
||||
if clicked:
|
||||
print("Successfully clicked on pagination element")
|
||||
# Wait for navigation to complete
|
||||
await page.wait_for_load_state('networkidle', timeout=10000)
|
||||
await asyncio.sleep(2)
|
||||
next_page_url = page.url
|
||||
else:
|
||||
print("No clickable pagination element found")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error clicking on pagination element: {e}")
|
||||
# Continue with the process even if clicking fails
|
||||
|
||||
print(f"Next page URL: {next_page_url}")
|
||||
|
||||
# If we successfully navigated to the next page, analyze the URL difference
|
||||
if next_page_url and next_page_url != original_url_before_navigation:
|
||||
print('Searching for pagination parameter')
|
||||
# Parse both URLs using Python instead of JavaScript to avoid execution context issues
|
||||
try:
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
original_parsed_url = urlparse(original_url_before_navigation)
|
||||
current_parsed_url = urlparse(next_page_url)
|
||||
|
||||
# Check for differences in query parameters
|
||||
param_diff = None
|
||||
original_params = parse_qs(original_parsed_url.query)
|
||||
current_params = parse_qs(current_parsed_url.query)
|
||||
|
||||
# Common pagination parameters to check
|
||||
pagination_params = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']
|
||||
|
||||
for param in pagination_params:
|
||||
original_value = original_params.get(param, [None])[0]
|
||||
current_value = current_params.get(param, [None])[0]
|
||||
|
||||
if original_value != current_value and current_value is not None:
|
||||
param_diff = {
|
||||
'name': param,
|
||||
'originalValue': original_value,
|
||||
'currentValue': current_value
|
||||
}
|
||||
break
|
||||
|
||||
# Check for path differences
|
||||
path_diff = None
|
||||
original_path = original_parsed_url.path
|
||||
current_path = current_parsed_url.path
|
||||
|
||||
if original_path != current_path:
|
||||
original_segments = [s for s in original_path.split('/') if s]
|
||||
current_segments = [s for s in current_path.split('/') if s]
|
||||
|
||||
# Case 1: Same number of segments - find the one that changed
|
||||
if len(original_segments) == len(current_segments):
|
||||
for i, (orig_seg, curr_seg) in enumerate(zip(original_segments, current_segments)):
|
||||
if orig_seg != curr_seg:
|
||||
# Check if the difference is numeric
|
||||
if orig_seg.isdigit() and curr_seg.isdigit():
|
||||
path_diff = {
|
||||
'type': 'replace',
|
||||
'index': i,
|
||||
'originalValue': orig_seg,
|
||||
'currentValue': curr_seg
|
||||
}
|
||||
break
|
||||
|
||||
# Case 2: Current path has more segments - check for added pagination segments
|
||||
elif len(current_segments) > len(original_segments):
|
||||
# Look for patterns like /page/NUMBER or /p/NUMBER at the end
|
||||
import re
|
||||
page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE)
|
||||
|
||||
# Check the last two segments of the current path
|
||||
if len(current_segments) >= 2:
|
||||
last_two_segments = '/'.join(current_segments[-2:])
|
||||
match = page_pattern.match(last_two_segments)
|
||||
|
||||
if match:
|
||||
path_diff = {
|
||||
'type': 'append',
|
||||
'pageSegment': match.group(1), # 'page' or 'p'
|
||||
'pageNumber': match.group(2), # the actual number
|
||||
'originalSegments': original_segments,
|
||||
'currentSegments': current_segments
|
||||
}
|
||||
|
||||
# If no pattern match, check if the last segment is numeric
|
||||
if not path_diff and current_segments:
|
||||
last_segment = current_segments[-1]
|
||||
if last_segment.isdigit():
|
||||
path_diff = {
|
||||
'type': 'append',
|
||||
'pageSegment': None,
|
||||
'pageNumber': last_segment,
|
||||
'originalSegments': original_segments,
|
||||
'currentSegments': current_segments
|
||||
}
|
||||
|
||||
original_parsed = {
|
||||
'paramDiff': param_diff,
|
||||
'pathDiff': path_diff,
|
||||
'originalUrl': original_url_before_navigation,
|
||||
'currentUrl': next_page_url
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during URL analysis: {e}")
|
||||
original_parsed = {
|
||||
'paramDiff': None,
|
||||
'pathDiff': None,
|
||||
'originalUrl': original_url_before_navigation,
|
||||
'currentUrl': next_page_url,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
# Determine the pagination parameter and create URL template
|
||||
if original_parsed:
|
||||
print(original_parsed)
|
||||
if original_parsed.get('paramDiff'):
|
||||
param_name = original_parsed['paramDiff']['name']
|
||||
pagination_parameter = {
|
||||
'type': 'query',
|
||||
'name': param_name,
|
||||
'value': original_parsed['paramDiff']['currentValue']
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
|
||||
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
|
||||
curr_val = original_parsed['paramDiff']['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for query parameter
|
||||
try:
|
||||
from urllib.parse import urlparse, urlencode, parse_qs
|
||||
|
||||
parsed_url = urlparse(original_url_before_navigation)
|
||||
params = parse_qs(parsed_url.query)
|
||||
params[param_name] = ['{PAGE_NUMBER}']
|
||||
|
||||
# Reconstruct the URL
|
||||
new_query = urlencode(params, doseq=True)
|
||||
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
|
||||
if new_query:
|
||||
url_template += f"?{new_query}"
|
||||
if parsed_url.fragment:
|
||||
url_template += f"#{parsed_url.fragment}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for query parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
elif original_parsed.get('pathDiff'):
|
||||
path_diff = original_parsed['pathDiff']
|
||||
|
||||
if path_diff.get('type') == 'replace':
|
||||
# Handle existing path segment replacement
|
||||
path_index = path_diff['index']
|
||||
pagination_parameter = {
|
||||
'type': 'path',
|
||||
'index': path_index,
|
||||
'value': path_diff['currentValue']
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
||||
orig_val = path_diff['originalValue']
|
||||
curr_val = path_diff['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for path parameter replacement
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed_url = urlparse(original_url_before_navigation)
|
||||
path_segments = [s for s in parsed_url.path.split('/') if s]
|
||||
path_segments[path_index] = '{PAGE_NUMBER}'
|
||||
|
||||
# Reconstruct the URL
|
||||
new_path = '/' + '/'.join(path_segments)
|
||||
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}"
|
||||
if parsed_url.query:
|
||||
url_template += f"?{parsed_url.query}"
|
||||
if parsed_url.fragment:
|
||||
url_template += f"#{parsed_url.fragment}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for path parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
elif path_diff.get('type') == 'append':
|
||||
# Handle new pagination segments being appended
|
||||
pagination_parameter = {
|
||||
'type': 'path_append',
|
||||
'pageSegment': path_diff.get('pageSegment'),
|
||||
'pageNumber': path_diff['pageNumber']
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR APPENDED PATH PARAM ---
|
||||
try:
|
||||
curr_val = path_diff['pageNumber']
|
||||
# For appended pagination, assume we started from page 1 (implicit)
|
||||
orig_num = 1
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for appended path parameter
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed_url = urlparse(original_url_before_navigation)
|
||||
new_path = parsed_url.path
|
||||
|
||||
# Remove trailing slash if present
|
||||
if new_path.endswith('/'):
|
||||
new_path = new_path[:-1]
|
||||
|
||||
# Append the pagination segment
|
||||
if path_diff.get('pageSegment'):
|
||||
new_path += f"/{path_diff['pageSegment']}/{{PAGE_NUMBER}}"
|
||||
else:
|
||||
new_path += "/{PAGE_NUMBER}"
|
||||
|
||||
# Reconstruct the URL
|
||||
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}"
|
||||
if parsed_url.query:
|
||||
url_template += f"?{parsed_url.query}"
|
||||
if parsed_url.fragment:
|
||||
url_template += f"#{parsed_url.fragment}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for appended path parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
if url_template:
|
||||
# Decode URL-encoded characters in the template
|
||||
import urllib.parse
|
||||
url_template = urllib.parse.unquote(url_template)
|
||||
|
||||
# If we couldn't determine the URL template from navigation, try to infer it
|
||||
if not url_template and pagination_data['detectedParameter']:
|
||||
param_name = pagination_data['detectedParameter']['name']
|
||||
try:
|
||||
from urllib.parse import urlparse, urlencode, parse_qs
|
||||
|
||||
parsed_url = urlparse(original_url_before_navigation)
|
||||
params = parse_qs(parsed_url.query)
|
||||
params[param_name] = ['{PAGE_NUMBER}']
|
||||
|
||||
# Reconstruct the URL
|
||||
new_query = urlencode(params, doseq=True)
|
||||
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
|
||||
if new_query:
|
||||
url_template += f"?{new_query}"
|
||||
if parsed_url.fragment:
|
||||
url_template += f"#{parsed_url.fragment}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating inferred URL template: {e}")
|
||||
url_template = None
|
||||
|
||||
# If still no template and we have pagination elements, try to infer from the current URL structure
|
||||
if not url_template and pagination_data['hasPagination']:
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
import re
|
||||
|
||||
parsed_url = urlparse(original_url_before_navigation)
|
||||
path = parsed_url.path
|
||||
|
||||
# Remove trailing slash if present
|
||||
if path.endswith('/'):
|
||||
path = path[:-1]
|
||||
|
||||
# Check if the current URL already has a pagination pattern
|
||||
page_pattern = re.compile(r'/(page|p)/\d+$', re.IGNORECASE)
|
||||
if page_pattern.search(path):
|
||||
# Replace the existing page number with placeholder
|
||||
path = page_pattern.sub(r'/\1/{PAGE_NUMBER}', path)
|
||||
else:
|
||||
# Add pagination pattern
|
||||
path += '/page/{PAGE_NUMBER}'
|
||||
|
||||
# Reconstruct the URL
|
||||
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{path}"
|
||||
if parsed_url.query:
|
||||
url_template += f"?{parsed_url.query}"
|
||||
if parsed_url.fragment:
|
||||
url_template += f"#{parsed_url.fragment}"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating fallback URL template: {e}")
|
||||
url_template = None
|
||||
|
||||
# Return the pagination detection results with a simplified structure
|
||||
result = {
|
||||
"status": "success",
|
||||
"hasPagination": pagination_data['hasPagination'],
|
||||
"urlTemplate": url_template,
|
||||
"lastPage": pagination_data['lastPageNumber'],
|
||||
"stepSize": step_size if step_size is not None and step_size >= 5 else 1
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during pagination detection: {e}")
|
||||
return {
|
||||
"status": "error",
|
||||
"hasPagination": False,
|
||||
"urlTemplate": None,
|
||||
"lastPage": None,
|
||||
"stepSize": None,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
# Perform the operation
|
||||
result = await safe_browser_operation(decoded_url, pagination_operation)
|
||||
|
||||
# Check if the result is an error from safe_browser_operation
|
||||
if isinstance(result, dict) and result.get("status") == "error":
|
||||
return result
|
||||
|
||||
return result
|
||||
|
||||
async def capture_outgoing_calls_service(decoded_url):
|
||||
"""Service function to capture outgoing API calls from a website"""
|
||||
print(f"Capturing outgoing calls from: {decoded_url}")
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime
|
||||
from app.database import DB_PATH, CACHE_EXPIRY_HOURS
|
||||
from app.database import db_manager, CACHE_EXPIRY_HOURS
|
||||
|
||||
def clear_cache():
|
||||
"""Clear the entire cache database"""
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn = db_manager.get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM cache")
|
||||
conn.commit()
|
||||
@@ -14,7 +13,7 @@ def clear_cache():
|
||||
|
||||
def get_cache_stats():
|
||||
"""Get cache statistics"""
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn = db_manager.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get total entries
|
||||
@@ -27,7 +26,7 @@ def get_cache_stats():
|
||||
|
||||
# Get recent entries (last 24 hours)
|
||||
recent_timestamp = int(time.time()) - (24 * 60 * 60)
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,))
|
||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > %s", (recent_timestamp,))
|
||||
recent_entries = cursor.fetchone()[0]
|
||||
|
||||
# Get oldest entry timestamp
|
||||
@@ -50,6 +49,7 @@ def get_cache_stats():
|
||||
"recent_entries": recent_entries,
|
||||
"oldest_entry": oldest_date,
|
||||
"newest_entry": newest_date,
|
||||
"cache_expiry_hours": CACHE_EXPIRY_HOURS
|
||||
"cache_expiry_hours": CACHE_EXPIRY_HOURS,
|
||||
"database_type": db_manager.db_type
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user