From 6051e41cfa2da07aed9a892bbd4e8893f2a089f4 Mon Sep 17 00:00:00 2001 From: Bram Date: Sat, 21 Jun 2025 20:31:38 +0200 Subject: [PATCH] why not --- Dockers/puppeteer-api/app/services/browser.py | 670 ++++++++++-------- Dockers/puppeteer-api/main.py | 481 +------------ Dockers/puppeteer-api/test_pagination.py | 59 ++ 3 files changed, 449 insertions(+), 761 deletions(-) create mode 100644 Dockers/puppeteer-api/test_pagination.py diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index c7b76ec..106fc63 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -153,153 +153,173 @@ async def detect_pagination_service(decoded_url): async def pagination_operation(page): try: # Navigate to the URL - await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000) - original_url = page.url + try: + await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000) + except Exception as e: + print(f"Error navigating to URL: {e}") + # Try to get the current URL even if navigation failed + try: + original_url = page.url + except: + original_url = decoded_url + else: + original_url = page.url # Check for common pagination indicators - pagination_data = await page.evaluate('''() => { - const data = { - hasPagination: false, - paginationType: null, - paginationElements: [], - detectedParameter: null, - lastPageNumber: null - }; + try: + pagination_data = await page.evaluate('''() => { + const data = { + hasPagination: false, + paginationType: null, + paginationElements: [], + detectedParameter: null, + lastPageNumber: null + }; - // Look for numbered pagination links (1, 2, 3...) - const numberedLinks = Array.from(document.querySelectorAll('a, button, span')) - .filter(el => { - const text = el.innerText.trim(); + // Look for numbered pagination links (1, 2, 3...) + const numberedLinks = Array.from(document.querySelectorAll('a, button, span')) + .filter(el => { + const text = el.innerText.trim(); - // check for data-page attribute - const dataPage = el.getAttribute('data-page'); - if (dataPage) { - return /^[0-9]+$/.test(dataPage); - } + // check for data-page attribute + const dataPage = el.getAttribute('data-page'); + if (dataPage) { + return /^[0-9]+$/.test(dataPage); + } - return /^[0-9]+$/.test(text) && - (el.tagName === 'A' || el.onclick || - el.closest('button, [role="button"]')); - }); - - console.log(numberedLinks); - - // Look for next/prev buttons - const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]')) - .filter(el => { - const text = el.innerText.trim().toLowerCase(); - const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; - const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]'); - - return text.includes('next') || - text.includes('›') || - text.includes('»') || - text.includes('→') || - ariaLabel.includes('next') || - hasNextIcon; - }); - - // Check for pagination containers - const paginationContainers = Array.from(document.querySelectorAll( - '.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]' - )); - - // Collect all potential pagination elements - if (numberedLinks.length > 0) { - data.hasPagination = true; - data.paginationType = 'numbered'; - - // Get href attributes or other identifiers from numbered links - data.paginationElements = numberedLinks.slice(0, 5).map(el => { - return { - text: el.innerText.trim(), - dataPage: el.getAttribute('data-page'), - href: el.tagName === 'A' ? el.href : null, - classes: el.className, - id: el.id - }; - }); - - // Try to find the last page number - const pageNumbers = numberedLinks - .map(el => parseInt(el.innerText.replace(/[^\d]/g, ''))) - .filter(num => !isNaN(num)); - - if (pageNumbers.length > 0) { - data.lastPageNumber = Math.max(...pageNumbers); - } - - // Also look for a "last page" element that might have text like "Last" or "»" - const lastPageElement = Array.from(document.querySelectorAll('a, button')) - .find(el => { - const text = el.innerText.trim().toLowerCase(); - const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; - return text.includes('last') || - text === '»' || - ariaLabel.includes('last page'); + return /^[0-9]+$/.test(text) && + (el.tagName === 'A' || el.onclick || + el.closest('button, [role="button"]')); }); - if (lastPageElement && lastPageElement.href) { - // Try to extract page number from the URL - try { - const url = new URL(lastPageElement.href); - // Check common pagination parameters - ['page', 'p', 'pg'].forEach(param => { - if (url.searchParams.has(param)) { - const value = parseInt(url.searchParams.get(param)); + console.log(numberedLinks); + + // Look for next/prev buttons + const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]')) + .filter(el => { + const text = el.innerText.trim().toLowerCase(); + const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; + const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]'); + + return text.includes('next') || + text.includes('›') || + text.includes('»') || + text.includes('→') || + ariaLabel.includes('next') || + hasNextIcon; + }); + + // Check for pagination containers + const paginationContainers = Array.from(document.querySelectorAll( + '.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]' + )); + + // Collect all potential pagination elements + if (numberedLinks.length > 0) { + data.hasPagination = true; + data.paginationType = 'numbered'; + + // Get href attributes or other identifiers from numbered links + data.paginationElements = numberedLinks.slice(0, 5).map(el => { + return { + text: el.innerText.trim(), + dataPage: el.getAttribute('data-page'), + href: el.tagName === 'A' ? el.href : null, + classes: el.className, + id: el.id + }; + }); + + // Try to find the last page number + const pageNumbers = numberedLinks + .map(el => parseInt(el.innerText.replace(/[^\d]/g, ''))) + .filter(num => !isNaN(num)); + + if (pageNumbers.length > 0) { + data.lastPageNumber = Math.max(...pageNumbers); + } + + // Also look for a "last page" element that might have text like "Last" or "»" + const lastPageElement = Array.from(document.querySelectorAll('a, button')) + .find(el => { + const text = el.innerText.trim().toLowerCase(); + const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; + return text.includes('last') || + text === '»' || + ariaLabel.includes('last page'); + }); + + if (lastPageElement && lastPageElement.href) { + // Try to extract page number from the URL + try { + const url = new URL(lastPageElement.href); + // Check common pagination parameters + ['page', 'p', 'pg'].forEach(param => { + if (url.searchParams.has(param)) { + const value = parseInt(url.searchParams.get(param)); + if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { + data.lastPageNumber = value; + } + } + }); + + // Check for path-based pagination (like /page/10) + const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i); + if (pathMatch && pathMatch[1]) { + const value = parseInt(pathMatch[1]); if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { data.lastPageNumber = value; } } - }); - - // Check for path-based pagination (like /page/10) - const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i); - if (pathMatch && pathMatch[1]) { - const value = parseInt(pathMatch[1]); - if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { - data.lastPageNumber = value; - } + } catch (e) { + console.error("Error parsing last page URL:", e); } - } catch (e) { - console.error("Error parsing last page URL:", e); + } + } else if (nextButtons.length > 0) { + data.hasPagination = true; + data.paginationType = 'next-prev'; + + // Get information about next buttons + data.paginationElements = nextButtons.slice(0, 3).map(el => { + return { + text: el.innerText.trim(), + href: el.tagName === 'A' ? el.href : null, + classes: el.className, + id: el.id + }; + }); + } + + // Check for URL parameters that might indicate pagination + const currentUrl = window.location.href; + const urlParams = new URL(currentUrl).searchParams; + + // Common pagination parameters + const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit']; + + for (const param of paginationParams) { + if (urlParams.has(param)) { + data.detectedParameter = { + name: param, + value: urlParams.get(param) + }; + break; } } - } else if (nextButtons.length > 0) { - data.hasPagination = true; - data.paginationType = 'next-prev'; - // Get information about next buttons - data.paginationElements = nextButtons.slice(0, 3).map(el => { - return { - text: el.innerText.trim(), - href: el.tagName === 'A' ? el.href : null, - classes: el.className, - id: el.id - }; - }); + return data; + }''') + except Exception as e: + print(f"Error during JavaScript evaluation for pagination detection: {e}") + # Return a safe default if JavaScript evaluation fails + pagination_data = { + 'hasPagination': False, + 'paginationType': None, + 'paginationElements': [], + 'detectedParameter': None, + 'lastPageNumber': None } - // Check for URL parameters that might indicate pagination - const currentUrl = window.location.href; - const urlParams = new URL(currentUrl).searchParams; - - // Common pagination parameters - const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit']; - - for (const param of paginationParams) { - if (urlParams.has(param)) { - data.detectedParameter = { - name: param, - value: urlParams.get(param) - }; - break; - } - } - - return data; - }''') - # If pagination is detected, try to navigate to the next page by clicking next_page_url = None pagination_parameter = None @@ -315,86 +335,91 @@ async def detect_pagination_service(decoded_url): # First try to click on a numbered link (preferably "2" if we're on page 1) try: clicked = await page.evaluate('''() => { - // First try to find and click on a "2" link or button - const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]')) - .filter(el => { - // Check for text content "2" - if (el.innerText.trim() === '2') { - return true; - } + try { + // First try to find and click on a "2" link or button + const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]')) + .filter(el => { + // Check for text content "2" + if (el.innerText.trim() === '2') { + return true; + } - // Check for href with page=2 or similar (for anchor elements) - if (el.tagName === 'A' && el.href) { - try { - const url = new URL(el.href, window.location.origin); - const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', - 'currentpage', 'pagenum', 'pageNumber', 'paged']; + // Check for href with page=2 or similar (for anchor elements) + if (el.tagName === 'A' && el.href) { + try { + const url = new URL(el.href, window.location.origin); + const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', + 'currentpage', 'pagenum', 'pageNumber', 'paged']; - for (const param of paginationParams) { - if (url.searchParams.has(param) && url.searchParams.get(param) === '2') { + for (const param of paginationParams) { + if (url.searchParams.has(param) && url.searchParams.get(param) === '2') { + return true; + } + } + + // Check for path-based pagination like /page/2/ + const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i); + if (pathMatch) { return true; } - } + } catch (e) {} + } - // Check for path-based pagination like /page/2/ - const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i); - if (pathMatch) { - return true; - } - } catch (e) {} - } + // Check for data attributes that might indicate pagination + if (el.getAttribute('data-page') === '2' || + el.getAttribute('data-pagenumber') === '2' || + el.getAttribute('data-page-number') === '2') { + return true; + } - // Check for data attributes that might indicate pagination - if (el.getAttribute('data-page') === '2' || - el.getAttribute('data-pagenumber') === '2' || - el.getAttribute('data-page-number') === '2') { - return true; - } + return false; + }); - return false; - }); + if (page2Elements.length > 0) { + console.log("Clicking on page 2 element"); + page2Elements[0].click(); + return true; + } - if (page2Elements.length > 0) { - console.log("Clicking on page 2 element"); - page2Elements[0].click(); - return true; + // If no "2" link found, try any numbered link or button + const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]')) + .filter(el => /^\d+$/.test(el.innerText.trim())); + + if (numberedElements.length > 0) { + // Sort by number and get the second one (likely page 2) + const sorted = numberedElements.sort((a, b) => { + return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim()); + }); + + // Get the second element if available (page 2), otherwise the first one + const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0]; + console.log("Clicking on numbered element: " + elementToClick.innerText); + elementToClick.click(); + return true; + } + + // If no numbered links, try next button + const nextTexts = ['next', '›', '»', '→']; + const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]')) + .filter(el => { + const text = el.textContent.trim().toLowerCase(); + const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; + return nextTexts.some(t => text.includes(t)) || + ariaLabel.includes('next') || + el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]'); + }); + + if (nextElements.length > 0) { + console.log("Clicking on next button"); + nextElements[0].click(); + return true; + } + + return false; + } catch (error) { + console.error("Error during pagination click operation:", error); + return false; } - - // If no "2" link found, try any numbered link or button - const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]')) - .filter(el => /^\d+$/.test(el.innerText.trim())); - - if (numberedElements.length > 0) { - // Sort by number and get the second one (likely page 2) - const sorted = numberedElements.sort((a, b) => { - return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim()); - }); - - // Get the second element if available (page 2), otherwise the first one - const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0]; - console.log("Clicking on numbered element: " + elementToClick.innerText); - elementToClick.click(); - return true; - } - - // If no numbered links, try next button - const nextTexts = ['next', '›', '»', '→']; - const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]')) - .filter(el => { - const text = el.textContent.trim().toLowerCase(); - const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; - return nextTexts.some(t => text.includes(t)) || - ariaLabel.includes('next') || - el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]'); - }); - - if (nextElements.length > 0) { - console.log("Clicking on next button"); - nextElements[0].click(); - return true; - } - - return false; }''') if clicked: @@ -407,126 +432,166 @@ async def detect_pagination_service(decoded_url): except Exception as e: print(f"Error clicking on pagination element: {e}") + # Continue with the process even if clicking fails # If we successfully navigated to the next page, analyze the URL difference if next_page_url: print('Searching for pagination parameter') # Parse both URLs - original_parsed = await page.evaluate(f'''(originalUrl) => {{ - const original = new URL(originalUrl); - const current = new URL(window.location.href); + try: + original_parsed = await page.evaluate(f'''(originalUrl) => {{ + try {{ + const original = new URL(originalUrl); + const current = new URL(window.location.href); - // Check for differences in query parameters - let paramDiff = null; + // Check for differences in query parameters + let paramDiff = null; - // Common pagination parameters to check - const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']; + // Common pagination parameters to check + const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']; - for (const param of paginationParams) {{ - const originalValue = original.searchParams.get(param); - const currentValue = current.searchParams.get(param); + for (const param of paginationParams) {{ + const originalValue = original.searchParams.get(param); + const currentValue = current.searchParams.get(param); - if (originalValue !== currentValue && currentValue !== null) {{ - paramDiff = {{ - name: param, - originalValue: originalValue, - currentValue: currentValue - }}; - break; - }} - }} + if (originalValue !== currentValue && currentValue !== null) {{ + paramDiff = {{ + name: param, + originalValue: originalValue, + currentValue: currentValue + }}; + break; + }} + }} - // Check for path differences (like /page/1 vs /page/2) - const originalPath = original.pathname; - const currentPath = current.pathname; + // Check for path differences (like /page/1 vs /page/2) + const originalPath = original.pathname; + const currentPath = current.pathname; - let pathDiff = null; - if (originalPath !== currentPath) {{ - const originalSegments = originalPath.split('/').filter(s => s); - const currentSegments = currentPath.split('/').filter(s => s); + let pathDiff = null; + if (originalPath !== currentPath) {{ + const originalSegments = originalPath.split('/').filter(s => s); + const currentSegments = currentPath.split('/').filter(s => s); - // Find the segment that changed - if (originalSegments.length === currentSegments.length) {{ - for (let i = 0; i < originalSegments.length; i++) {{ - if (originalSegments[i] !== currentSegments[i]) {{ - // Check if the difference is numeric - if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{ - pathDiff = {{ - index: i, - originalValue: originalSegments[i], - currentValue: currentSegments[i] - }}; + // Find the segment that changed + if (originalSegments.length === currentSegments.length) {{ + for (let i = 0; i < originalSegments.length; i++) {{ + if (originalSegments[i] !== currentSegments[i]) {{ + // Check if the difference is numeric + if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{ + pathDiff = {{ + index: i, + originalValue: originalSegments[i], + currentValue: currentSegments[i] + }}; + }} + }} }} }} }} + + return {{ + paramDiff, + pathDiff, + originalUrl: originalUrl, + currentUrl: window.location.href + }}; + }} catch (error) {{ + console.error("Error during URL analysis:", error); + return {{ + paramDiff: null, + pathDiff: null, + originalUrl: originalUrl, + currentUrl: window.location.href, + error: error.message + }}; }} - }} - - return {{ - paramDiff, - pathDiff, - originalUrl: originalUrl, - currentUrl: window.location.href - }}; - }}''', original_url) - - # Determine the pagination parameter and create URL template - print(original_parsed) - if original_parsed['paramDiff']: - param_name = original_parsed['paramDiff']['name'] - pagination_parameter = { - 'type': 'query', - 'name': param_name, - 'value': original_parsed['paramDiff']['currentValue'] + }}''', original_url) + except Exception as e: + print(f"Error during URL analysis: {e}") + original_parsed = { + 'paramDiff': None, + 'pathDiff': None, + 'originalUrl': original_url, + 'currentUrl': next_page_url, + 'error': str(e) } - # --- STEP SIZE DETECTION FOR QUERY PARAM --- - orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0 - curr_val = original_parsed['paramDiff']['currentValue'] - try: - if orig_val is not None and curr_val is not None: - orig_num = int(orig_val) - curr_num = int(curr_val) - step_size = abs(curr_num - orig_num) - except Exception: - step_size = None + # Determine the pagination parameter and create URL template + print(original_parsed) + if original_parsed['paramDiff']: + param_name = original_parsed['paramDiff']['name'] + pagination_parameter = { + 'type': 'query', + 'name': param_name, + 'value': original_parsed['paramDiff']['currentValue'] + } - # Create URL template for query parameter + # --- STEP SIZE DETECTION FOR QUERY PARAM --- + orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0 + curr_val = original_parsed['paramDiff']['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None + + # Create URL template for query parameter + try: url_obj = await page.evaluate(f'''(url, paramName) => {{ - const urlObj = new URL(url); - urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); - return urlObj.toString(); + try {{ + const urlObj = new URL(url); + urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating URL template:", error); + return null; + }} }}''', original_url, param_name) url_template = url_obj + except Exception as e: + print(f"Error creating URL template for query parameter: {e}") + url_template = None - elif original_parsed['pathDiff']: - path_index = original_parsed['pathDiff']['index'] - pagination_parameter = { - 'type': 'path', - 'index': path_index, - 'value': original_parsed['pathDiff']['currentValue'] - } + elif original_parsed['pathDiff']: + path_index = original_parsed['pathDiff']['index'] + pagination_parameter = { + 'type': 'path', + 'index': path_index, + 'value': original_parsed['pathDiff']['currentValue'] + } - # --- STEP SIZE DETECTION FOR PATH PARAM --- - orig_val = original_parsed['pathDiff']['originalValue'] - curr_val = original_parsed['pathDiff']['currentValue'] - try: - if orig_val is not None and curr_val is not None: - orig_num = int(orig_val) - curr_num = int(curr_val) - step_size = abs(curr_num - orig_num) - except Exception: - step_size = None + # --- STEP SIZE DETECTION FOR PATH PARAM --- + orig_val = original_parsed['pathDiff']['originalValue'] + curr_val = original_parsed['pathDiff']['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None - # Create URL template for path parameter + # Create URL template for path parameter + try: url_template = await page.evaluate(f'''(url, pathIndex) => {{ - const urlObj = new URL(url); - const pathSegments = urlObj.pathname.split('/').filter(s => s); - pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; - urlObj.pathname = '/' + pathSegments.join('/'); - return urlObj.toString(); + try {{ + const urlObj = new URL(url); + const pathSegments = urlObj.pathname.split('/').filter(s => s); + pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; + urlObj.pathname = '/' + pathSegments.join('/'); + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating path URL template:", error); + return null; + }} }}''', original_url, path_index) + except Exception as e: + print(f"Error creating URL template for path parameter: {e}") + url_template = None if url_template: # Decode URL-encoded characters in the template @@ -536,11 +601,20 @@ async def detect_pagination_service(decoded_url): # If we couldn't determine the URL template from navigation, try to infer it if not url_template and pagination_data['detectedParameter']: param_name = pagination_data['detectedParameter']['name'] - url_template = await page.evaluate(f'''(url, paramName) => {{ - const urlObj = new URL(url); - urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); - return urlObj.toString(); - }}''', original_url, param_name) + try: + url_template = await page.evaluate(f'''(url, paramName) => {{ + try {{ + const urlObj = new URL(url); + urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating inferred URL template:", error); + return null; + }} + }}''', original_url, param_name) + except Exception as e: + print(f"Error creating inferred URL template: {e}") + url_template = None # Return the pagination detection results with a simplified structure result = { diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index b5b72b9..206c5f7 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -655,475 +655,30 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)): if not x_api_key or x_api_key != API_KEY: raise HTTPException(status_code=401, detail="Invalid API key") - conn = sqlite3.connect(DB_PATH) - cursor = conn.cursor() - - # Get total entries - cursor.execute("SELECT COUNT(*) FROM cache") - total_entries = cursor.fetchone()[0] - - # Get entries by route - cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route") - routes = {route: count for route, count in cursor.fetchall()} - - # Get recent entries (last 24 hours) - recent_timestamp = int(time.time()) - (24 * 60 * 60) - cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,)) - recent_entries = cursor.fetchone()[0] - - # Get oldest entry timestamp - cursor.execute("SELECT MIN(timestamp) FROM cache") - oldest_timestamp = cursor.fetchone()[0] - oldest_date = datetime.fromtimestamp(oldest_timestamp).isoformat() if oldest_timestamp else None - - # Get newest entry timestamp - cursor.execute("SELECT MAX(timestamp) FROM cache") - newest_timestamp = cursor.fetchone()[0] - newest_date = datetime.fromtimestamp(newest_timestamp).isoformat() if newest_timestamp else None - - conn.close() - - return { - "status": "success", - "stats": { - "total_entries": total_entries, - "entries_by_route": routes, - "recent_entries": recent_entries, - "oldest_entry": oldest_date, - "newest_entry": newest_date, - "cache_expiry_hours": CACHE_EXPIRY_HOURS, - "cleanup_schedule": CLEANUP_CRON - } - } - -@app.get("/pagination") -async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)): - """Detect pagination on a website and determine the pagination pattern""" - # Validate API key - if not x_api_key or x_api_key != API_KEY: - raise HTTPException(status_code=401, detail="Invalid API key") - - # Decode URL if it's encoded - decoded_url = unquote(url) - - # Check cache first - cached_result = get_cached_data(decoded_url, "pagination") - if cached_result: - return cached_result - try: - print(f"Detecting pagination on: {decoded_url}") + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() - # Define the operation to perform with the browser - async def pagination_operation(page): - try: - # Navigate to the URL - await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000) - original_url = page.url + # Get total count + cursor.execute("SELECT COUNT(*) FROM cache") + total_count = cursor.fetchone()[0] - print(f"Successfully loaded page: {original_url}") + # Get count by route + cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route") + route_counts = dict(cursor.fetchall()) - # Analyze the page for pagination information - pagination_info = await page.evaluate('''() => { - // Find the last page number if available - const findLastPageNumber = () => { - // Get all links on the page - const links = Array.from(document.querySelectorAll('a')); + # Get oldest and newest entries + cursor.execute("SELECT MIN(timestamp), MAX(timestamp) FROM cache") + min_time, max_time = cursor.fetchone() - // Strategy 1: Find numeric links (page numbers) - const numericLinks = links.filter(link => { - const text = link.innerText.trim(); - return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; - }); - - if (numericLinks.length > 0) { - const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); - return Math.max(...numericValues); - } - - // Strategy 2: Look for "last page" link - const lastLinks = links.filter(link => { - const text = link.innerText.trim().toLowerCase(); - const classes = (link.className || '').toLowerCase(); - const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase(); - - return (text === 'last' || - classes.includes('last') || - ariaLabel.includes('last') || - link.getAttribute('rel') === 'last'); - }); - - if (lastLinks.length > 0) { - const lastLink = lastLinks[0]; - const href = lastLink.href; - - // Common patterns: page=X, /page/X, etc. - const pagePatterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/ - ]; - - for (const pattern of pagePatterns) { - const match = href.match(pattern); - if (match && match[1]) { - return parseInt(match[1]); - } - } - } - - // Strategy 3: Analyze all URLs for page numbers - const pageNumbersFromUrls = []; - links.forEach(link => { - if (!link.href || link.href === '#') return; - - // Check for common pagination URL patterns - const patterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/, - /\/pages\/(\d+)/ - ]; - - for (const pattern of patterns) { - const match = link.href.match(pattern); - if (match && match[1]) { - pageNumbersFromUrls.push(parseInt(match[1])); - } - } - }); - - if (pageNumbersFromUrls.length > 0) { - return Math.max(...pageNumbersFromUrls); - } - - return null; - }; - - // Find a pagination link to click - const findPaginationLink = () => { - const links = Array.from(document.querySelectorAll('a')); - - // Try to find a page "2" link first (most reliable) - const page2Link = links.find(link => { - const text = link.innerText.trim(); - return text === '2' && link.href && link.href !== '#'; - }); - - if (page2Link) { - return { element: page2Link, href: page2Link.href, type: 'numeric' }; - } - - // Try common "next page" selectors - const nextSelectors = [ - 'a.next', - 'a.page-next', - 'a[rel="next"]', - 'a[aria-label="Next page"]', - 'a[aria-label="next"]' - ]; - - for (const selector of nextSelectors) { - const element = document.querySelector(selector); - if (element && element.href && element.href !== '#') { - return { element, href: element.href, type: 'next' }; - } - } - - // Look for any link that might be pagination - const paginationLinks = links.filter(link => { - if (!link.href || link.href === '#') return false; - - const text = link.innerText.trim(); - const href = link.href; - - // Check for numeric text or next/prev indicators - const isNumeric = /^[0-9]+$/.test(text) && text !== '1'; - const isNextPrev = /next|prev|previous|older|newer/i.test(text) || - /[»«‹›<>]/.test(text); - - // Check for page parameter in URL - const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href); - - return (isNumeric || isNextPrev || hasPageParam); - }); - - if (paginationLinks.length > 0) { - const link = paginationLinks[0]; - return { element: link, href: link.href, type: 'other' }; - } - - return null; - }; - - const lastPage = findLastPageNumber(); - const paginationLink = findPaginationLink(); - - if (paginationLink) { - // Click the link - paginationLink.element.click(); - return { - clicked: true, - href: paginationLink.href, - type: paginationLink.type, - lastPage - }; - } - - return { clicked: false, lastPage }; - }''') - - # If no pagination was found or clicked - if not pagination_info.get('clicked', False): - return { - "status": "success", - "url": decoded_url, - "hasPagination": False, - "urlTemplate": None, - "lastPage": pagination_info.get('lastPage') - } - - # Wait for navigation to complete after the click - try: - await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'}) - except Exception as e: - print(f"Navigation timeout: {e}") - - # Get the new URL after clicking - next_page_url = page.url - - # If URL didn't change, pagination might be handled by AJAX - if next_page_url == original_url: - return { - "status": "success", - "url": decoded_url, - "hasPagination": True, - "urlTemplate": "AJAX pagination (URL doesn't change)", - "lastPage": pagination_info.get('lastPage') - } - - print(f"Navigation successful: {original_url} -> {next_page_url}") - - # Analyze the URL structure to determine pagination pattern - url_template = await page.evaluate('''(originalUrl, nextPageUrl) => { - // Helper function to parse URL query parameters - const parseQueryParams = (url) => { - const params = {}; - if (url.includes('?')) { - const queryString = url.split('?')[1].split('#')[0]; - queryString.split('&').forEach(param => { - if (param.includes('=')) { - const [key, value] = param.split('=', 2); - params[key] = value; - } - }); - } - return params; - }; - - // Check for query parameter based pagination - if (nextPageUrl.includes('?')) { - const originalParams = parseQueryParams(originalUrl); - const nextParams = parseQueryParams(nextPageUrl); - - // Find parameters that changed or were added - let paginationParam = null; - - // First check for common pagination parameter names - const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber']; - - for (const key of commonPaginationParams) { - if (key in nextParams && - (!(key in originalParams) || originalParams[key] !== nextParams[key])) { - if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) { - paginationParam = key; - break; - } - } - } - - // If no common parameter found, check all parameters - if (!paginationParam) { - for (const [key, value] of Object.entries(nextParams)) { - // Check if parameter is new or changed - if (!(key in originalParams) || originalParams[key] !== value) { - // Check if the value is numeric and could be a page number - if (/^\d+$/.test(value) && parseInt(value) > 1) { - paginationParam = key; - break; - } - } - } - } - - // If we found a pagination parameter - if (paginationParam) { - const baseUrl = nextPageUrl.split('?')[0]; - - // Reconstruct the URL template with all parameters - const queryParts = []; - for (const [key, value] of Object.entries(nextParams)) { - if (key === paginationParam) { - queryParts.push(`${key}={PAGE_NUMBER}`); - } else { - queryParts.push(`${key}=${value}`); - } - } - - return `${baseUrl}?${queryParts.join('&')}`; - } - } - - // Check for path-based pagination - const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/']; - for (const pattern of pathPatterns) { - if (nextPageUrl.includes(pattern)) { - const parts = nextPageUrl.split(pattern); - let template = `${parts[0]}${pattern}{PAGE_NUMBER}`; - - // Add any suffix after the page number - if (parts.length > 1 && parts[1].includes('/')) { - const suffix = parts[1].split('/', 1)[1]; - if (suffix) { - template += `/${suffix}`; - } - } - - return template; - } - } - - // If we couldn't determine the pattern, try to make an educated guess - // For the specific case where a parameter like current_page=2 is added - const originalUrlObj = new URL(originalUrl); - const nextUrlObj = new URL(nextPageUrl); - - // Check if the paths are the same but query params differ - if (originalUrlObj.pathname === nextUrlObj.pathname) { - const originalParams = parseQueryParams(originalUrl); - const nextParams = parseQueryParams(nextPageUrl); - - // Find parameters that exist in next but not in original - const newParams = Object.keys(nextParams).filter(key => !(key in originalParams)); - - // If there's exactly one new parameter and it has a numeric value - if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) { - const paginationParam = newParams[0]; - const baseUrl = nextPageUrl.split('?')[0]; - - // Reconstruct the URL template - const queryParts = []; - for (const [key, value] of Object.entries(nextParams)) { - if (key === paginationParam) { - queryParts.push(`${key}={PAGE_NUMBER}`); - } else { - queryParts.push(`${key}=${value}`); - } - } - - return `${baseUrl}?${queryParts.join('&')}`; - } - } - - // If we still couldn't determine the pattern, return both URLs as examples - return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`; - }''', original_url, next_page_url) - - # Try to extract last page number from the next page if we didn't find it on the first page - if not pagination_info.get('lastPage'): - last_page_from_next = await page.evaluate('''() => { - // Get all links on the page - const links = Array.from(document.querySelectorAll('a')); - - // Strategy 1: Find numeric links (page numbers) - const numericLinks = links.filter(link => { - const text = link.innerText.trim(); - return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; - }); - - if (numericLinks.length > 0) { - const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); - return Math.max(...numericValues); - } - - // Strategy 2: Analyze all URLs for page numbers - const pageNumbersFromUrls = []; - links.forEach(link => { - if (!link.href || link.href === '#') return; - - // Check for common pagination URL patterns - const patterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/, - /\/pages\/(\d+)/ - ]; - - for (const pattern of patterns) { - const match = link.href.match(pattern); - if (match && match[1]) { - pageNumbersFromUrls.push(parseInt(match[1])); - } - } - }); - - if (pageNumbersFromUrls.length > 0) { - return Math.max(...pageNumbersFromUrls); - } - - return null; - }''') - - if last_page_from_next: - pagination_info['lastPage'] = last_page_from_next - - # Check if the pattern is unclear - has_pagination = True - url_string_template = str(url_template) - if url_string_template and url_string_template.startswith("Pattern unclear"): - has_pagination = False - url_template = None - - result = { - "status": "success", - "url": decoded_url, - "hasPagination": has_pagination, - "urlTemplate": url_template, - "lastPage": pagination_info.get('lastPage'), - "originalUrl": original_url, - "nextPageUrl": next_page_url - } - - return result - - except Exception as e: - print(f"Error during pagination detection: {e}") - return { - "status": "error", - "url": decoded_url, - "error": str(e), - "hasPagination": False, - "urlTemplate": None, - "lastPage": None - } - - # Perform the operation - result = await safe_browser_operation(decoded_url, pagination_operation) - - # Save to cache - save_to_cache(decoded_url, "pagination", result) - - return result + conn.close() + return { + "total_entries": total_count, + "route_counts": route_counts, + "oldest_entry": min_time, + "newest_entry": max_time + } except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/Dockers/puppeteer-api/test_pagination.py b/Dockers/puppeteer-api/test_pagination.py new file mode 100644 index 0000000..0a44678 --- /dev/null +++ b/Dockers/puppeteer-api/test_pagination.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +Test script for the pagination route to verify error handling and browser cleanup. +""" + +import asyncio +import aiohttp +import json +import sys + +async def test_pagination(url, api_key): + """Test the pagination endpoint with a given URL""" + async with aiohttp.ClientSession() as session: + headers = {"X-API-Key": api_key} + + # Test URL with pagination + test_url = f"http://localhost:8000/pagination?url={url}" + + print(f"Testing pagination for: {url}") + + try: + async with session.get(test_url, headers=headers) as response: + if response.status == 200: + result = await response.json() + print(f"✅ Success: {json.dumps(result, indent=2)}") + return True + else: + error_text = await response.text() + print(f"❌ Error {response.status}: {error_text}") + return False + except Exception as e: + print(f"❌ Exception: {e}") + return False + +async def main(): + """Main test function""" + # Test URLs - some with pagination, some without + test_urls = [ + "https://example.com", # No pagination + "https://httpbin.org/get", # No pagination + "https://news.ycombinator.com", # Has pagination + ] + + api_key = "test-key" # Replace with your actual API key + + print("Testing pagination route...") + print("=" * 50) + + for url in test_urls: + success = await test_pagination(url, api_key) + print("-" * 30) + + # Small delay between tests + await asyncio.sleep(1) + + print("Test completed!") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file