From e452511de032d50af908e97e37cb3b4b56efb2ed Mon Sep 17 00:00:00 2001 From: Bram Kelchtermans Date: Fri, 2 May 2025 13:08:12 +0200 Subject: [PATCH] add pagination route --- Dockers/puppeteer-api/main.py | 392 ++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index 23ec808..b8000a0 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -522,6 +522,398 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)): } } +@app.get("/pagination") +async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)): + """Detect pagination on a website and determine the pagination pattern""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + # Decode URL if it's encoded + decoded_url = unquote(url) + + # Check cache first + cached_result = get_cached_data(decoded_url, "pagination") + if cached_result: + return cached_result + + try: + print(f"Detecting pagination on: {decoded_url}") + + # Define the operation to perform with the browser + async def pagination_operation(page): + try: + # Navigate to the URL + await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000) + original_url = page.url + + print(f"Successfully loaded page: {original_url}") + + # Before clicking any pagination links, try to determine the last page + # by analyzing all pagination-related links on the current page + last_page = await page.evaluate('''() => { + console.log("Analyzing page for last page number before navigation"); + + // Get all links on the page + const links = Array.from(document.querySelectorAll('a')); + console.log(`Found ${links.length} links to analyze`); + + let lastPage = null; + let lastPageSource = ''; + + // Strategy 1: Find numeric links (e.g., page numbers) + const numericLinks = links.filter(link => { + const text = link.innerText.trim(); + return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; + }); + + if (numericLinks.length > 0) { + const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); + const maxNumeric = Math.max(...numericValues); + console.log(`Found highest numeric link: ${maxNumeric}`); + if (maxNumeric > 1) { + lastPage = maxNumeric; + lastPageSource = 'numeric-links'; + } + } + + // Strategy 2: Look for "last page" link + const lastLinks = links.filter(link => { + const text = link.innerText.trim().toLowerCase(); + const classes = (link.className || '').toLowerCase(); + const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase(); + + return (text === 'last' || + classes.includes('last') || + ariaLabel.includes('last') || + link.getAttribute('rel') === 'last'); + }); + + if (lastLinks.length > 0) { + console.log(`Found ${lastLinks.length} "last" links`); + // Try to extract page number from the URL + const lastLink = lastLinks[0]; + const href = lastLink.href; + console.log(`Last link href: ${href}`); + + // Common patterns: page=X, /page/X, etc. + const pagePatterns = [ + /[?&]page=(\d+)/, + /[?&]p=(\d+)/, + /[?&]pg=(\d+)/, + /\/page\/(\d+)/, + /\/p\/(\d+)/, + /\/paged\/(\d+)/ + ]; + + for (const pattern of pagePatterns) { + const match = href.match(pattern); + if (match && match[1]) { + const pageNum = parseInt(match[1]); + console.log(`Found page number ${pageNum} in last link URL`); + if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) { + lastPage = pageNum; + lastPageSource = 'last-link-url'; + break; + } + } + } + } + + // Strategy 3: Look for pagination text like "Page 1 of 42" + const paginationTexts = []; + document.querySelectorAll('.pagination, .pager, .paginator, .paging, .page-numbers, .pages, .page-navigation') + .forEach(el => paginationTexts.push(el.innerText)); + + // Also check for any element that might contain pagination info + document.querySelectorAll('[class*="pag"], [id*="pag"]') + .forEach(el => paginationTexts.push(el.innerText)); + + const pageOfPatterns = [ + /page\s+\d+\s+of\s+(\d+)/i, + /page\s+\d+\s*\/\s*(\d+)/i, + /\d+\s*\/\s*(\d+)\s+pages/i, + /\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i, + /showing\s+\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i, + /\d+\s*-\s*\d+\s+of\s+\d+\s+items\s+\(\s*(\d+)\s+pages\s*\)/i + ]; + + for (const text of paginationTexts) { + for (const pattern of pageOfPatterns) { + const match = text.match(pattern); + if (match && match[1]) { + const pageNum = parseInt(match[1]); + console.log(`Found page count ${pageNum} in text`); + if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) { + lastPage = pageNum; + lastPageSource = 'pagination-text'; + break; + } + } + } + } + + // NEW STRATEGY: Analyze all link URLs for page numbers + console.log("Analyzing all link URLs for page numbers"); + + // Common URL patterns that indicate pagination + const urlPagePatterns = [ + /[?&]page=(\d+)/, + /[?&]p=(\d+)/, + /[?&]pg=(\d+)/, + /\/page\/(\d+)/, + /\/p\/(\d+)/, + /\/paged\/(\d+)/, + /\/pages\/(\d+)/, + /[?&]offset=(\d+)/, + /[?&]start=(\d+)/, + /[?&]from=(\d+)/, + /[?&]paged=(\d+)/, + /[?&]pagenum=(\d+)/, + /[?&]pageNumber=(\d+)/, + /[?&]currentpage=(\d+)/ + ]; + + // Extract page numbers from all link URLs + const pageNumbersFromUrls = []; + links.forEach(link => { + if (!link.href || link.href === '#') return; + + const href = link.href; + for (const pattern of urlPagePatterns) { + const match = href.match(pattern); + if (match && match[1]) { + const pageNum = parseInt(match[1]); + if (pageNum > 1) { + pageNumbersFromUrls.push(pageNum); + break; + } + } + } + }); + + if (pageNumbersFromUrls.length > 0) { + const maxPageFromUrls = Math.max(...pageNumbersFromUrls); + console.log(`Found highest page number in URLs: ${maxPageFromUrls} (from ${pageNumbersFromUrls.length} links)`); + if (maxPageFromUrls > 1 && (lastPage === null || maxPageFromUrls > lastPage)) { + lastPage = maxPageFromUrls; + lastPageSource = 'url-analysis'; + } + } + + console.log(`Pre-navigation last page detection: ${lastPage} (source: ${lastPageSource})`); + return { lastPage, lastPageSource }; + }'''); + + print(f"Pre-navigation last page detection: {last_page.get('lastPage')} (source: {last_page.get('lastPageSource')})") + + # Find and click on pagination elements + pagination_clicked = await page.evaluate('''async () => { + // Common pagination link selectors to try + const selectors = [ + 'a.next', 'a.page-next', 'a[rel="next"]', + 'a[aria-label="Next page"]', 'a[aria-label="next"]', + '.pagination a:nth-child(2)', // Often the "2" link + '.pagination li:nth-child(2) a', + 'a:contains("2")', 'a:contains("Next")', 'a:contains("»")' + ]; + + // Try to find a page "2" link first + const links = Array.from(document.querySelectorAll('a')); + const page2Link = links.find(link => { + const text = link.innerText.trim(); + return text === '2' && link.href && link.href !== '#'; + }); + + if (page2Link) { + // Store the href before clicking + const href = page2Link.href; + // Click the link + page2Link.click(); + return { clicked: true, href: href, type: 'numeric' }; + } + + // Try common selectors + for (const selector of selectors) { + try { + if (selector.includes(':contains')) { + // Handle jQuery-style :contains selector + const text = selector.match(/:contains\\("(.+)"\\)/)[1]; + const link = links.find(l => l.innerText.includes(text) && l.href && l.href !== '#'); + if (link) { + const href = link.href; + link.click(); + return { clicked: true, href: href, type: 'selector' }; + } + } else { + const element = document.querySelector(selector); + if (element && element.href && element.href !== '#') { + const href = element.href; + element.click(); + return { clicked: true, href: href, type: 'selector' }; + } + } + } catch (e) { + // Ignore errors for invalid selectors + } + } + + // Look for any link that might be pagination + const paginationLinks = links.filter(link => { + if (!link.href || link.href === '#') return false; + + const text = link.innerText.trim(); + const href = link.href; + + // Check for numeric text or next/prev indicators + const isNumeric = /^[0-9]+$/.test(text) && text !== '1'; + const isNextPrev = /next|prev|previous|older|newer/i.test(text) || + /[»«‹›<>]/.test(text); + + // Check for page parameter in URL + const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href); + + return (isNumeric || isNextPrev || hasPageParam); + }); + + if (paginationLinks.length > 0) { + const link = paginationLinks[0]; + const href = link.href; + link.click(); + return { clicked: true, href: href, type: 'other' }; + } + + return { clicked: false }; + }''') + + # If no pagination was found or clicked + if not pagination_clicked.get('clicked', False): + return { + "status": "success", + "url": decoded_url, + "hasPagination": False, + "urlTemplate": None, + "lastPage": last_page.get('lastPage') + } + + # Wait for navigation to complete after the click + try: + await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'}) + except Exception as e: + print(f"Navigation timeout: {e}") + + # Get the new URL after clicking + next_page_url = page.url + + # If URL didn't change, pagination might be handled by AJAX + if next_page_url == original_url: + return { + "status": "success", + "url": decoded_url, + "hasPagination": True, + "urlTemplate": "AJAX pagination (URL doesn't change)", + "lastPage": last_page.get('lastPage'), + "nextPageUrl": next_page_url, + "originalUrl": original_url, + "expectedHref": pagination_clicked.get('href') + } + + print(f"Navigation successful: {original_url} -> {next_page_url}") + + # Analyze the URL structure to determine pagination pattern + url_template = None + + # Compare the original and new URLs to find the pagination pattern + if '?' in next_page_url: + # Extract query parameters from both URLs + original_params = {} + if '?' in original_url: + original_query = original_url.split('?')[1].split('#')[0] + for param in original_query.split('&'): + if '=' in param: + key, value = param.split('=', 1) + original_params[key] = value + + next_query = next_page_url.split('?')[1].split('#')[0] + next_params = {} + for param in next_query.split('&'): + if '=' in param: + key, value = param.split('=', 1) + next_params[key] = value + + # Find parameters that changed or were added + pagination_param = None + for key, value in next_params.items(): + # Check if parameter is new or changed + if key not in original_params or original_params[key] != value: + # Check if the value is numeric and could be a page number + if value.isdigit() and int(value) > 1: + pagination_param = key + break + + # If we found a pagination parameter + if pagination_param: + base_url = next_page_url.split('?')[0] + + # Reconstruct the URL template with all parameters + query_parts = [] + for key, value in next_params.items(): + if key == pagination_param: + query_parts.append(f"{key}={{PAGE_NUMBER}}") + else: + query_parts.append(f"{key}={value}") + + url_template = f"{base_url}?{'&'.join(query_parts)}" + + # If we couldn't determine from query parameters, check for path-based pagination + if not url_template: + # Path-based pagination patterns + path_patterns = ['/page/', '/p/', '/paged/', '/pages/'] + for pattern in path_patterns: + if pattern in next_page_url: + parts = next_page_url.split(pattern) + url_template = f"{parts[0]}{pattern}{{PAGE_NUMBER}}" + if len(parts) > 1 and '/' in parts[1]: + suffix = parts[1].split('/', 1)[1] + if suffix: + url_template += f"/{suffix}" + break + + # If we still couldn't determine the pattern, use the original and next URLs as examples + if not url_template: + url_template = f"Pattern unclear. Example: {original_url} → {next_page_url}" + + result = { + "status": "success", + "hasPagination": True, + "urlTemplate": url_template, + "lastPage": last_page.get('lastPage') + } + + return result + + except Exception as e: + print(f"Error during pagination detection: {e}") + return { + "status": "error", + "url": decoded_url, + "error": str(e), + "hasPagination": False, + "urlTemplate": None, + "lastPage": None + } + + # Perform the operation + result = await safe_browser_operation(decoded_url, pagination_operation) + + # Save to cache + save_to_cache(decoded_url, "pagination", result) + + return result + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000)