From 707ad9457d979c3ad0f953187de39ad9de2a2d8b Mon Sep 17 00:00:00 2001 From: Bram Kelchtermans Date: Tue, 6 May 2025 12:29:37 +0200 Subject: [PATCH] better pagination detection --- Dockers/puppeteer-api/app/services/browser.py | 691 +++++++++--------- 1 file changed, 339 insertions(+), 352 deletions(-) diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 4efab20..874e5fb 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -1,5 +1,5 @@ from app.utils.browser_utils import safe_browser_operation - +import asyncio async def visit_url_service(decoded_url): """Service function to visit a URL and get its content""" print(f"Visiting URL: {decoded_url}") @@ -165,386 +165,374 @@ async def detect_pagination_service(decoded_url): await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000) original_url = page.url - print(f"Successfully loaded page: {original_url}") - - # Analyze the page for pagination information - pagination_info = await page.evaluate('''() => { - // Find the last page number if available - const findLastPageNumber = () => { - // Get all links on the page - const links = Array.from(document.querySelectorAll('a')); - - // Strategy 1: Find numeric links (page numbers) - const numericLinks = links.filter(link => { - const text = link.innerText.trim(); - return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; - }); - - if (numericLinks.length > 0) { - const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); - return Math.max(...numericValues); - } - - // Strategy 2: Look for "last page" link - const lastLinks = links.filter(link => { - const text = link.innerText.trim().toLowerCase(); - const classes = (link.className || '').toLowerCase(); - const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase(); - - return (text === 'last' || - classes.includes('last') || - ariaLabel.includes('last') || - link.getAttribute('rel') === 'last'); - }); - - if (lastLinks.length > 0) { - const lastLink = lastLinks[0]; - const href = lastLink.href; - - // Common patterns: page=X, /page/X, etc. - const pagePatterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/ - ]; - - for (const pattern of pagePatterns) { - const match = href.match(pattern); - if (match && match[1]) { - return parseInt(match[1]); - } - } - } - - // Strategy 3: Analyze all URLs for page numbers - const pageNumbersFromUrls = []; - links.forEach(link => { - if (!link.href || link.href === '#') return; - - // Check for common pagination URL patterns - const patterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/, - /\/pages\/(\d+)/ - ]; - - for (const pattern of patterns) { - const match = link.href.match(pattern); - if (match && match[1]) { - pageNumbersFromUrls.push(parseInt(match[1])); - } - } - }); - - if (pageNumbersFromUrls.length > 0) { - return Math.max(...pageNumbersFromUrls); - } - - return null; + # Check for common pagination indicators + pagination_data = await page.evaluate('''() => { + const data = { + hasPagination: false, + paginationType: null, + paginationElements: [], + detectedParameter: null, + lastPageNumber: null }; - // Find a pagination link to click - const findPaginationLink = () => { - const links = Array.from(document.querySelectorAll('a')); + // Look for numbered pagination links (1, 2, 3...) + const numberedLinks = Array.from(document.querySelectorAll('a, button, span')) + .filter(el => { + const text = el.innerText.trim(); - // Try to find a page "2" link first (most reliable) - const page2Link = links.find(link => { - const text = link.innerText.trim(); - return text === '2' && link.href && link.href !== '#'; - }); - - if (page2Link) { - return { element: page2Link, href: page2Link.href, type: 'numeric' }; - } - - // Try common "next page" selectors - const nextSelectors = [ - 'a.next', - 'a.page-next', - 'a[rel="next"]', - 'a[aria-label="Next page"]', - 'a[aria-label="next"]' - ]; - - for (const selector of nextSelectors) { - const element = document.querySelector(selector); - if (element && element.href && element.href !== '#') { - return { element, href: element.href, type: 'next' }; + // check for data-page attribute + const dataPage = el.getAttribute('data-page'); + if (dataPage) { + return /^[0-9]+$/.test(dataPage); } - } - // Look for any link that might be pagination - const paginationLinks = links.filter(link => { - if (!link.href || link.href === '#') return false; - - const text = link.innerText.trim(); - const href = link.href; - - // Check for numeric text or next/prev indicators - const isNumeric = /^[0-9]+$/.test(text) && text !== '1'; - const isNextPrev = /next|prev|previous|older|newer/i.test(text) || - /[»«‹›<>]/.test(text); - - // Check for page parameter in URL - const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href); - - return (isNumeric || isNextPrev || hasPageParam); + return /^[0-9]+$/.test(text) && + (el.tagName === 'A' || el.onclick || + el.closest('button, [role="button"]')); }); - if (paginationLinks.length > 0) { - const link = paginationLinks[0]; - return { element: link, href: link.href, type: 'other' }; + console.log(numberedLinks); + + // Look for next/prev buttons + const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]')) + .filter(el => { + const text = el.innerText.trim().toLowerCase(); + const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; + const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]'); + + return text.includes('next') || + text.includes('›') || + text.includes('»') || + text.includes('→') || + ariaLabel.includes('next') || + hasNextIcon; + }); + + // Check for pagination containers + const paginationContainers = Array.from(document.querySelectorAll( + '.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]' + )); + + // Collect all potential pagination elements + if (numberedLinks.length > 0) { + data.hasPagination = true; + data.paginationType = 'numbered'; + + // Get href attributes or other identifiers from numbered links + data.paginationElements = numberedLinks.slice(0, 5).map(el => { + return { + text: el.innerText.trim(), + dataPage: el.getAttribute('data-page'), + href: el.tagName === 'A' ? el.href : null, + classes: el.className, + id: el.id + }; + }); + + // Try to find the last page number + const pageNumbers = numberedLinks + .map(el => parseInt(el.innerText.trim())) + .filter(num => !isNaN(num)); + + if (pageNumbers.length > 0) { + data.lastPageNumber = Math.max(...pageNumbers); } - return null; - }; - - const lastPage = findLastPageNumber(); - const paginationLink = findPaginationLink(); - - if (paginationLink) { - // Click the link - paginationLink.element.click(); - return { - clicked: true, - href: paginationLink.href, - type: paginationLink.type, - lastPage - }; - } - - return { clicked: false, lastPage }; - }''') - - # If no pagination was found or clicked - if not pagination_info.get('clicked', False): - return { - "status": "success", - "url": decoded_url, - "hasPagination": False, - "urlTemplate": None, - "lastPage": pagination_info.get('lastPage') - } - - # Wait for navigation to complete after the click - try: - await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'}) - except Exception as e: - print(f"Navigation timeout: {e}") - - # Get the new URL after clicking - next_page_url = page.url - - # If URL didn't change, pagination might be handled by AJAX - if next_page_url == original_url: - return { - "status": "success", - "url": decoded_url, - "hasPagination": True, - "urlTemplate": "AJAX pagination (URL doesn't change)", - "lastPage": pagination_info.get('lastPage') - } - - print(f"Navigation successful: {original_url} -> {next_page_url}") - - # Analyze the URL structure to determine pagination pattern - url_template = await page.evaluate('''(originalUrl, nextPageUrl) => { - // Helper function to parse URL query parameters - const parseQueryParams = (url) => { - const params = {}; - if (url.includes('?')) { - const queryString = url.split('?')[1].split('#')[0]; - queryString.split('&').forEach(param => { - if (param.includes('=')) { - const [key, value] = param.split('=', 2); - params[key] = value; - } + // Also look for a "last page" element that might have text like "Last" or "»" + const lastPageElement = Array.from(document.querySelectorAll('a, button')) + .find(el => { + const text = el.innerText.trim().toLowerCase(); + const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; + return text.includes('last') || + text === '»' || + ariaLabel.includes('last page'); }); - } - return params; - }; - // Check for query parameter based pagination - if (nextPageUrl.includes('?')) { - const originalParams = parseQueryParams(originalUrl); - const nextParams = parseQueryParams(nextPageUrl); + if (lastPageElement && lastPageElement.href) { + // Try to extract page number from the URL + try { + const url = new URL(lastPageElement.href); + // Check common pagination parameters + ['page', 'p', 'pg'].forEach(param => { + if (url.searchParams.has(param)) { + const value = parseInt(url.searchParams.get(param)); + if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { + data.lastPageNumber = value; + } + } + }); - // Find parameters that changed or were added - let paginationParam = null; - - // First check for common pagination parameter names - const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber']; - - for (const key of commonPaginationParams) { - if (key in nextParams && - (!(key in originalParams) || originalParams[key] !== nextParams[key])) { - if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) { - paginationParam = key; - break; - } - } - } - - // If no common parameter found, check all parameters - if (!paginationParam) { - for (const [key, value] of Object.entries(nextParams)) { - // Check if parameter is new or changed - if (!(key in originalParams) || originalParams[key] !== value) { - // Check if the value is numeric and could be a page number - if (/^\d+$/.test(value) && parseInt(value) > 1) { - paginationParam = key; - break; + // Check for path-based pagination (like /page/10) + const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i); + if (pathMatch && pathMatch[1]) { + const value = parseInt(pathMatch[1]); + if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { + data.lastPageNumber = value; } } + } catch (e) { + console.error("Error parsing last page URL:", e); } } + } else if (nextButtons.length > 0) { + data.hasPagination = true; + data.paginationType = 'next-prev'; - // If we found a pagination parameter - if (paginationParam) { - const baseUrl = nextPageUrl.split('?')[0]; - - // Reconstruct the URL template with all parameters - const queryParts = []; - for (const [key, value] of Object.entries(nextParams)) { - if (key === paginationParam) { - queryParts.push(`${key}={PAGE_NUMBER}`); - } else { - queryParts.push(`${key}=${value}`); - } - } - - return `${baseUrl}?${queryParts.join('&')}`; - } - } - - // Check for path-based pagination - const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/']; - for (const pattern of pathPatterns) { - if (nextPageUrl.includes(pattern)) { - const parts = nextPageUrl.split(pattern); - let template = `${parts[0]}${pattern}{PAGE_NUMBER}`; - - // Add any suffix after the page number - if (parts.length > 1 && parts[1].includes('/')) { - const suffix = parts[1].split('/', 1)[1]; - if (suffix) { - template += `/${suffix}`; - } - } - - return template; - } - } - - // If we couldn't determine the pattern, try to make an educated guess - // For the specific case where a parameter like current_page=2 is added - const originalUrlObj = new URL(originalUrl); - const nextUrlObj = new URL(nextPageUrl); - - // Check if the paths are the same but query params differ - if (originalUrlObj.pathname === nextUrlObj.pathname) { - const originalParams = parseQueryParams(originalUrl); - const nextParams = parseQueryParams(nextPageUrl); - - // Find parameters that exist in next but not in original - const newParams = Object.keys(nextParams).filter(key => !(key in originalParams)); - - // If there's exactly one new parameter and it has a numeric value - if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) { - const paginationParam = newParams[0]; - const baseUrl = nextPageUrl.split('?')[0]; - - // Reconstruct the URL template - const queryParts = []; - for (const [key, value] of Object.entries(nextParams)) { - if (key === paginationParam) { - queryParts.push(`${key}={PAGE_NUMBER}`); - } else { - queryParts.push(`${key}=${value}`); - } - } - - return `${baseUrl}?${queryParts.join('&')}`; - } - } - - // If we still couldn't determine the pattern, return both URLs as examples - return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`; - }''', original_url, next_page_url) - - # Try to extract last page number from the next page if we didn't find it on the first page - if not pagination_info.get('lastPage'): - last_page_from_next = await page.evaluate('''() => { - // Get all links on the page - const links = Array.from(document.querySelectorAll('a')); - - // Strategy 1: Find numeric links (page numbers) - const numericLinks = links.filter(link => { - const text = link.innerText.trim(); - return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; + // Get information about next buttons + data.paginationElements = nextButtons.slice(0, 3).map(el => { + return { + text: el.innerText.trim(), + href: el.tagName === 'A' ? el.href : null, + classes: el.className, + id: el.id + }; }); + } - if (numericLinks.length > 0) { - const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); - return Math.max(...numericValues); + // Check for URL parameters that might indicate pagination + const currentUrl = window.location.href; + const urlParams = new URL(currentUrl).searchParams; + + // Common pagination parameters + const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit']; + + for (const param of paginationParams) { + if (urlParams.has(param)) { + data.detectedParameter = { + name: param, + value: urlParams.get(param) + }; + break; } + } - // Strategy 2: Analyze all URLs for page numbers - const pageNumbersFromUrls = []; - links.forEach(link => { - if (!link.href || link.href === '#') return; + return data; + }''') - // Check for common pagination URL patterns - const patterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/, - /\/pages\/(\d+)/ - ]; + # If pagination is detected, try to navigate to the next page by clicking + next_page_url = None + pagination_parameter = None + url_template = None - for (const pattern of patterns) { - const match = link.href.match(pattern); - if (match && match[1]) { - pageNumbersFromUrls.push(parseInt(match[1])); - } + if pagination_data['hasPagination']: + print("Pagination detected, attempting to click on a pagination element") + + # Always try to click on a pagination element, regardless of type + clicked = False + + # First try to click on a numbered link (preferably "2" if we're on page 1) + try: + clicked = await page.evaluate('''() => { + // First try to find and click on a "2" link or button + const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]')) + .filter(el => { + // Check for text content "2" + if (el.innerText.trim() === '2') { + return true; + } + + // Check for href with page=2 or similar (for anchor elements) + if (el.tagName === 'A' && el.href) { + try { + const url = new URL(el.href, window.location.origin); + const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', + 'currentpage', 'pagenum', 'pageNumber', 'paged']; + + for (const param of paginationParams) { + if (url.searchParams.has(param) && url.searchParams.get(param) === '2') { + return true; + } + } + + // Check for path-based pagination like /page/2/ + const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i); + if (pathMatch) { + return true; + } + } catch (e) {} + } + + // Check for data attributes that might indicate pagination + if (el.getAttribute('data-page') === '2' || + el.getAttribute('data-pagenumber') === '2' || + el.getAttribute('data-page-number') === '2') { + return true; + } + + return false; + }); + + if (page2Elements.length > 0) { + console.log("Clicking on page 2 element"); + page2Elements[0].click(); + return true; } - }); - if (pageNumbersFromUrls.length > 0) { - return Math.max(...pageNumbersFromUrls); + // If no "2" link found, try any numbered link or button + const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]')) + .filter(el => /^\d+$/.test(el.innerText.trim())); + + if (numberedElements.length > 0) { + // Sort by number and get the second one (likely page 2) + const sorted = numberedElements.sort((a, b) => { + return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim()); + }); + + // Get the second element if available (page 2), otherwise the first one + const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0]; + console.log("Clicking on numbered element: " + elementToClick.innerText); + elementToClick.click(); + return true; + } + + // If no numbered links, try next button + const nextTexts = ['next', '›', '»', '→']; + const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]')) + .filter(el => { + const text = el.textContent.trim().toLowerCase(); + const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || ''; + return nextTexts.some(t => text.includes(t)) || + ariaLabel.includes('next') || + el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]'); + }); + + if (nextElements.length > 0) { + console.log("Clicking on next button"); + nextElements[0].click(); + return true; + } + + return false; + }''') + + if clicked: + print("Successfully clicked on pagination element") + # Wait for navigation to complete + await asyncio.sleep(1) + next_page_url = page.url + else: + print("No clickable pagination element found") + + except Exception as e: + print(f"Error clicking on pagination element: {e}") + + # If we successfully navigated to the next page, analyze the URL difference + if next_page_url: + print('Searching for pagination parameter') + # Parse both URLs + original_parsed = await page.evaluate(f'''(originalUrl) => {{ + const original = new URL(originalUrl); + const current = new URL(window.location.href); + + // Check for differences in query parameters + let paramDiff = null; + + // Common pagination parameters to check + const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']; + + for (const param of paginationParams) {{ + const originalValue = original.searchParams.get(param); + const currentValue = current.searchParams.get(param); + + if (originalValue !== currentValue && currentValue !== null) {{ + paramDiff = {{ + name: param, + originalValue: originalValue, + currentValue: currentValue + }}; + break; + }} + }} + + // Check for path differences (like /page/1 vs /page/2) + const originalPath = original.pathname; + const currentPath = current.pathname; + + let pathDiff = null; + if (originalPath !== currentPath) {{ + const originalSegments = originalPath.split('/').filter(s => s); + const currentSegments = currentPath.split('/').filter(s => s); + + // Find the segment that changed + if (originalSegments.length === currentSegments.length) {{ + for (let i = 0; i < originalSegments.length; i++) {{ + if (originalSegments[i] !== currentSegments[i]) {{ + // Check if the difference is numeric + if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{ + pathDiff = {{ + index: i, + originalValue: originalSegments[i], + currentValue: currentSegments[i] + }}; + }} + }} + }} + }} + }} + + return {{ + paramDiff, + pathDiff, + originalUrl: originalUrl, + currentUrl: window.location.href + }}; + }}''', original_url) + + # Determine the pagination parameter and create URL template + if original_parsed['paramDiff']: + param_name = original_parsed['paramDiff']['name'] + pagination_parameter = { + 'type': 'query', + 'name': param_name, + 'value': original_parsed['paramDiff']['currentValue'] } - return null; - }''') + # Create URL template for query parameter + url_obj = await page.evaluate(f'''(url, paramName) => {{ + const urlObj = new URL(url); + urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); + return urlObj.toString(); + }}''', original_url, param_name) - if last_page_from_next: - pagination_info['lastPage'] = last_page_from_next + url_template = url_obj - # Check if the pattern is unclear - has_pagination = True - url_string_template = str(url_template) - if url_string_template and (url_string_template.startswith("Pattern unclear") or url_string_template.startswith("AJAX pagination")): - has_pagination = False - url_template = None + elif original_parsed['pathDiff']: + path_index = original_parsed['pathDiff']['index'] + pagination_parameter = { + 'type': 'path', + 'index': path_index, + 'value': original_parsed['pathDiff']['currentValue'] + } + # Create URL template for path parameter + url_template = await page.evaluate(f'''(url, pathIndex) => {{ + const urlObj = new URL(url); + const pathSegments = urlObj.pathname.split('/').filter(s => s); + pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; + urlObj.pathname = '/' + pathSegments.join('/'); + return urlObj.toString(); + }}''', original_url, path_index) + + if url_template: + # Decode URL-encoded characters in the template + import urllib.parse + url_template = urllib.parse.unquote(url_template) + + # If we couldn't determine the URL template from navigation, try to infer it + if not url_template and pagination_data['detectedParameter']: + param_name = pagination_data['detectedParameter']['name'] + url_template = await page.evaluate(f'''(url, paramName) => {{ + const urlObj = new URL(url); + urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); + return urlObj.toString(); + }}''', original_url, param_name) + + # Return the pagination detection results with a simplified structure result = { "status": "success", - "url": decoded_url, - "hasPagination": has_pagination, + "hasPagination": pagination_data['hasPagination'], "urlTemplate": url_template, - "lastPage": pagination_info.get('lastPage'), - "originalUrl": original_url, - "nextPageUrl": next_page_url + "lastPage": pagination_data['lastPageNumber'] } return result @@ -553,12 +541,11 @@ async def detect_pagination_service(decoded_url): print(f"Error during pagination detection: {e}") return { "status": "error", - "url": decoded_url, - "error": str(e), "hasPagination": False, "urlTemplate": None, - "lastPage": None + "lastPage": None, + "error": str(e) } # Perform the operation - return await safe_browser_operation(decoded_url, pagination_operation) \ No newline at end of file + return await safe_browser_operation(decoded_url, pagination_operation)