From 2b893ca47aa878fc5a6d6172900e3c2b193043b0 Mon Sep 17 00:00:00 2001 From: Bram Kelchtermans Date: Fri, 2 May 2025 19:04:36 +0200 Subject: [PATCH] better pagination detection --- Dockers/puppeteer-api/main.py | 587 ++++++++++++++++++---------------- 1 file changed, 313 insertions(+), 274 deletions(-) diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index b8000a0..4ffc6e3 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -549,251 +549,170 @@ async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)): print(f"Successfully loaded page: {original_url}") - # Before clicking any pagination links, try to determine the last page - # by analyzing all pagination-related links on the current page - last_page = await page.evaluate('''() => { - console.log("Analyzing page for last page number before navigation"); + # Analyze the page for pagination information + pagination_info = await page.evaluate('''() => { + // Find the last page number if available + const findLastPageNumber = () => { + // Get all links on the page + const links = Array.from(document.querySelectorAll('a')); - // Get all links on the page - const links = Array.from(document.querySelectorAll('a')); - console.log(`Found ${links.length} links to analyze`); + // Strategy 1: Find numeric links (page numbers) + const numericLinks = links.filter(link => { + const text = link.innerText.trim(); + return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; + }); - let lastPage = null; - let lastPageSource = ''; - - // Strategy 1: Find numeric links (e.g., page numbers) - const numericLinks = links.filter(link => { - const text = link.innerText.trim(); - return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; - }); - - if (numericLinks.length > 0) { - const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); - const maxNumeric = Math.max(...numericValues); - console.log(`Found highest numeric link: ${maxNumeric}`); - if (maxNumeric > 1) { - lastPage = maxNumeric; - lastPageSource = 'numeric-links'; + if (numericLinks.length > 0) { + const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); + return Math.max(...numericValues); } - } - // Strategy 2: Look for "last page" link - const lastLinks = links.filter(link => { - const text = link.innerText.trim().toLowerCase(); - const classes = (link.className || '').toLowerCase(); - const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase(); + // Strategy 2: Look for "last page" link + const lastLinks = links.filter(link => { + const text = link.innerText.trim().toLowerCase(); + const classes = (link.className || '').toLowerCase(); + const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase(); - return (text === 'last' || - classes.includes('last') || - ariaLabel.includes('last') || - link.getAttribute('rel') === 'last'); - }); + return (text === 'last' || + classes.includes('last') || + ariaLabel.includes('last') || + link.getAttribute('rel') === 'last'); + }); - if (lastLinks.length > 0) { - console.log(`Found ${lastLinks.length} "last" links`); - // Try to extract page number from the URL - const lastLink = lastLinks[0]; - const href = lastLink.href; - console.log(`Last link href: ${href}`); + if (lastLinks.length > 0) { + const lastLink = lastLinks[0]; + const href = lastLink.href; - // Common patterns: page=X, /page/X, etc. - const pagePatterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/ + // Common patterns: page=X, /page/X, etc. + const pagePatterns = [ + /[?&]page=(\d+)/, + /[?&]p=(\d+)/, + /[?&]pg=(\d+)/, + /\/page\/(\d+)/, + /\/p\/(\d+)/, + /\/paged\/(\d+)/ + ]; + + for (const pattern of pagePatterns) { + const match = href.match(pattern); + if (match && match[1]) { + return parseInt(match[1]); + } + } + } + + // Strategy 3: Analyze all URLs for page numbers + const pageNumbersFromUrls = []; + links.forEach(link => { + if (!link.href || link.href === '#') return; + + // Check for common pagination URL patterns + const patterns = [ + /[?&]page=(\d+)/, + /[?&]p=(\d+)/, + /[?&]pg=(\d+)/, + /\/page\/(\d+)/, + /\/p\/(\d+)/, + /\/paged\/(\d+)/, + /\/pages\/(\d+)/ + ]; + + for (const pattern of patterns) { + const match = link.href.match(pattern); + if (match && match[1]) { + pageNumbersFromUrls.push(parseInt(match[1])); + } + } + }); + + if (pageNumbersFromUrls.length > 0) { + return Math.max(...pageNumbersFromUrls); + } + + return null; + }; + + // Find a pagination link to click + const findPaginationLink = () => { + const links = Array.from(document.querySelectorAll('a')); + + // Try to find a page "2" link first (most reliable) + const page2Link = links.find(link => { + const text = link.innerText.trim(); + return text === '2' && link.href && link.href !== '#'; + }); + + if (page2Link) { + return { element: page2Link, href: page2Link.href, type: 'numeric' }; + } + + // Try common "next page" selectors + const nextSelectors = [ + 'a.next', + 'a.page-next', + 'a[rel="next"]', + 'a[aria-label="Next page"]', + 'a[aria-label="next"]' ]; - for (const pattern of pagePatterns) { - const match = href.match(pattern); - if (match && match[1]) { - const pageNum = parseInt(match[1]); - console.log(`Found page number ${pageNum} in last link URL`); - if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) { - lastPage = pageNum; - lastPageSource = 'last-link-url'; - break; - } + for (const selector of nextSelectors) { + const element = document.querySelector(selector); + if (element && element.href && element.href !== '#') { + return { element, href: element.href, type: 'next' }; } } - } - // Strategy 3: Look for pagination text like "Page 1 of 42" - const paginationTexts = []; - document.querySelectorAll('.pagination, .pager, .paginator, .paging, .page-numbers, .pages, .page-navigation') - .forEach(el => paginationTexts.push(el.innerText)); + // Look for any link that might be pagination + const paginationLinks = links.filter(link => { + if (!link.href || link.href === '#') return false; - // Also check for any element that might contain pagination info - document.querySelectorAll('[class*="pag"], [id*="pag"]') - .forEach(el => paginationTexts.push(el.innerText)); + const text = link.innerText.trim(); + const href = link.href; - const pageOfPatterns = [ - /page\s+\d+\s+of\s+(\d+)/i, - /page\s+\d+\s*\/\s*(\d+)/i, - /\d+\s*\/\s*(\d+)\s+pages/i, - /\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i, - /showing\s+\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i, - /\d+\s*-\s*\d+\s+of\s+\d+\s+items\s+\(\s*(\d+)\s+pages\s*\)/i - ]; + // Check for numeric text or next/prev indicators + const isNumeric = /^[0-9]+$/.test(text) && text !== '1'; + const isNextPrev = /next|prev|previous|older|newer/i.test(text) || + /[»«‹›<>]/.test(text); - for (const text of paginationTexts) { - for (const pattern of pageOfPatterns) { - const match = text.match(pattern); - if (match && match[1]) { - const pageNum = parseInt(match[1]); - console.log(`Found page count ${pageNum} in text`); - if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) { - lastPage = pageNum; - lastPageSource = 'pagination-text'; - break; - } - } + // Check for page parameter in URL + const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href); + + return (isNumeric || isNextPrev || hasPageParam); + }); + + if (paginationLinks.length > 0) { + const link = paginationLinks[0]; + return { element: link, href: link.href, type: 'other' }; } - } - // NEW STRATEGY: Analyze all link URLs for page numbers - console.log("Analyzing all link URLs for page numbers"); + return null; + }; - // Common URL patterns that indicate pagination - const urlPagePatterns = [ - /[?&]page=(\d+)/, - /[?&]p=(\d+)/, - /[?&]pg=(\d+)/, - /\/page\/(\d+)/, - /\/p\/(\d+)/, - /\/paged\/(\d+)/, - /\/pages\/(\d+)/, - /[?&]offset=(\d+)/, - /[?&]start=(\d+)/, - /[?&]from=(\d+)/, - /[?&]paged=(\d+)/, - /[?&]pagenum=(\d+)/, - /[?&]pageNumber=(\d+)/, - /[?&]currentpage=(\d+)/ - ]; + const lastPage = findLastPageNumber(); + const paginationLink = findPaginationLink(); - // Extract page numbers from all link URLs - const pageNumbersFromUrls = []; - links.forEach(link => { - if (!link.href || link.href === '#') return; - - const href = link.href; - for (const pattern of urlPagePatterns) { - const match = href.match(pattern); - if (match && match[1]) { - const pageNum = parseInt(match[1]); - if (pageNum > 1) { - pageNumbersFromUrls.push(pageNum); - break; - } - } - } - }); - - if (pageNumbersFromUrls.length > 0) { - const maxPageFromUrls = Math.max(...pageNumbersFromUrls); - console.log(`Found highest page number in URLs: ${maxPageFromUrls} (from ${pageNumbersFromUrls.length} links)`); - if (maxPageFromUrls > 1 && (lastPage === null || maxPageFromUrls > lastPage)) { - lastPage = maxPageFromUrls; - lastPageSource = 'url-analysis'; - } - } - - console.log(`Pre-navigation last page detection: ${lastPage} (source: ${lastPageSource})`); - return { lastPage, lastPageSource }; - }'''); - - print(f"Pre-navigation last page detection: {last_page.get('lastPage')} (source: {last_page.get('lastPageSource')})") - - # Find and click on pagination elements - pagination_clicked = await page.evaluate('''async () => { - // Common pagination link selectors to try - const selectors = [ - 'a.next', 'a.page-next', 'a[rel="next"]', - 'a[aria-label="Next page"]', 'a[aria-label="next"]', - '.pagination a:nth-child(2)', // Often the "2" link - '.pagination li:nth-child(2) a', - 'a:contains("2")', 'a:contains("Next")', 'a:contains("»")' - ]; - - // Try to find a page "2" link first - const links = Array.from(document.querySelectorAll('a')); - const page2Link = links.find(link => { - const text = link.innerText.trim(); - return text === '2' && link.href && link.href !== '#'; - }); - - if (page2Link) { - // Store the href before clicking - const href = page2Link.href; + if (paginationLink) { // Click the link - page2Link.click(); - return { clicked: true, href: href, type: 'numeric' }; + paginationLink.element.click(); + return { + clicked: true, + href: paginationLink.href, + type: paginationLink.type, + lastPage + }; } - // Try common selectors - for (const selector of selectors) { - try { - if (selector.includes(':contains')) { - // Handle jQuery-style :contains selector - const text = selector.match(/:contains\\("(.+)"\\)/)[1]; - const link = links.find(l => l.innerText.includes(text) && l.href && l.href !== '#'); - if (link) { - const href = link.href; - link.click(); - return { clicked: true, href: href, type: 'selector' }; - } - } else { - const element = document.querySelector(selector); - if (element && element.href && element.href !== '#') { - const href = element.href; - element.click(); - return { clicked: true, href: href, type: 'selector' }; - } - } - } catch (e) { - // Ignore errors for invalid selectors - } - } - - // Look for any link that might be pagination - const paginationLinks = links.filter(link => { - if (!link.href || link.href === '#') return false; - - const text = link.innerText.trim(); - const href = link.href; - - // Check for numeric text or next/prev indicators - const isNumeric = /^[0-9]+$/.test(text) && text !== '1'; - const isNextPrev = /next|prev|previous|older|newer/i.test(text) || - /[»«‹›<>]/.test(text); - - // Check for page parameter in URL - const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href); - - return (isNumeric || isNextPrev || hasPageParam); - }); - - if (paginationLinks.length > 0) { - const link = paginationLinks[0]; - const href = link.href; - link.click(); - return { clicked: true, href: href, type: 'other' }; - } - - return { clicked: false }; + return { clicked: false, lastPage }; }''') # If no pagination was found or clicked - if not pagination_clicked.get('clicked', False): + if not pagination_info.get('clicked', False): return { "status": "success", "url": decoded_url, "hasPagination": False, "urlTemplate": None, - "lastPage": last_page.get('lastPage') + "lastPage": pagination_info.get('lastPage') } # Wait for navigation to complete after the click @@ -812,82 +731,202 @@ async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)): "url": decoded_url, "hasPagination": True, "urlTemplate": "AJAX pagination (URL doesn't change)", - "lastPage": last_page.get('lastPage'), - "nextPageUrl": next_page_url, - "originalUrl": original_url, - "expectedHref": pagination_clicked.get('href') + "lastPage": pagination_info.get('lastPage') } print(f"Navigation successful: {original_url} -> {next_page_url}") # Analyze the URL structure to determine pagination pattern - url_template = None + url_template = await page.evaluate('''(originalUrl, nextPageUrl) => { + // Helper function to parse URL query parameters + const parseQueryParams = (url) => { + const params = {}; + if (url.includes('?')) { + const queryString = url.split('?')[1].split('#')[0]; + queryString.split('&').forEach(param => { + if (param.includes('=')) { + const [key, value] = param.split('=', 2); + params[key] = value; + } + }); + } + return params; + }; - # Compare the original and new URLs to find the pagination pattern - if '?' in next_page_url: - # Extract query parameters from both URLs - original_params = {} - if '?' in original_url: - original_query = original_url.split('?')[1].split('#')[0] - for param in original_query.split('&'): - if '=' in param: - key, value = param.split('=', 1) - original_params[key] = value + // Check for query parameter based pagination + if (nextPageUrl.includes('?')) { + const originalParams = parseQueryParams(originalUrl); + const nextParams = parseQueryParams(nextPageUrl); - next_query = next_page_url.split('?')[1].split('#')[0] - next_params = {} - for param in next_query.split('&'): - if '=' in param: - key, value = param.split('=', 1) - next_params[key] = value + // Find parameters that changed or were added + let paginationParam = null; - # Find parameters that changed or were added - pagination_param = None - for key, value in next_params.items(): - # Check if parameter is new or changed - if key not in original_params or original_params[key] != value: - # Check if the value is numeric and could be a page number - if value.isdigit() and int(value) > 1: - pagination_param = key - break + // First check for common pagination parameter names + const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber']; - # If we found a pagination parameter - if pagination_param: - base_url = next_page_url.split('?')[0] + for (const key of commonPaginationParams) { + if (key in nextParams && + (!(key in originalParams) || originalParams[key] !== nextParams[key])) { + if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) { + paginationParam = key; + break; + } + } + } - # Reconstruct the URL template with all parameters - query_parts = [] - for key, value in next_params.items(): - if key == pagination_param: - query_parts.append(f"{key}={{PAGE_NUMBER}}") - else: - query_parts.append(f"{key}={value}") + // If no common parameter found, check all parameters + if (!paginationParam) { + for (const [key, value] of Object.entries(nextParams)) { + // Check if parameter is new or changed + if (!(key in originalParams) || originalParams[key] !== value) { + // Check if the value is numeric and could be a page number + if (/^\d+$/.test(value) && parseInt(value) > 1) { + paginationParam = key; + break; + } + } + } + } - url_template = f"{base_url}?{'&'.join(query_parts)}" + // If we found a pagination parameter + if (paginationParam) { + const baseUrl = nextPageUrl.split('?')[0]; - # If we couldn't determine from query parameters, check for path-based pagination - if not url_template: - # Path-based pagination patterns - path_patterns = ['/page/', '/p/', '/paged/', '/pages/'] - for pattern in path_patterns: - if pattern in next_page_url: - parts = next_page_url.split(pattern) - url_template = f"{parts[0]}{pattern}{{PAGE_NUMBER}}" - if len(parts) > 1 and '/' in parts[1]: - suffix = parts[1].split('/', 1)[1] - if suffix: - url_template += f"/{suffix}" - break + // Reconstruct the URL template with all parameters + const queryParts = []; + for (const [key, value] of Object.entries(nextParams)) { + if (key === paginationParam) { + queryParts.push(`${key}={PAGE_NUMBER}`); + } else { + queryParts.push(`${key}=${value}`); + } + } - # If we still couldn't determine the pattern, use the original and next URLs as examples - if not url_template: - url_template = f"Pattern unclear. Example: {original_url} → {next_page_url}" + return `${baseUrl}?${queryParts.join('&')}`; + } + } + + // Check for path-based pagination + const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/']; + for (const pattern of pathPatterns) { + if (nextPageUrl.includes(pattern)) { + const parts = nextPageUrl.split(pattern); + let template = `${parts[0]}${pattern}{PAGE_NUMBER}`; + + // Add any suffix after the page number + if (parts.length > 1 && parts[1].includes('/')) { + const suffix = parts[1].split('/', 1)[1]; + if (suffix) { + template += `/${suffix}`; + } + } + + return template; + } + } + + // If we couldn't determine the pattern, try to make an educated guess + // For the specific case where a parameter like current_page=2 is added + const originalUrlObj = new URL(originalUrl); + const nextUrlObj = new URL(nextPageUrl); + + // Check if the paths are the same but query params differ + if (originalUrlObj.pathname === nextUrlObj.pathname) { + const originalParams = parseQueryParams(originalUrl); + const nextParams = parseQueryParams(nextPageUrl); + + // Find parameters that exist in next but not in original + const newParams = Object.keys(nextParams).filter(key => !(key in originalParams)); + + // If there's exactly one new parameter and it has a numeric value + if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) { + const paginationParam = newParams[0]; + const baseUrl = nextPageUrl.split('?')[0]; + + // Reconstruct the URL template + const queryParts = []; + for (const [key, value] of Object.entries(nextParams)) { + if (key === paginationParam) { + queryParts.push(`${key}={PAGE_NUMBER}`); + } else { + queryParts.push(`${key}=${value}`); + } + } + + return `${baseUrl}?${queryParts.join('&')}`; + } + } + + // If we still couldn't determine the pattern, return both URLs as examples + return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`; + }''', original_url, next_page_url) + + # Try to extract last page number from the next page if we didn't find it on the first page + if not pagination_info.get('lastPage'): + last_page_from_next = await page.evaluate('''() => { + // Get all links on the page + const links = Array.from(document.querySelectorAll('a')); + + // Strategy 1: Find numeric links (page numbers) + const numericLinks = links.filter(link => { + const text = link.innerText.trim(); + return /^[0-9]+$/.test(text) && link.href && link.href !== '#'; + }); + + if (numericLinks.length > 0) { + const numericValues = numericLinks.map(link => parseInt(link.innerText.trim())); + return Math.max(...numericValues); + } + + // Strategy 2: Analyze all URLs for page numbers + const pageNumbersFromUrls = []; + links.forEach(link => { + if (!link.href || link.href === '#') return; + + // Check for common pagination URL patterns + const patterns = [ + /[?&]page=(\d+)/, + /[?&]p=(\d+)/, + /[?&]pg=(\d+)/, + /\/page\/(\d+)/, + /\/p\/(\d+)/, + /\/paged\/(\d+)/, + /\/pages\/(\d+)/ + ]; + + for (const pattern of patterns) { + const match = link.href.match(pattern); + if (match && match[1]) { + pageNumbersFromUrls.push(parseInt(match[1])); + } + } + }); + + if (pageNumbersFromUrls.length > 0) { + return Math.max(...pageNumbersFromUrls); + } + + return null; + }''') + + if last_page_from_next: + pagination_info['lastPage'] = last_page_from_next + + # Check if the pattern is unclear + has_pagination = True + url_string_template = str(url_template) + if url_string_template and url_string_template.startswith("Pattern unclear"): + has_pagination = False + url_template = None result = { "status": "success", - "hasPagination": True, + "url": decoded_url, + "hasPagination": has_pagination, "urlTemplate": url_template, - "lastPage": last_page.get('lastPage') + "lastPage": pagination_info.get('lastPage'), + "originalUrl": original_url, + "nextPageUrl": next_page_url } return result