Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

better pagination detection
Build and Push Docker Images / build-and-push (push) Successful in 20s
Details

Browse Source
This commit is contained in:
Bram
2025-05-02 19:04:36 +02:00
parent e452511de0
commit 2b893ca47a
1 changed files with 313 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/main.py
+313 -274
View File
@@ -549,251 +549,170 @@ async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)):
print(f"Successfully loaded page: {original_url}") print(f"Successfully loaded page: {original_url}")
# Before clicking any pagination links, try to determine the last page # Analyze the page for pagination information
# by analyzing all pagination-related links on the current page pagination_info = await page.evaluate('''() => {
last_page = await page.evaluate('''() => { // Find the last page number if available
console.log("Analyzing page for last page number before navigation"); const findLastPageNumber = () => {
// Get all links on the page
const links = Array.from(document.querySelectorAll('a'));
// Get all links on the page // Strategy 1: Find numeric links (page numbers)
const links = Array.from(document.querySelectorAll('a')); const numericLinks = links.filter(link => {
console.log(`Found ${links.length} links to analyze`); const text = link.innerText.trim();
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
});
let lastPage = null; if (numericLinks.length > 0) {
let lastPageSource = ''; const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
return Math.max(...numericValues);
// Strategy 1: Find numeric links (e.g., page numbers)
const numericLinks = links.filter(link => {
const text = link.innerText.trim();
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
});
if (numericLinks.length > 0) {
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
const maxNumeric = Math.max(...numericValues);
console.log(`Found highest numeric link: ${maxNumeric}`);
if (maxNumeric > 1) {
lastPage = maxNumeric;
lastPageSource = 'numeric-links';
} }
}
// Strategy 2: Look for "last page" link // Strategy 2: Look for "last page" link
const lastLinks = links.filter(link => { const lastLinks = links.filter(link => {
const text = link.innerText.trim().toLowerCase(); const text = link.innerText.trim().toLowerCase();
const classes = (link.className || '').toLowerCase(); const classes = (link.className || '').toLowerCase();
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase(); const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
return (text === 'last' || return (text === 'last' ||
classes.includes('last') || classes.includes('last') ||
ariaLabel.includes('last') || ariaLabel.includes('last') ||
link.getAttribute('rel') === 'last'); link.getAttribute('rel') === 'last');
}); });
if (lastLinks.length > 0) { if (lastLinks.length > 0) {
console.log(`Found ${lastLinks.length} "last" links`); const lastLink = lastLinks[0];
// Try to extract page number from the URL const href = lastLink.href;
const lastLink = lastLinks[0];
const href = lastLink.href;
console.log(`Last link href: ${href}`);
// Common patterns: page=X, /page/X, etc. // Common patterns: page=X, /page/X, etc.
const pagePatterns = [ const pagePatterns = [
/[?&]page=(\d+)/, /[?&]page=(\d+)/,
/[?&]p=(\d+)/, /[?&]p=(\d+)/,
/[?&]pg=(\d+)/, /[?&]pg=(\d+)/,
/\/page\/(\d+)/, /\/page\/(\d+)/,
/\/p\/(\d+)/, /\/p\/(\d+)/,
/\/paged\/(\d+)/ /\/paged\/(\d+)/
];
for (const pattern of pagePatterns) {
const match = href.match(pattern);
if (match && match[1]) {
return parseInt(match[1]);
}
}
}
// Strategy 3: Analyze all URLs for page numbers
const pageNumbersFromUrls = [];
links.forEach(link => {
if (!link.href || link.href === '#') return;
// Check for common pagination URL patterns
const patterns = [
/[?&]page=(\d+)/,
/[?&]p=(\d+)/,
/[?&]pg=(\d+)/,
/\/page\/(\d+)/,
/\/p\/(\d+)/,
/\/paged\/(\d+)/,
/\/pages\/(\d+)/
];
for (const pattern of patterns) {
const match = link.href.match(pattern);
if (match && match[1]) {
pageNumbersFromUrls.push(parseInt(match[1]));
}
}
});
if (pageNumbersFromUrls.length > 0) {
return Math.max(...pageNumbersFromUrls);
}
return null;
};
// Find a pagination link to click
const findPaginationLink = () => {
const links = Array.from(document.querySelectorAll('a'));
// Try to find a page "2" link first (most reliable)
const page2Link = links.find(link => {
const text = link.innerText.trim();
return text === '2' && link.href && link.href !== '#';
});
if (page2Link) {
return { element: page2Link, href: page2Link.href, type: 'numeric' };
}
// Try common "next page" selectors
const nextSelectors = [
'a.next',
'a.page-next',
'a[rel="next"]',
'a[aria-label="Next page"]',
'a[aria-label="next"]'
]; ];
for (const pattern of pagePatterns) { for (const selector of nextSelectors) {
const match = href.match(pattern); const element = document.querySelector(selector);
if (match && match[1]) { if (element && element.href && element.href !== '#') {
const pageNum = parseInt(match[1]); return { element, href: element.href, type: 'next' };
console.log(`Found page number ${pageNum} in last link URL`);
if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) {
lastPage = pageNum;
lastPageSource = 'last-link-url';
break;
}
} }
} }
}
// Strategy 3: Look for pagination text like "Page 1 of 42" // Look for any link that might be pagination
const paginationTexts = []; const paginationLinks = links.filter(link => {
document.querySelectorAll('.pagination, .pager, .paginator, .paging, .page-numbers, .pages, .page-navigation') if (!link.href || link.href === '#') return false;
.forEach(el => paginationTexts.push(el.innerText));
// Also check for any element that might contain pagination info const text = link.innerText.trim();
document.querySelectorAll('[class*="pag"], [id*="pag"]') const href = link.href;
.forEach(el => paginationTexts.push(el.innerText));
const pageOfPatterns = [ // Check for numeric text or next/prev indicators
/page\s+\d+\s+of\s+(\d+)/i, const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
/page\s+\d+\s*\/\s*(\d+)/i, const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
/\d+\s*\/\s*(\d+)\s+pages/i, /[»«‹›<>]/.test(text);
/\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i,
/showing\s+\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i,
/\d+\s*-\s*\d+\s+of\s+\d+\s+items\s+\(\s*(\d+)\s+pages\s*\)/i
];
for (const text of paginationTexts) { // Check for page parameter in URL
for (const pattern of pageOfPatterns) { const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
const match = text.match(pattern);
if (match && match[1]) { return (isNumeric || isNextPrev || hasPageParam);
const pageNum = parseInt(match[1]); });
console.log(`Found page count ${pageNum} in text`);
if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) { if (paginationLinks.length > 0) {
lastPage = pageNum; const link = paginationLinks[0];
lastPageSource = 'pagination-text'; return { element: link, href: link.href, type: 'other' };
break;
}
}
} }
}
// NEW STRATEGY: Analyze all link URLs for page numbers return null;
console.log("Analyzing all link URLs for page numbers"); };
// Common URL patterns that indicate pagination const lastPage = findLastPageNumber();
const urlPagePatterns = [ const paginationLink = findPaginationLink();
/[?&]page=(\d+)/,
/[?&]p=(\d+)/,
/[?&]pg=(\d+)/,
/\/page\/(\d+)/,
/\/p\/(\d+)/,
/\/paged\/(\d+)/,
/\/pages\/(\d+)/,
/[?&]offset=(\d+)/,
/[?&]start=(\d+)/,
/[?&]from=(\d+)/,
/[?&]paged=(\d+)/,
/[?&]pagenum=(\d+)/,
/[?&]pageNumber=(\d+)/,
/[?&]currentpage=(\d+)/
];
// Extract page numbers from all link URLs if (paginationLink) {
const pageNumbersFromUrls = [];
links.forEach(link => {
if (!link.href || link.href === '#') return;
const href = link.href;
for (const pattern of urlPagePatterns) {
const match = href.match(pattern);
if (match && match[1]) {
const pageNum = parseInt(match[1]);
if (pageNum > 1) {
pageNumbersFromUrls.push(pageNum);
break;
}
}
}
});
if (pageNumbersFromUrls.length > 0) {
const maxPageFromUrls = Math.max(...pageNumbersFromUrls);
console.log(`Found highest page number in URLs: ${maxPageFromUrls} (from ${pageNumbersFromUrls.length} links)`);
if (maxPageFromUrls > 1 && (lastPage === null || maxPageFromUrls > lastPage)) {
lastPage = maxPageFromUrls;
lastPageSource = 'url-analysis';
}
}
console.log(`Pre-navigation last page detection: ${lastPage} (source: ${lastPageSource})`);
return { lastPage, lastPageSource };
}''');
print(f"Pre-navigation last page detection: {last_page.get('lastPage')} (source: {last_page.get('lastPageSource')})")
# Find and click on pagination elements
pagination_clicked = await page.evaluate('''async () => {
// Common pagination link selectors to try
const selectors = [
'a.next', 'a.page-next', 'a[rel="next"]',
'a[aria-label="Next page"]', 'a[aria-label="next"]',
'.pagination a:nth-child(2)', // Often the "2" link
'.pagination li:nth-child(2) a',
'a:contains("2")', 'a:contains("Next")', 'a:contains("»")'
];
// Try to find a page "2" link first
const links = Array.from(document.querySelectorAll('a'));
const page2Link = links.find(link => {
const text = link.innerText.trim();
return text === '2' && link.href && link.href !== '#';
});
if (page2Link) {
// Store the href before clicking
const href = page2Link.href;
// Click the link // Click the link
page2Link.click(); paginationLink.element.click();
return { clicked: true, href: href, type: 'numeric' }; return {
clicked: true,
href: paginationLink.href,
type: paginationLink.type,
lastPage
};
} }
// Try common selectors return { clicked: false, lastPage };
for (const selector of selectors) {
try {
if (selector.includes(':contains')) {
// Handle jQuery-style :contains selector
const text = selector.match(/:contains\\("(.+)"\\)/)[1];
const link = links.find(l => l.innerText.includes(text) && l.href && l.href !== '#');
if (link) {
const href = link.href;
link.click();
return { clicked: true, href: href, type: 'selector' };
}
} else {
const element = document.querySelector(selector);
if (element && element.href && element.href !== '#') {
const href = element.href;
element.click();
return { clicked: true, href: href, type: 'selector' };
}
}
} catch (e) {
// Ignore errors for invalid selectors
}
}
// Look for any link that might be pagination
const paginationLinks = links.filter(link => {
if (!link.href || link.href === '#') return false;
const text = link.innerText.trim();
const href = link.href;
// Check for numeric text or next/prev indicators
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
/[»«‹›<>]/.test(text);
// Check for page parameter in URL
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
return (isNumeric || isNextPrev || hasPageParam);
});
if (paginationLinks.length > 0) {
const link = paginationLinks[0];
const href = link.href;
link.click();
return { clicked: true, href: href, type: 'other' };
}
return { clicked: false };
}''') }''')
# If no pagination was found or clicked # If no pagination was found or clicked
if not pagination_clicked.get('clicked', False): if not pagination_info.get('clicked', False):
return { return {
"status": "success", "status": "success",
"url": decoded_url, "url": decoded_url,
"hasPagination": False, "hasPagination": False,
"urlTemplate": None, "urlTemplate": None,
"lastPage": last_page.get('lastPage') "lastPage": pagination_info.get('lastPage')
} }
# Wait for navigation to complete after the click # Wait for navigation to complete after the click
@@ -812,82 +731,202 @@ async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)):
"url": decoded_url, "url": decoded_url,
"hasPagination": True, "hasPagination": True,
"urlTemplate": "AJAX pagination (URL doesn't change)", "urlTemplate": "AJAX pagination (URL doesn't change)",
"lastPage": last_page.get('lastPage'), "lastPage": pagination_info.get('lastPage')
"nextPageUrl": next_page_url,
"originalUrl": original_url,
"expectedHref": pagination_clicked.get('href')
} }
print(f"Navigation successful: {original_url} -> {next_page_url}") print(f"Navigation successful: {original_url} -> {next_page_url}")
# Analyze the URL structure to determine pagination pattern # Analyze the URL structure to determine pagination pattern
url_template = None url_template = await page.evaluate('''(originalUrl, nextPageUrl) => {
// Helper function to parse URL query parameters
const parseQueryParams = (url) => {
const params = {};
if (url.includes('?')) {
const queryString = url.split('?')[1].split('#')[0];
queryString.split('&').forEach(param => {
if (param.includes('=')) {
const [key, value] = param.split('=', 2);
params[key] = value;
}
});
}
return params;
};
# Compare the original and new URLs to find the pagination pattern // Check for query parameter based pagination
if '?' in next_page_url: if (nextPageUrl.includes('?')) {
# Extract query parameters from both URLs const originalParams = parseQueryParams(originalUrl);
original_params = {} const nextParams = parseQueryParams(nextPageUrl);
if '?' in original_url:
original_query = original_url.split('?')[1].split('#')[0]
for param in original_query.split('&'):
if '=' in param:
key, value = param.split('=', 1)
original_params[key] = value
next_query = next_page_url.split('?')[1].split('#')[0] // Find parameters that changed or were added
next_params = {} let paginationParam = null;
for param in next_query.split('&'):
if '=' in param:
key, value = param.split('=', 1)
next_params[key] = value
# Find parameters that changed or were added // First check for common pagination parameter names
pagination_param = None const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber'];
for key, value in next_params.items():
# Check if parameter is new or changed
if key not in original_params or original_params[key] != value:
# Check if the value is numeric and could be a page number
if value.isdigit() and int(value) > 1:
pagination_param = key
break
# If we found a pagination parameter for (const key of commonPaginationParams) {
if pagination_param: if (key in nextParams &&
base_url = next_page_url.split('?')[0] (!(key in originalParams) || originalParams[key] !== nextParams[key])) {
if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) {
paginationParam = key;
break;
}
}
}
# Reconstruct the URL template with all parameters // If no common parameter found, check all parameters
query_parts = [] if (!paginationParam) {
for key, value in next_params.items(): for (const [key, value] of Object.entries(nextParams)) {
if key == pagination_param: // Check if parameter is new or changed
query_parts.append(f"{key}={{PAGE_NUMBER}}") if (!(key in originalParams) || originalParams[key] !== value) {
else: // Check if the value is numeric and could be a page number
query_parts.append(f"{key}={value}") if (/^\d+$/.test(value) && parseInt(value) > 1) {
paginationParam = key;
break;
}
}
}
}
url_template = f"{base_url}?{'&'.join(query_parts)}" // If we found a pagination parameter
if (paginationParam) {
const baseUrl = nextPageUrl.split('?')[0];
# If we couldn't determine from query parameters, check for path-based pagination // Reconstruct the URL template with all parameters
if not url_template: const queryParts = [];
# Path-based pagination patterns for (const [key, value] of Object.entries(nextParams)) {
path_patterns = ['/page/', '/p/', '/paged/', '/pages/'] if (key === paginationParam) {
for pattern in path_patterns: queryParts.push(`${key}={PAGE_NUMBER}`);
if pattern in next_page_url: } else {
parts = next_page_url.split(pattern) queryParts.push(`${key}=${value}`);
url_template = f"{parts[0]}{pattern}{{PAGE_NUMBER}}" }
if len(parts) > 1 and '/' in parts[1]: }
suffix = parts[1].split('/', 1)[1]
if suffix:
url_template += f"/{suffix}"
break
# If we still couldn't determine the pattern, use the original and next URLs as examples return `${baseUrl}?${queryParts.join('&')}`;
if not url_template: }
url_template = f"Pattern unclear. Example: {original_url} → {next_page_url}" }
// Check for path-based pagination
const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/'];
for (const pattern of pathPatterns) {
if (nextPageUrl.includes(pattern)) {
const parts = nextPageUrl.split(pattern);
let template = `${parts[0]}${pattern}{PAGE_NUMBER}`;
// Add any suffix after the page number
if (parts.length > 1 && parts[1].includes('/')) {
const suffix = parts[1].split('/', 1)[1];
if (suffix) {
template += `/${suffix}`;
}
}
return template;
}
}
// If we couldn't determine the pattern, try to make an educated guess
// For the specific case where a parameter like current_page=2 is added
const originalUrlObj = new URL(originalUrl);
const nextUrlObj = new URL(nextPageUrl);
// Check if the paths are the same but query params differ
if (originalUrlObj.pathname === nextUrlObj.pathname) {
const originalParams = parseQueryParams(originalUrl);
const nextParams = parseQueryParams(nextPageUrl);
// Find parameters that exist in next but not in original
const newParams = Object.keys(nextParams).filter(key => !(key in originalParams));
// If there's exactly one new parameter and it has a numeric value
if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) {
const paginationParam = newParams[0];
const baseUrl = nextPageUrl.split('?')[0];
// Reconstruct the URL template
const queryParts = [];
for (const [key, value] of Object.entries(nextParams)) {
if (key === paginationParam) {
queryParts.push(`${key}={PAGE_NUMBER}`);
} else {
queryParts.push(`${key}=${value}`);
}
}
return `${baseUrl}?${queryParts.join('&')}`;
}
}
// If we still couldn't determine the pattern, return both URLs as examples
return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`;
}''', original_url, next_page_url)
# Try to extract last page number from the next page if we didn't find it on the first page
if not pagination_info.get('lastPage'):
last_page_from_next = await page.evaluate('''() => {
// Get all links on the page
const links = Array.from(document.querySelectorAll('a'));
// Strategy 1: Find numeric links (page numbers)
const numericLinks = links.filter(link => {
const text = link.innerText.trim();
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
});
if (numericLinks.length > 0) {
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
return Math.max(...numericValues);
}
// Strategy 2: Analyze all URLs for page numbers
const pageNumbersFromUrls = [];
links.forEach(link => {
if (!link.href || link.href === '#') return;
// Check for common pagination URL patterns
const patterns = [
/[?&]page=(\d+)/,
/[?&]p=(\d+)/,
/[?&]pg=(\d+)/,
/\/page\/(\d+)/,
/\/p\/(\d+)/,
/\/paged\/(\d+)/,
/\/pages\/(\d+)/
];
for (const pattern of patterns) {
const match = link.href.match(pattern);
if (match && match[1]) {
pageNumbersFromUrls.push(parseInt(match[1]));
}
}
});
if (pageNumbersFromUrls.length > 0) {
return Math.max(...pageNumbersFromUrls);
}
return null;
}''')
if last_page_from_next:
pagination_info['lastPage'] = last_page_from_next
# Check if the pattern is unclear
has_pagination = True
url_string_template = str(url_template)
if url_string_template and url_string_template.startswith("Pattern unclear"):
has_pagination = False
url_template = None
result = { result = {
"status": "success", "status": "success",
"hasPagination": True, "url": decoded_url,
"hasPagination": has_pagination,
"urlTemplate": url_template, "urlTemplate": url_template,
"lastPage": last_page.get('lastPage') "lastPage": pagination_info.get('lastPage'),
"originalUrl": original_url,
"nextPageUrl": next_page_url
} }
return result return result