Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

better pagination detection
Build and Push Docker Images / build-and-push (push) Successful in 30s
Details

Browse Source
This commit is contained in:
Bram
2025-05-06 12:29:37 +02:00
parent e7fe91d037
commit 707ad9457d
1 changed files with 339 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+336 -349
View File
@@ -1,5 +1,5 @@
from app.utils.browser_utils import safe_browser_operation
import asyncio
async def visit_url_service(decoded_url):
"""Service function to visit a URL and get its content"""
print(f"Visiting URL: {decoded_url}")
@@ -165,386 +165,374 @@ async def detect_pagination_service(decoded_url):
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
original_url = page.url
print(f"Successfully loaded page: {original_url}")
# Analyze the page for pagination information
pagination_info = await page.evaluate('''() => {
// Find the last page number if available
const findLastPageNumber = () => {
// Get all links on the page
const links = Array.from(document.querySelectorAll('a'));
// Strategy 1: Find numeric links (page numbers)
const numericLinks = links.filter(link => {
const text = link.innerText.trim();
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
});
if (numericLinks.length > 0) {
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
return Math.max(...numericValues);
}
// Strategy 2: Look for "last page" link
const lastLinks = links.filter(link => {
const text = link.innerText.trim().toLowerCase();
const classes = (link.className || '').toLowerCase();
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
return (text === 'last' ||
classes.includes('last') ||
ariaLabel.includes('last') ||
link.getAttribute('rel') === 'last');
});
if (lastLinks.length > 0) {
const lastLink = lastLinks[0];
const href = lastLink.href;
// Common patterns: page=X, /page/X, etc.
const pagePatterns = [
/[?&]page=(\d+)/,
/[?&]p=(\d+)/,
/[?&]pg=(\d+)/,
/\/page\/(\d+)/,
/\/p\/(\d+)/,
/\/paged\/(\d+)/
];
for (const pattern of pagePatterns) {
const match = href.match(pattern);
if (match && match[1]) {
return parseInt(match[1]);
}
}
}
// Strategy 3: Analyze all URLs for page numbers
const pageNumbersFromUrls = [];
links.forEach(link => {
if (!link.href || link.href === '#') return;
// Check for common pagination URL patterns
const patterns = [
/[?&]page=(\d+)/,
/[?&]p=(\d+)/,
/[?&]pg=(\d+)/,
/\/page\/(\d+)/,
/\/p\/(\d+)/,
/\/paged\/(\d+)/,
/\/pages\/(\d+)/
];
for (const pattern of patterns) {
const match = link.href.match(pattern);
if (match && match[1]) {
pageNumbersFromUrls.push(parseInt(match[1]));
}
}
});
if (pageNumbersFromUrls.length > 0) {
return Math.max(...pageNumbersFromUrls);
}
return null;
# Check for common pagination indicators
pagination_data = await page.evaluate('''() => {
const data = {
hasPagination: false,
paginationType: null,
paginationElements: [],
detectedParameter: null,
lastPageNumber: null
};
// Find a pagination link to click
const findPaginationLink = () => {
const links = Array.from(document.querySelectorAll('a'));
// Look for numbered pagination links (1, 2, 3...)
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
.filter(el => {
const text = el.innerText.trim();
// Try to find a page "2" link first (most reliable)
const page2Link = links.find(link => {
const text = link.innerText.trim();
return text === '2' && link.href && link.href !== '#';
// check for data-page attribute
const dataPage = el.getAttribute('data-page');
if (dataPage) {
return /^[0-9]+$/.test(dataPage);
}
return /^[0-9]+$/.test(text) &&
(el.tagName === 'A' || el.onclick ||
el.closest('button, [role="button"]'));
});
if (page2Link) {
return { element: page2Link, href: page2Link.href, type: 'numeric' };
}
console.log(numberedLinks);
// Try common "next page" selectors
const nextSelectors = [
'a.next',
'a.page-next',
'a[rel="next"]',
'a[aria-label="Next page"]',
'a[aria-label="next"]'
];
// Look for next/prev buttons
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
.filter(el => {
const text = el.innerText.trim().toLowerCase();
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
for (const selector of nextSelectors) {
const element = document.querySelector(selector);
if (element && element.href && element.href !== '#') {
return { element, href: element.href, type: 'next' };
}
}
// Look for any link that might be pagination
const paginationLinks = links.filter(link => {
if (!link.href || link.href === '#') return false;
const text = link.innerText.trim();
const href = link.href;
// Check for numeric text or next/prev indicators
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
/[»«‹›<>]/.test(text);
// Check for page parameter in URL
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
return (isNumeric || isNextPrev || hasPageParam);
return text.includes('next') ||
text.includes('›') ||
text.includes('»') ||
text.includes('→') ||
ariaLabel.includes('next') ||
hasNextIcon;
});
if (paginationLinks.length > 0) {
const link = paginationLinks[0];
return { element: link, href: link.href, type: 'other' };
}
// Check for pagination containers
const paginationContainers = Array.from(document.querySelectorAll(
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
));
return null;
};
// Collect all potential pagination elements
if (numberedLinks.length > 0) {
data.hasPagination = true;
data.paginationType = 'numbered';
const lastPage = findLastPageNumber();
const paginationLink = findPaginationLink();
if (paginationLink) {
// Click the link
paginationLink.element.click();
// Get href attributes or other identifiers from numbered links
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
return {
clicked: true,
href: paginationLink.href,
type: paginationLink.type,
lastPage
text: el.innerText.trim(),
dataPage: el.getAttribute('data-page'),
href: el.tagName === 'A' ? el.href : null,
classes: el.className,
id: el.id
};
});
// Try to find the last page number
const pageNumbers = numberedLinks
.map(el => parseInt(el.innerText.trim()))
.filter(num => !isNaN(num));
if (pageNumbers.length > 0) {
data.lastPageNumber = Math.max(...pageNumbers);
}
return { clicked: false, lastPage };
}''')
// Also look for a "last page" element that might have text like "Last" or "»"
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
.find(el => {
const text = el.innerText.trim().toLowerCase();
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
return text.includes('last') ||
text === '»' ||
ariaLabel.includes('last page');
});
# If no pagination was found or clicked
if not pagination_info.get('clicked', False):
return {
"status": "success",
"url": decoded_url,
"hasPagination": False,
"urlTemplate": None,
"lastPage": pagination_info.get('lastPage')
if (lastPageElement && lastPageElement.href) {
// Try to extract page number from the URL
try {
const url = new URL(lastPageElement.href);
// Check common pagination parameters
['page', 'p', 'pg'].forEach(param => {
if (url.searchParams.has(param)) {
const value = parseInt(url.searchParams.get(param));
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
data.lastPageNumber = value;
}
# Wait for navigation to complete after the click
try:
await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'})
except Exception as e:
print(f"Navigation timeout: {e}")
# Get the new URL after clicking
next_page_url = page.url
# If URL didn't change, pagination might be handled by AJAX
if next_page_url == original_url:
return {
"status": "success",
"url": decoded_url,
"hasPagination": True,
"urlTemplate": "AJAX pagination (URL doesn't change)",
"lastPage": pagination_info.get('lastPage')
}
print(f"Navigation successful: {original_url} -> {next_page_url}")
# Analyze the URL structure to determine pagination pattern
url_template = await page.evaluate('''(originalUrl, nextPageUrl) => {
// Helper function to parse URL query parameters
const parseQueryParams = (url) => {
const params = {};
if (url.includes('?')) {
const queryString = url.split('?')[1].split('#')[0];
queryString.split('&').forEach(param => {
if (param.includes('=')) {
const [key, value] = param.split('=', 2);
params[key] = value;
}
});
// Check for path-based pagination (like /page/10)
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
if (pathMatch && pathMatch[1]) {
const value = parseInt(pathMatch[1]);
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
data.lastPageNumber = value;
}
return params;
}
} catch (e) {
console.error("Error parsing last page URL:", e);
}
}
} else if (nextButtons.length > 0) {
data.hasPagination = true;
data.paginationType = 'next-prev';
// Get information about next buttons
data.paginationElements = nextButtons.slice(0, 3).map(el => {
return {
text: el.innerText.trim(),
href: el.tagName === 'A' ? el.href : null,
classes: el.className,
id: el.id
};
});
}
// Check for query parameter based pagination
if (nextPageUrl.includes('?')) {
const originalParams = parseQueryParams(originalUrl);
const nextParams = parseQueryParams(nextPageUrl);
// Check for URL parameters that might indicate pagination
const currentUrl = window.location.href;
const urlParams = new URL(currentUrl).searchParams;
// Find parameters that changed or were added
let paginationParam = null;
// Common pagination parameters
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
// First check for common pagination parameter names
const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber'];
for (const key of commonPaginationParams) {
if (key in nextParams &&
(!(key in originalParams) || originalParams[key] !== nextParams[key])) {
if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) {
paginationParam = key;
for (const param of paginationParams) {
if (urlParams.has(param)) {
data.detectedParameter = {
name: param,
value: urlParams.get(param)
};
break;
}
}
}
// If no common parameter found, check all parameters
if (!paginationParam) {
for (const [key, value] of Object.entries(nextParams)) {
// Check if parameter is new or changed
if (!(key in originalParams) || originalParams[key] !== value) {
// Check if the value is numeric and could be a page number
if (/^\d+$/.test(value) && parseInt(value) > 1) {
paginationParam = key;
break;
}
}
}
}
// If we found a pagination parameter
if (paginationParam) {
const baseUrl = nextPageUrl.split('?')[0];
// Reconstruct the URL template with all parameters
const queryParts = [];
for (const [key, value] of Object.entries(nextParams)) {
if (key === paginationParam) {
queryParts.push(`${key}={PAGE_NUMBER}`);
} else {
queryParts.push(`${key}=${value}`);
}
}
return `${baseUrl}?${queryParts.join('&')}`;
}
}
// Check for path-based pagination
const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/'];
for (const pattern of pathPatterns) {
if (nextPageUrl.includes(pattern)) {
const parts = nextPageUrl.split(pattern);
let template = `${parts[0]}${pattern}{PAGE_NUMBER}`;
// Add any suffix after the page number
if (parts.length > 1 && parts[1].includes('/')) {
const suffix = parts[1].split('/', 1)[1];
if (suffix) {
template += `/${suffix}`;
}
}
return template;
}
}
// If we couldn't determine the pattern, try to make an educated guess
// For the specific case where a parameter like current_page=2 is added
const originalUrlObj = new URL(originalUrl);
const nextUrlObj = new URL(nextPageUrl);
// Check if the paths are the same but query params differ
if (originalUrlObj.pathname === nextUrlObj.pathname) {
const originalParams = parseQueryParams(originalUrl);
const nextParams = parseQueryParams(nextPageUrl);
// Find parameters that exist in next but not in original
const newParams = Object.keys(nextParams).filter(key => !(key in originalParams));
// If there's exactly one new parameter and it has a numeric value
if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) {
const paginationParam = newParams[0];
const baseUrl = nextPageUrl.split('?')[0];
// Reconstruct the URL template
const queryParts = [];
for (const [key, value] of Object.entries(nextParams)) {
if (key === paginationParam) {
queryParts.push(`${key}={PAGE_NUMBER}`);
} else {
queryParts.push(`${key}=${value}`);
}
}
return `${baseUrl}?${queryParts.join('&')}`;
}
}
// If we still couldn't determine the pattern, return both URLs as examples
return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`;
}''', original_url, next_page_url)
# Try to extract last page number from the next page if we didn't find it on the first page
if not pagination_info.get('lastPage'):
last_page_from_next = await page.evaluate('''() => {
// Get all links on the page
const links = Array.from(document.querySelectorAll('a'));
// Strategy 1: Find numeric links (page numbers)
const numericLinks = links.filter(link => {
const text = link.innerText.trim();
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
});
if (numericLinks.length > 0) {
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
return Math.max(...numericValues);
}
// Strategy 2: Analyze all URLs for page numbers
const pageNumbersFromUrls = [];
links.forEach(link => {
if (!link.href || link.href === '#') return;
// Check for common pagination URL patterns
const patterns = [
/[?&]page=(\d+)/,
/[?&]p=(\d+)/,
/[?&]pg=(\d+)/,
/\/page\/(\d+)/,
/\/p\/(\d+)/,
/\/paged\/(\d+)/,
/\/pages\/(\d+)/
];
for (const pattern of patterns) {
const match = link.href.match(pattern);
if (match && match[1]) {
pageNumbersFromUrls.push(parseInt(match[1]));
}
}
});
if (pageNumbersFromUrls.length > 0) {
return Math.max(...pageNumbersFromUrls);
}
return null;
return data;
}''')
if last_page_from_next:
pagination_info['lastPage'] = last_page_from_next
# Check if the pattern is unclear
has_pagination = True
url_string_template = str(url_template)
if url_string_template and (url_string_template.startswith("Pattern unclear") or url_string_template.startswith("AJAX pagination")):
has_pagination = False
# If pagination is detected, try to navigate to the next page by clicking
next_page_url = None
pagination_parameter = None
url_template = None
if pagination_data['hasPagination']:
print("Pagination detected, attempting to click on a pagination element")
# Always try to click on a pagination element, regardless of type
clicked = False
# First try to click on a numbered link (preferably "2" if we're on page 1)
try:
clicked = await page.evaluate('''() => {
// First try to find and click on a "2" link or button
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
.filter(el => {
// Check for text content "2"
if (el.innerText.trim() === '2') {
return true;
}
// Check for href with page=2 or similar (for anchor elements)
if (el.tagName === 'A' && el.href) {
try {
const url = new URL(el.href, window.location.origin);
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
'currentpage', 'pagenum', 'pageNumber', 'paged'];
for (const param of paginationParams) {
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
return true;
}
}
// Check for path-based pagination like /page/2/
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
if (pathMatch) {
return true;
}
} catch (e) {}
}
// Check for data attributes that might indicate pagination
if (el.getAttribute('data-page') === '2' ||
el.getAttribute('data-pagenumber') === '2' ||
el.getAttribute('data-page-number') === '2') {
return true;
}
return false;
});
if (page2Elements.length > 0) {
console.log("Clicking on page 2 element");
page2Elements[0].click();
return true;
}
// If no "2" link found, try any numbered link or button
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
.filter(el => /^\d+$/.test(el.innerText.trim()));
if (numberedElements.length > 0) {
// Sort by number and get the second one (likely page 2)
const sorted = numberedElements.sort((a, b) => {
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
});
// Get the second element if available (page 2), otherwise the first one
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
console.log("Clicking on numbered element: " + elementToClick.innerText);
elementToClick.click();
return true;
}
// If no numbered links, try next button
const nextTexts = ['next', '›', '»', '→'];
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
.filter(el => {
const text = el.textContent.trim().toLowerCase();
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
return nextTexts.some(t => text.includes(t)) ||
ariaLabel.includes('next') ||
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
});
if (nextElements.length > 0) {
console.log("Clicking on next button");
nextElements[0].click();
return true;
}
return false;
}''')
if clicked:
print("Successfully clicked on pagination element")
# Wait for navigation to complete
await asyncio.sleep(1)
next_page_url = page.url
else:
print("No clickable pagination element found")
except Exception as e:
print(f"Error clicking on pagination element: {e}")
# If we successfully navigated to the next page, analyze the URL difference
if next_page_url:
print('Searching for pagination parameter')
# Parse both URLs
original_parsed = await page.evaluate(f'''(originalUrl) => {{
const original = new URL(originalUrl);
const current = new URL(window.location.href);
// Check for differences in query parameters
let paramDiff = null;
// Common pagination parameters to check
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
for (const param of paginationParams) {{
const originalValue = original.searchParams.get(param);
const currentValue = current.searchParams.get(param);
if (originalValue !== currentValue && currentValue !== null) {{
paramDiff = {{
name: param,
originalValue: originalValue,
currentValue: currentValue
}};
break;
}}
}}
// Check for path differences (like /page/1 vs /page/2)
const originalPath = original.pathname;
const currentPath = current.pathname;
let pathDiff = null;
if (originalPath !== currentPath) {{
const originalSegments = originalPath.split('/').filter(s => s);
const currentSegments = currentPath.split('/').filter(s => s);
// Find the segment that changed
if (originalSegments.length === currentSegments.length) {{
for (let i = 0; i < originalSegments.length; i++) {{
if (originalSegments[i] !== currentSegments[i]) {{
// Check if the difference is numeric
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
pathDiff = {{
index: i,
originalValue: originalSegments[i],
currentValue: currentSegments[i]
}};
}}
}}
}}
}}
}}
return {{
paramDiff,
pathDiff,
originalUrl: originalUrl,
currentUrl: window.location.href
}};
}}''', original_url)
# Determine the pagination parameter and create URL template
if original_parsed['paramDiff']:
param_name = original_parsed['paramDiff']['name']
pagination_parameter = {
'type': 'query',
'name': param_name,
'value': original_parsed['paramDiff']['currentValue']
}
# Create URL template for query parameter
url_obj = await page.evaluate(f'''(url, paramName) => {{
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
return urlObj.toString();
}}''', original_url, param_name)
url_template = url_obj
elif original_parsed['pathDiff']:
path_index = original_parsed['pathDiff']['index']
pagination_parameter = {
'type': 'path',
'index': path_index,
'value': original_parsed['pathDiff']['currentValue']
}
# Create URL template for path parameter
url_template = await page.evaluate(f'''(url, pathIndex) => {{
const urlObj = new URL(url);
const pathSegments = urlObj.pathname.split('/').filter(s => s);
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString();
}}''', original_url, path_index)
if url_template:
# Decode URL-encoded characters in the template
import urllib.parse
url_template = urllib.parse.unquote(url_template)
# If we couldn't determine the URL template from navigation, try to infer it
if not url_template and pagination_data['detectedParameter']:
param_name = pagination_data['detectedParameter']['name']
url_template = await page.evaluate(f'''(url, paramName) => {{
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
return urlObj.toString();
}}''', original_url, param_name)
# Return the pagination detection results with a simplified structure
result = {
"status": "success",
"url": decoded_url,
"hasPagination": has_pagination,
"hasPagination": pagination_data['hasPagination'],
"urlTemplate": url_template,
"lastPage": pagination_info.get('lastPage'),
"originalUrl": original_url,
"nextPageUrl": next_page_url
"lastPage": pagination_data['lastPageNumber']
}
return result
@@ -553,11 +541,10 @@ async def detect_pagination_service(decoded_url):
print(f"Error during pagination detection: {e}")
return {
"status": "error",
"url": decoded_url,
"error": str(e),
"hasPagination": False,
"urlTemplate": None,
"lastPage": None
"lastPage": None,
"error": str(e)
}
# Perform the operation