This commit is contained in:
+313
-274
@@ -549,251 +549,170 @@ async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
|
||||
print(f"Successfully loaded page: {original_url}")
|
||||
|
||||
# Before clicking any pagination links, try to determine the last page
|
||||
# by analyzing all pagination-related links on the current page
|
||||
last_page = await page.evaluate('''() => {
|
||||
console.log("Analyzing page for last page number before navigation");
|
||||
# Analyze the page for pagination information
|
||||
pagination_info = await page.evaluate('''() => {
|
||||
// Find the last page number if available
|
||||
const findLastPageNumber = () => {
|
||||
// Get all links on the page
|
||||
const links = Array.from(document.querySelectorAll('a'));
|
||||
|
||||
// Get all links on the page
|
||||
const links = Array.from(document.querySelectorAll('a'));
|
||||
console.log(`Found ${links.length} links to analyze`);
|
||||
// Strategy 1: Find numeric links (page numbers)
|
||||
const numericLinks = links.filter(link => {
|
||||
const text = link.innerText.trim();
|
||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
||||
});
|
||||
|
||||
let lastPage = null;
|
||||
let lastPageSource = '';
|
||||
|
||||
// Strategy 1: Find numeric links (e.g., page numbers)
|
||||
const numericLinks = links.filter(link => {
|
||||
const text = link.innerText.trim();
|
||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
||||
});
|
||||
|
||||
if (numericLinks.length > 0) {
|
||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
||||
const maxNumeric = Math.max(...numericValues);
|
||||
console.log(`Found highest numeric link: ${maxNumeric}`);
|
||||
if (maxNumeric > 1) {
|
||||
lastPage = maxNumeric;
|
||||
lastPageSource = 'numeric-links';
|
||||
if (numericLinks.length > 0) {
|
||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
||||
return Math.max(...numericValues);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: Look for "last page" link
|
||||
const lastLinks = links.filter(link => {
|
||||
const text = link.innerText.trim().toLowerCase();
|
||||
const classes = (link.className || '').toLowerCase();
|
||||
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
|
||||
// Strategy 2: Look for "last page" link
|
||||
const lastLinks = links.filter(link => {
|
||||
const text = link.innerText.trim().toLowerCase();
|
||||
const classes = (link.className || '').toLowerCase();
|
||||
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
|
||||
|
||||
return (text === 'last' ||
|
||||
classes.includes('last') ||
|
||||
ariaLabel.includes('last') ||
|
||||
link.getAttribute('rel') === 'last');
|
||||
});
|
||||
return (text === 'last' ||
|
||||
classes.includes('last') ||
|
||||
ariaLabel.includes('last') ||
|
||||
link.getAttribute('rel') === 'last');
|
||||
});
|
||||
|
||||
if (lastLinks.length > 0) {
|
||||
console.log(`Found ${lastLinks.length} "last" links`);
|
||||
// Try to extract page number from the URL
|
||||
const lastLink = lastLinks[0];
|
||||
const href = lastLink.href;
|
||||
console.log(`Last link href: ${href}`);
|
||||
if (lastLinks.length > 0) {
|
||||
const lastLink = lastLinks[0];
|
||||
const href = lastLink.href;
|
||||
|
||||
// Common patterns: page=X, /page/X, etc.
|
||||
const pagePatterns = [
|
||||
/[?&]page=(\d+)/,
|
||||
/[?&]p=(\d+)/,
|
||||
/[?&]pg=(\d+)/,
|
||||
/\/page\/(\d+)/,
|
||||
/\/p\/(\d+)/,
|
||||
/\/paged\/(\d+)/
|
||||
// Common patterns: page=X, /page/X, etc.
|
||||
const pagePatterns = [
|
||||
/[?&]page=(\d+)/,
|
||||
/[?&]p=(\d+)/,
|
||||
/[?&]pg=(\d+)/,
|
||||
/\/page\/(\d+)/,
|
||||
/\/p\/(\d+)/,
|
||||
/\/paged\/(\d+)/
|
||||
];
|
||||
|
||||
for (const pattern of pagePatterns) {
|
||||
const match = href.match(pattern);
|
||||
if (match && match[1]) {
|
||||
return parseInt(match[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: Analyze all URLs for page numbers
|
||||
const pageNumbersFromUrls = [];
|
||||
links.forEach(link => {
|
||||
if (!link.href || link.href === '#') return;
|
||||
|
||||
// Check for common pagination URL patterns
|
||||
const patterns = [
|
||||
/[?&]page=(\d+)/,
|
||||
/[?&]p=(\d+)/,
|
||||
/[?&]pg=(\d+)/,
|
||||
/\/page\/(\d+)/,
|
||||
/\/p\/(\d+)/,
|
||||
/\/paged\/(\d+)/,
|
||||
/\/pages\/(\d+)/
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
const match = link.href.match(pattern);
|
||||
if (match && match[1]) {
|
||||
pageNumbersFromUrls.push(parseInt(match[1]));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (pageNumbersFromUrls.length > 0) {
|
||||
return Math.max(...pageNumbersFromUrls);
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
// Find a pagination link to click
|
||||
const findPaginationLink = () => {
|
||||
const links = Array.from(document.querySelectorAll('a'));
|
||||
|
||||
// Try to find a page "2" link first (most reliable)
|
||||
const page2Link = links.find(link => {
|
||||
const text = link.innerText.trim();
|
||||
return text === '2' && link.href && link.href !== '#';
|
||||
});
|
||||
|
||||
if (page2Link) {
|
||||
return { element: page2Link, href: page2Link.href, type: 'numeric' };
|
||||
}
|
||||
|
||||
// Try common "next page" selectors
|
||||
const nextSelectors = [
|
||||
'a.next',
|
||||
'a.page-next',
|
||||
'a[rel="next"]',
|
||||
'a[aria-label="Next page"]',
|
||||
'a[aria-label="next"]'
|
||||
];
|
||||
|
||||
for (const pattern of pagePatterns) {
|
||||
const match = href.match(pattern);
|
||||
if (match && match[1]) {
|
||||
const pageNum = parseInt(match[1]);
|
||||
console.log(`Found page number ${pageNum} in last link URL`);
|
||||
if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) {
|
||||
lastPage = pageNum;
|
||||
lastPageSource = 'last-link-url';
|
||||
break;
|
||||
}
|
||||
for (const selector of nextSelectors) {
|
||||
const element = document.querySelector(selector);
|
||||
if (element && element.href && element.href !== '#') {
|
||||
return { element, href: element.href, type: 'next' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 3: Look for pagination text like "Page 1 of 42"
|
||||
const paginationTexts = [];
|
||||
document.querySelectorAll('.pagination, .pager, .paginator, .paging, .page-numbers, .pages, .page-navigation')
|
||||
.forEach(el => paginationTexts.push(el.innerText));
|
||||
// Look for any link that might be pagination
|
||||
const paginationLinks = links.filter(link => {
|
||||
if (!link.href || link.href === '#') return false;
|
||||
|
||||
// Also check for any element that might contain pagination info
|
||||
document.querySelectorAll('[class*="pag"], [id*="pag"]')
|
||||
.forEach(el => paginationTexts.push(el.innerText));
|
||||
const text = link.innerText.trim();
|
||||
const href = link.href;
|
||||
|
||||
const pageOfPatterns = [
|
||||
/page\s+\d+\s+of\s+(\d+)/i,
|
||||
/page\s+\d+\s*\/\s*(\d+)/i,
|
||||
/\d+\s*\/\s*(\d+)\s+pages/i,
|
||||
/\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i,
|
||||
/showing\s+\d+\s*-\s*\d+\s+of\s+\d+\s+\(\s*(\d+)\s+pages\s*\)/i,
|
||||
/\d+\s*-\s*\d+\s+of\s+\d+\s+items\s+\(\s*(\d+)\s+pages\s*\)/i
|
||||
];
|
||||
// Check for numeric text or next/prev indicators
|
||||
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
|
||||
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
|
||||
/[»«‹›<>]/.test(text);
|
||||
|
||||
for (const text of paginationTexts) {
|
||||
for (const pattern of pageOfPatterns) {
|
||||
const match = text.match(pattern);
|
||||
if (match && match[1]) {
|
||||
const pageNum = parseInt(match[1]);
|
||||
console.log(`Found page count ${pageNum} in text`);
|
||||
if (pageNum > 1 && (lastPage === null || pageNum > lastPage)) {
|
||||
lastPage = pageNum;
|
||||
lastPageSource = 'pagination-text';
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Check for page parameter in URL
|
||||
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
|
||||
|
||||
return (isNumeric || isNextPrev || hasPageParam);
|
||||
});
|
||||
|
||||
if (paginationLinks.length > 0) {
|
||||
const link = paginationLinks[0];
|
||||
return { element: link, href: link.href, type: 'other' };
|
||||
}
|
||||
}
|
||||
|
||||
// NEW STRATEGY: Analyze all link URLs for page numbers
|
||||
console.log("Analyzing all link URLs for page numbers");
|
||||
return null;
|
||||
};
|
||||
|
||||
// Common URL patterns that indicate pagination
|
||||
const urlPagePatterns = [
|
||||
/[?&]page=(\d+)/,
|
||||
/[?&]p=(\d+)/,
|
||||
/[?&]pg=(\d+)/,
|
||||
/\/page\/(\d+)/,
|
||||
/\/p\/(\d+)/,
|
||||
/\/paged\/(\d+)/,
|
||||
/\/pages\/(\d+)/,
|
||||
/[?&]offset=(\d+)/,
|
||||
/[?&]start=(\d+)/,
|
||||
/[?&]from=(\d+)/,
|
||||
/[?&]paged=(\d+)/,
|
||||
/[?&]pagenum=(\d+)/,
|
||||
/[?&]pageNumber=(\d+)/,
|
||||
/[?&]currentpage=(\d+)/
|
||||
];
|
||||
const lastPage = findLastPageNumber();
|
||||
const paginationLink = findPaginationLink();
|
||||
|
||||
// Extract page numbers from all link URLs
|
||||
const pageNumbersFromUrls = [];
|
||||
links.forEach(link => {
|
||||
if (!link.href || link.href === '#') return;
|
||||
|
||||
const href = link.href;
|
||||
for (const pattern of urlPagePatterns) {
|
||||
const match = href.match(pattern);
|
||||
if (match && match[1]) {
|
||||
const pageNum = parseInt(match[1]);
|
||||
if (pageNum > 1) {
|
||||
pageNumbersFromUrls.push(pageNum);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (pageNumbersFromUrls.length > 0) {
|
||||
const maxPageFromUrls = Math.max(...pageNumbersFromUrls);
|
||||
console.log(`Found highest page number in URLs: ${maxPageFromUrls} (from ${pageNumbersFromUrls.length} links)`);
|
||||
if (maxPageFromUrls > 1 && (lastPage === null || maxPageFromUrls > lastPage)) {
|
||||
lastPage = maxPageFromUrls;
|
||||
lastPageSource = 'url-analysis';
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Pre-navigation last page detection: ${lastPage} (source: ${lastPageSource})`);
|
||||
return { lastPage, lastPageSource };
|
||||
}''');
|
||||
|
||||
print(f"Pre-navigation last page detection: {last_page.get('lastPage')} (source: {last_page.get('lastPageSource')})")
|
||||
|
||||
# Find and click on pagination elements
|
||||
pagination_clicked = await page.evaluate('''async () => {
|
||||
// Common pagination link selectors to try
|
||||
const selectors = [
|
||||
'a.next', 'a.page-next', 'a[rel="next"]',
|
||||
'a[aria-label="Next page"]', 'a[aria-label="next"]',
|
||||
'.pagination a:nth-child(2)', // Often the "2" link
|
||||
'.pagination li:nth-child(2) a',
|
||||
'a:contains("2")', 'a:contains("Next")', 'a:contains("»")'
|
||||
];
|
||||
|
||||
// Try to find a page "2" link first
|
||||
const links = Array.from(document.querySelectorAll('a'));
|
||||
const page2Link = links.find(link => {
|
||||
const text = link.innerText.trim();
|
||||
return text === '2' && link.href && link.href !== '#';
|
||||
});
|
||||
|
||||
if (page2Link) {
|
||||
// Store the href before clicking
|
||||
const href = page2Link.href;
|
||||
if (paginationLink) {
|
||||
// Click the link
|
||||
page2Link.click();
|
||||
return { clicked: true, href: href, type: 'numeric' };
|
||||
paginationLink.element.click();
|
||||
return {
|
||||
clicked: true,
|
||||
href: paginationLink.href,
|
||||
type: paginationLink.type,
|
||||
lastPage
|
||||
};
|
||||
}
|
||||
|
||||
// Try common selectors
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
if (selector.includes(':contains')) {
|
||||
// Handle jQuery-style :contains selector
|
||||
const text = selector.match(/:contains\\("(.+)"\\)/)[1];
|
||||
const link = links.find(l => l.innerText.includes(text) && l.href && l.href !== '#');
|
||||
if (link) {
|
||||
const href = link.href;
|
||||
link.click();
|
||||
return { clicked: true, href: href, type: 'selector' };
|
||||
}
|
||||
} else {
|
||||
const element = document.querySelector(selector);
|
||||
if (element && element.href && element.href !== '#') {
|
||||
const href = element.href;
|
||||
element.click();
|
||||
return { clicked: true, href: href, type: 'selector' };
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// Ignore errors for invalid selectors
|
||||
}
|
||||
}
|
||||
|
||||
// Look for any link that might be pagination
|
||||
const paginationLinks = links.filter(link => {
|
||||
if (!link.href || link.href === '#') return false;
|
||||
|
||||
const text = link.innerText.trim();
|
||||
const href = link.href;
|
||||
|
||||
// Check for numeric text or next/prev indicators
|
||||
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
|
||||
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
|
||||
/[»«‹›<>]/.test(text);
|
||||
|
||||
// Check for page parameter in URL
|
||||
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
|
||||
|
||||
return (isNumeric || isNextPrev || hasPageParam);
|
||||
});
|
||||
|
||||
if (paginationLinks.length > 0) {
|
||||
const link = paginationLinks[0];
|
||||
const href = link.href;
|
||||
link.click();
|
||||
return { clicked: true, href: href, type: 'other' };
|
||||
}
|
||||
|
||||
return { clicked: false };
|
||||
return { clicked: false, lastPage };
|
||||
}''')
|
||||
|
||||
# If no pagination was found or clicked
|
||||
if not pagination_clicked.get('clicked', False):
|
||||
if not pagination_info.get('clicked', False):
|
||||
return {
|
||||
"status": "success",
|
||||
"url": decoded_url,
|
||||
"hasPagination": False,
|
||||
"urlTemplate": None,
|
||||
"lastPage": last_page.get('lastPage')
|
||||
"lastPage": pagination_info.get('lastPage')
|
||||
}
|
||||
|
||||
# Wait for navigation to complete after the click
|
||||
@@ -812,82 +731,202 @@ async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)):
|
||||
"url": decoded_url,
|
||||
"hasPagination": True,
|
||||
"urlTemplate": "AJAX pagination (URL doesn't change)",
|
||||
"lastPage": last_page.get('lastPage'),
|
||||
"nextPageUrl": next_page_url,
|
||||
"originalUrl": original_url,
|
||||
"expectedHref": pagination_clicked.get('href')
|
||||
"lastPage": pagination_info.get('lastPage')
|
||||
}
|
||||
|
||||
print(f"Navigation successful: {original_url} -> {next_page_url}")
|
||||
|
||||
# Analyze the URL structure to determine pagination pattern
|
||||
url_template = None
|
||||
url_template = await page.evaluate('''(originalUrl, nextPageUrl) => {
|
||||
// Helper function to parse URL query parameters
|
||||
const parseQueryParams = (url) => {
|
||||
const params = {};
|
||||
if (url.includes('?')) {
|
||||
const queryString = url.split('?')[1].split('#')[0];
|
||||
queryString.split('&').forEach(param => {
|
||||
if (param.includes('=')) {
|
||||
const [key, value] = param.split('=', 2);
|
||||
params[key] = value;
|
||||
}
|
||||
});
|
||||
}
|
||||
return params;
|
||||
};
|
||||
|
||||
# Compare the original and new URLs to find the pagination pattern
|
||||
if '?' in next_page_url:
|
||||
# Extract query parameters from both URLs
|
||||
original_params = {}
|
||||
if '?' in original_url:
|
||||
original_query = original_url.split('?')[1].split('#')[0]
|
||||
for param in original_query.split('&'):
|
||||
if '=' in param:
|
||||
key, value = param.split('=', 1)
|
||||
original_params[key] = value
|
||||
// Check for query parameter based pagination
|
||||
if (nextPageUrl.includes('?')) {
|
||||
const originalParams = parseQueryParams(originalUrl);
|
||||
const nextParams = parseQueryParams(nextPageUrl);
|
||||
|
||||
next_query = next_page_url.split('?')[1].split('#')[0]
|
||||
next_params = {}
|
||||
for param in next_query.split('&'):
|
||||
if '=' in param:
|
||||
key, value = param.split('=', 1)
|
||||
next_params[key] = value
|
||||
// Find parameters that changed or were added
|
||||
let paginationParam = null;
|
||||
|
||||
# Find parameters that changed or were added
|
||||
pagination_param = None
|
||||
for key, value in next_params.items():
|
||||
# Check if parameter is new or changed
|
||||
if key not in original_params or original_params[key] != value:
|
||||
# Check if the value is numeric and could be a page number
|
||||
if value.isdigit() and int(value) > 1:
|
||||
pagination_param = key
|
||||
break
|
||||
// First check for common pagination parameter names
|
||||
const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber'];
|
||||
|
||||
# If we found a pagination parameter
|
||||
if pagination_param:
|
||||
base_url = next_page_url.split('?')[0]
|
||||
for (const key of commonPaginationParams) {
|
||||
if (key in nextParams &&
|
||||
(!(key in originalParams) || originalParams[key] !== nextParams[key])) {
|
||||
if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) {
|
||||
paginationParam = key;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Reconstruct the URL template with all parameters
|
||||
query_parts = []
|
||||
for key, value in next_params.items():
|
||||
if key == pagination_param:
|
||||
query_parts.append(f"{key}={{PAGE_NUMBER}}")
|
||||
else:
|
||||
query_parts.append(f"{key}={value}")
|
||||
// If no common parameter found, check all parameters
|
||||
if (!paginationParam) {
|
||||
for (const [key, value] of Object.entries(nextParams)) {
|
||||
// Check if parameter is new or changed
|
||||
if (!(key in originalParams) || originalParams[key] !== value) {
|
||||
// Check if the value is numeric and could be a page number
|
||||
if (/^\d+$/.test(value) && parseInt(value) > 1) {
|
||||
paginationParam = key;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
url_template = f"{base_url}?{'&'.join(query_parts)}"
|
||||
// If we found a pagination parameter
|
||||
if (paginationParam) {
|
||||
const baseUrl = nextPageUrl.split('?')[0];
|
||||
|
||||
# If we couldn't determine from query parameters, check for path-based pagination
|
||||
if not url_template:
|
||||
# Path-based pagination patterns
|
||||
path_patterns = ['/page/', '/p/', '/paged/', '/pages/']
|
||||
for pattern in path_patterns:
|
||||
if pattern in next_page_url:
|
||||
parts = next_page_url.split(pattern)
|
||||
url_template = f"{parts[0]}{pattern}{{PAGE_NUMBER}}"
|
||||
if len(parts) > 1 and '/' in parts[1]:
|
||||
suffix = parts[1].split('/', 1)[1]
|
||||
if suffix:
|
||||
url_template += f"/{suffix}"
|
||||
break
|
||||
// Reconstruct the URL template with all parameters
|
||||
const queryParts = [];
|
||||
for (const [key, value] of Object.entries(nextParams)) {
|
||||
if (key === paginationParam) {
|
||||
queryParts.push(`${key}={PAGE_NUMBER}`);
|
||||
} else {
|
||||
queryParts.push(`${key}=${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
# If we still couldn't determine the pattern, use the original and next URLs as examples
|
||||
if not url_template:
|
||||
url_template = f"Pattern unclear. Example: {original_url} → {next_page_url}"
|
||||
return `${baseUrl}?${queryParts.join('&')}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for path-based pagination
|
||||
const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/'];
|
||||
for (const pattern of pathPatterns) {
|
||||
if (nextPageUrl.includes(pattern)) {
|
||||
const parts = nextPageUrl.split(pattern);
|
||||
let template = `${parts[0]}${pattern}{PAGE_NUMBER}`;
|
||||
|
||||
// Add any suffix after the page number
|
||||
if (parts.length > 1 && parts[1].includes('/')) {
|
||||
const suffix = parts[1].split('/', 1)[1];
|
||||
if (suffix) {
|
||||
template += `/${suffix}`;
|
||||
}
|
||||
}
|
||||
|
||||
return template;
|
||||
}
|
||||
}
|
||||
|
||||
// If we couldn't determine the pattern, try to make an educated guess
|
||||
// For the specific case where a parameter like current_page=2 is added
|
||||
const originalUrlObj = new URL(originalUrl);
|
||||
const nextUrlObj = new URL(nextPageUrl);
|
||||
|
||||
// Check if the paths are the same but query params differ
|
||||
if (originalUrlObj.pathname === nextUrlObj.pathname) {
|
||||
const originalParams = parseQueryParams(originalUrl);
|
||||
const nextParams = parseQueryParams(nextPageUrl);
|
||||
|
||||
// Find parameters that exist in next but not in original
|
||||
const newParams = Object.keys(nextParams).filter(key => !(key in originalParams));
|
||||
|
||||
// If there's exactly one new parameter and it has a numeric value
|
||||
if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) {
|
||||
const paginationParam = newParams[0];
|
||||
const baseUrl = nextPageUrl.split('?')[0];
|
||||
|
||||
// Reconstruct the URL template
|
||||
const queryParts = [];
|
||||
for (const [key, value] of Object.entries(nextParams)) {
|
||||
if (key === paginationParam) {
|
||||
queryParts.push(`${key}={PAGE_NUMBER}`);
|
||||
} else {
|
||||
queryParts.push(`${key}=${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
return `${baseUrl}?${queryParts.join('&')}`;
|
||||
}
|
||||
}
|
||||
|
||||
// If we still couldn't determine the pattern, return both URLs as examples
|
||||
return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`;
|
||||
}''', original_url, next_page_url)
|
||||
|
||||
# Try to extract last page number from the next page if we didn't find it on the first page
|
||||
if not pagination_info.get('lastPage'):
|
||||
last_page_from_next = await page.evaluate('''() => {
|
||||
// Get all links on the page
|
||||
const links = Array.from(document.querySelectorAll('a'));
|
||||
|
||||
// Strategy 1: Find numeric links (page numbers)
|
||||
const numericLinks = links.filter(link => {
|
||||
const text = link.innerText.trim();
|
||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
||||
});
|
||||
|
||||
if (numericLinks.length > 0) {
|
||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
||||
return Math.max(...numericValues);
|
||||
}
|
||||
|
||||
// Strategy 2: Analyze all URLs for page numbers
|
||||
const pageNumbersFromUrls = [];
|
||||
links.forEach(link => {
|
||||
if (!link.href || link.href === '#') return;
|
||||
|
||||
// Check for common pagination URL patterns
|
||||
const patterns = [
|
||||
/[?&]page=(\d+)/,
|
||||
/[?&]p=(\d+)/,
|
||||
/[?&]pg=(\d+)/,
|
||||
/\/page\/(\d+)/,
|
||||
/\/p\/(\d+)/,
|
||||
/\/paged\/(\d+)/,
|
||||
/\/pages\/(\d+)/
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
const match = link.href.match(pattern);
|
||||
if (match && match[1]) {
|
||||
pageNumbersFromUrls.push(parseInt(match[1]));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (pageNumbersFromUrls.length > 0) {
|
||||
return Math.max(...pageNumbersFromUrls);
|
||||
}
|
||||
|
||||
return null;
|
||||
}''')
|
||||
|
||||
if last_page_from_next:
|
||||
pagination_info['lastPage'] = last_page_from_next
|
||||
|
||||
# Check if the pattern is unclear
|
||||
has_pagination = True
|
||||
url_string_template = str(url_template)
|
||||
if url_string_template and url_string_template.startswith("Pattern unclear"):
|
||||
has_pagination = False
|
||||
url_template = None
|
||||
|
||||
result = {
|
||||
"status": "success",
|
||||
"hasPagination": True,
|
||||
"url": decoded_url,
|
||||
"hasPagination": has_pagination,
|
||||
"urlTemplate": url_template,
|
||||
"lastPage": last_page.get('lastPage')
|
||||
"lastPage": pagination_info.get('lastPage'),
|
||||
"originalUrl": original_url,
|
||||
"nextPageUrl": next_page_url
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user