This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
from app.utils.browser_utils import safe_browser_operation
|
from app.utils.browser_utils import safe_browser_operation
|
||||||
|
import asyncio
|
||||||
async def visit_url_service(decoded_url):
|
async def visit_url_service(decoded_url):
|
||||||
"""Service function to visit a URL and get its content"""
|
"""Service function to visit a URL and get its content"""
|
||||||
print(f"Visiting URL: {decoded_url}")
|
print(f"Visiting URL: {decoded_url}")
|
||||||
@@ -165,386 +165,374 @@ async def detect_pagination_service(decoded_url):
|
|||||||
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||||
original_url = page.url
|
original_url = page.url
|
||||||
|
|
||||||
print(f"Successfully loaded page: {original_url}")
|
# Check for common pagination indicators
|
||||||
|
pagination_data = await page.evaluate('''() => {
|
||||||
# Analyze the page for pagination information
|
const data = {
|
||||||
pagination_info = await page.evaluate('''() => {
|
hasPagination: false,
|
||||||
// Find the last page number if available
|
paginationType: null,
|
||||||
const findLastPageNumber = () => {
|
paginationElements: [],
|
||||||
// Get all links on the page
|
detectedParameter: null,
|
||||||
const links = Array.from(document.querySelectorAll('a'));
|
lastPageNumber: null
|
||||||
|
|
||||||
// Strategy 1: Find numeric links (page numbers)
|
|
||||||
const numericLinks = links.filter(link => {
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
|
||||||
});
|
|
||||||
|
|
||||||
if (numericLinks.length > 0) {
|
|
||||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
|
||||||
return Math.max(...numericValues);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strategy 2: Look for "last page" link
|
|
||||||
const lastLinks = links.filter(link => {
|
|
||||||
const text = link.innerText.trim().toLowerCase();
|
|
||||||
const classes = (link.className || '').toLowerCase();
|
|
||||||
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
|
|
||||||
|
|
||||||
return (text === 'last' ||
|
|
||||||
classes.includes('last') ||
|
|
||||||
ariaLabel.includes('last') ||
|
|
||||||
link.getAttribute('rel') === 'last');
|
|
||||||
});
|
|
||||||
|
|
||||||
if (lastLinks.length > 0) {
|
|
||||||
const lastLink = lastLinks[0];
|
|
||||||
const href = lastLink.href;
|
|
||||||
|
|
||||||
// Common patterns: page=X, /page/X, etc.
|
|
||||||
const pagePatterns = [
|
|
||||||
/[?&]page=(\d+)/,
|
|
||||||
/[?&]p=(\d+)/,
|
|
||||||
/[?&]pg=(\d+)/,
|
|
||||||
/\/page\/(\d+)/,
|
|
||||||
/\/p\/(\d+)/,
|
|
||||||
/\/paged\/(\d+)/
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const pattern of pagePatterns) {
|
|
||||||
const match = href.match(pattern);
|
|
||||||
if (match && match[1]) {
|
|
||||||
return parseInt(match[1]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strategy 3: Analyze all URLs for page numbers
|
|
||||||
const pageNumbersFromUrls = [];
|
|
||||||
links.forEach(link => {
|
|
||||||
if (!link.href || link.href === '#') return;
|
|
||||||
|
|
||||||
// Check for common pagination URL patterns
|
|
||||||
const patterns = [
|
|
||||||
/[?&]page=(\d+)/,
|
|
||||||
/[?&]p=(\d+)/,
|
|
||||||
/[?&]pg=(\d+)/,
|
|
||||||
/\/page\/(\d+)/,
|
|
||||||
/\/p\/(\d+)/,
|
|
||||||
/\/paged\/(\d+)/,
|
|
||||||
/\/pages\/(\d+)/
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const pattern of patterns) {
|
|
||||||
const match = link.href.match(pattern);
|
|
||||||
if (match && match[1]) {
|
|
||||||
pageNumbersFromUrls.push(parseInt(match[1]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (pageNumbersFromUrls.length > 0) {
|
|
||||||
return Math.max(...pageNumbersFromUrls);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Find a pagination link to click
|
// Look for numbered pagination links (1, 2, 3...)
|
||||||
const findPaginationLink = () => {
|
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
|
||||||
const links = Array.from(document.querySelectorAll('a'));
|
.filter(el => {
|
||||||
|
const text = el.innerText.trim();
|
||||||
|
|
||||||
// Try to find a page "2" link first (most reliable)
|
// check for data-page attribute
|
||||||
const page2Link = links.find(link => {
|
const dataPage = el.getAttribute('data-page');
|
||||||
const text = link.innerText.trim();
|
if (dataPage) {
|
||||||
return text === '2' && link.href && link.href !== '#';
|
return /^[0-9]+$/.test(dataPage);
|
||||||
});
|
|
||||||
|
|
||||||
if (page2Link) {
|
|
||||||
return { element: page2Link, href: page2Link.href, type: 'numeric' };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try common "next page" selectors
|
|
||||||
const nextSelectors = [
|
|
||||||
'a.next',
|
|
||||||
'a.page-next',
|
|
||||||
'a[rel="next"]',
|
|
||||||
'a[aria-label="Next page"]',
|
|
||||||
'a[aria-label="next"]'
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const selector of nextSelectors) {
|
|
||||||
const element = document.querySelector(selector);
|
|
||||||
if (element && element.href && element.href !== '#') {
|
|
||||||
return { element, href: element.href, type: 'next' };
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Look for any link that might be pagination
|
return /^[0-9]+$/.test(text) &&
|
||||||
const paginationLinks = links.filter(link => {
|
(el.tagName === 'A' || el.onclick ||
|
||||||
if (!link.href || link.href === '#') return false;
|
el.closest('button, [role="button"]'));
|
||||||
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
const href = link.href;
|
|
||||||
|
|
||||||
// Check for numeric text or next/prev indicators
|
|
||||||
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
|
|
||||||
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
|
|
||||||
/[»«‹›<>]/.test(text);
|
|
||||||
|
|
||||||
// Check for page parameter in URL
|
|
||||||
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
|
|
||||||
|
|
||||||
return (isNumeric || isNextPrev || hasPageParam);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (paginationLinks.length > 0) {
|
console.log(numberedLinks);
|
||||||
const link = paginationLinks[0];
|
|
||||||
return { element: link, href: link.href, type: 'other' };
|
// Look for next/prev buttons
|
||||||
|
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||||
|
.filter(el => {
|
||||||
|
const text = el.innerText.trim().toLowerCase();
|
||||||
|
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||||
|
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||||
|
|
||||||
|
return text.includes('next') ||
|
||||||
|
text.includes('›') ||
|
||||||
|
text.includes('»') ||
|
||||||
|
text.includes('→') ||
|
||||||
|
ariaLabel.includes('next') ||
|
||||||
|
hasNextIcon;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Check for pagination containers
|
||||||
|
const paginationContainers = Array.from(document.querySelectorAll(
|
||||||
|
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
|
||||||
|
));
|
||||||
|
|
||||||
|
// Collect all potential pagination elements
|
||||||
|
if (numberedLinks.length > 0) {
|
||||||
|
data.hasPagination = true;
|
||||||
|
data.paginationType = 'numbered';
|
||||||
|
|
||||||
|
// Get href attributes or other identifiers from numbered links
|
||||||
|
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
|
||||||
|
return {
|
||||||
|
text: el.innerText.trim(),
|
||||||
|
dataPage: el.getAttribute('data-page'),
|
||||||
|
href: el.tagName === 'A' ? el.href : null,
|
||||||
|
classes: el.className,
|
||||||
|
id: el.id
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Try to find the last page number
|
||||||
|
const pageNumbers = numberedLinks
|
||||||
|
.map(el => parseInt(el.innerText.trim()))
|
||||||
|
.filter(num => !isNaN(num));
|
||||||
|
|
||||||
|
if (pageNumbers.length > 0) {
|
||||||
|
data.lastPageNumber = Math.max(...pageNumbers);
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
// Also look for a "last page" element that might have text like "Last" or "»"
|
||||||
};
|
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
|
||||||
|
.find(el => {
|
||||||
const lastPage = findLastPageNumber();
|
const text = el.innerText.trim().toLowerCase();
|
||||||
const paginationLink = findPaginationLink();
|
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||||
|
return text.includes('last') ||
|
||||||
if (paginationLink) {
|
text === '»' ||
|
||||||
// Click the link
|
ariaLabel.includes('last page');
|
||||||
paginationLink.element.click();
|
|
||||||
return {
|
|
||||||
clicked: true,
|
|
||||||
href: paginationLink.href,
|
|
||||||
type: paginationLink.type,
|
|
||||||
lastPage
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { clicked: false, lastPage };
|
|
||||||
}''')
|
|
||||||
|
|
||||||
# If no pagination was found or clicked
|
|
||||||
if not pagination_info.get('clicked', False):
|
|
||||||
return {
|
|
||||||
"status": "success",
|
|
||||||
"url": decoded_url,
|
|
||||||
"hasPagination": False,
|
|
||||||
"urlTemplate": None,
|
|
||||||
"lastPage": pagination_info.get('lastPage')
|
|
||||||
}
|
|
||||||
|
|
||||||
# Wait for navigation to complete after the click
|
|
||||||
try:
|
|
||||||
await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'})
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Navigation timeout: {e}")
|
|
||||||
|
|
||||||
# Get the new URL after clicking
|
|
||||||
next_page_url = page.url
|
|
||||||
|
|
||||||
# If URL didn't change, pagination might be handled by AJAX
|
|
||||||
if next_page_url == original_url:
|
|
||||||
return {
|
|
||||||
"status": "success",
|
|
||||||
"url": decoded_url,
|
|
||||||
"hasPagination": True,
|
|
||||||
"urlTemplate": "AJAX pagination (URL doesn't change)",
|
|
||||||
"lastPage": pagination_info.get('lastPage')
|
|
||||||
}
|
|
||||||
|
|
||||||
print(f"Navigation successful: {original_url} -> {next_page_url}")
|
|
||||||
|
|
||||||
# Analyze the URL structure to determine pagination pattern
|
|
||||||
url_template = await page.evaluate('''(originalUrl, nextPageUrl) => {
|
|
||||||
// Helper function to parse URL query parameters
|
|
||||||
const parseQueryParams = (url) => {
|
|
||||||
const params = {};
|
|
||||||
if (url.includes('?')) {
|
|
||||||
const queryString = url.split('?')[1].split('#')[0];
|
|
||||||
queryString.split('&').forEach(param => {
|
|
||||||
if (param.includes('=')) {
|
|
||||||
const [key, value] = param.split('=', 2);
|
|
||||||
params[key] = value;
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
|
||||||
return params;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Check for query parameter based pagination
|
if (lastPageElement && lastPageElement.href) {
|
||||||
if (nextPageUrl.includes('?')) {
|
// Try to extract page number from the URL
|
||||||
const originalParams = parseQueryParams(originalUrl);
|
try {
|
||||||
const nextParams = parseQueryParams(nextPageUrl);
|
const url = new URL(lastPageElement.href);
|
||||||
|
// Check common pagination parameters
|
||||||
|
['page', 'p', 'pg'].forEach(param => {
|
||||||
|
if (url.searchParams.has(param)) {
|
||||||
|
const value = parseInt(url.searchParams.get(param));
|
||||||
|
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||||
|
data.lastPageNumber = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Find parameters that changed or were added
|
// Check for path-based pagination (like /page/10)
|
||||||
let paginationParam = null;
|
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
|
||||||
|
if (pathMatch && pathMatch[1]) {
|
||||||
// First check for common pagination parameter names
|
const value = parseInt(pathMatch[1]);
|
||||||
const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber'];
|
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||||
|
data.lastPageNumber = value;
|
||||||
for (const key of commonPaginationParams) {
|
|
||||||
if (key in nextParams &&
|
|
||||||
(!(key in originalParams) || originalParams[key] !== nextParams[key])) {
|
|
||||||
if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) {
|
|
||||||
paginationParam = key;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no common parameter found, check all parameters
|
|
||||||
if (!paginationParam) {
|
|
||||||
for (const [key, value] of Object.entries(nextParams)) {
|
|
||||||
// Check if parameter is new or changed
|
|
||||||
if (!(key in originalParams) || originalParams[key] !== value) {
|
|
||||||
// Check if the value is numeric and could be a page number
|
|
||||||
if (/^\d+$/.test(value) && parseInt(value) > 1) {
|
|
||||||
paginationParam = key;
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error("Error parsing last page URL:", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else if (nextButtons.length > 0) {
|
||||||
|
data.hasPagination = true;
|
||||||
|
data.paginationType = 'next-prev';
|
||||||
|
|
||||||
// If we found a pagination parameter
|
// Get information about next buttons
|
||||||
if (paginationParam) {
|
data.paginationElements = nextButtons.slice(0, 3).map(el => {
|
||||||
const baseUrl = nextPageUrl.split('?')[0];
|
return {
|
||||||
|
text: el.innerText.trim(),
|
||||||
// Reconstruct the URL template with all parameters
|
href: el.tagName === 'A' ? el.href : null,
|
||||||
const queryParts = [];
|
classes: el.className,
|
||||||
for (const [key, value] of Object.entries(nextParams)) {
|
id: el.id
|
||||||
if (key === paginationParam) {
|
};
|
||||||
queryParts.push(`${key}={PAGE_NUMBER}`);
|
|
||||||
} else {
|
|
||||||
queryParts.push(`${key}=${value}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${baseUrl}?${queryParts.join('&')}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for path-based pagination
|
|
||||||
const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/'];
|
|
||||||
for (const pattern of pathPatterns) {
|
|
||||||
if (nextPageUrl.includes(pattern)) {
|
|
||||||
const parts = nextPageUrl.split(pattern);
|
|
||||||
let template = `${parts[0]}${pattern}{PAGE_NUMBER}`;
|
|
||||||
|
|
||||||
// Add any suffix after the page number
|
|
||||||
if (parts.length > 1 && parts[1].includes('/')) {
|
|
||||||
const suffix = parts[1].split('/', 1)[1];
|
|
||||||
if (suffix) {
|
|
||||||
template += `/${suffix}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return template;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we couldn't determine the pattern, try to make an educated guess
|
|
||||||
// For the specific case where a parameter like current_page=2 is added
|
|
||||||
const originalUrlObj = new URL(originalUrl);
|
|
||||||
const nextUrlObj = new URL(nextPageUrl);
|
|
||||||
|
|
||||||
// Check if the paths are the same but query params differ
|
|
||||||
if (originalUrlObj.pathname === nextUrlObj.pathname) {
|
|
||||||
const originalParams = parseQueryParams(originalUrl);
|
|
||||||
const nextParams = parseQueryParams(nextPageUrl);
|
|
||||||
|
|
||||||
// Find parameters that exist in next but not in original
|
|
||||||
const newParams = Object.keys(nextParams).filter(key => !(key in originalParams));
|
|
||||||
|
|
||||||
// If there's exactly one new parameter and it has a numeric value
|
|
||||||
if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) {
|
|
||||||
const paginationParam = newParams[0];
|
|
||||||
const baseUrl = nextPageUrl.split('?')[0];
|
|
||||||
|
|
||||||
// Reconstruct the URL template
|
|
||||||
const queryParts = [];
|
|
||||||
for (const [key, value] of Object.entries(nextParams)) {
|
|
||||||
if (key === paginationParam) {
|
|
||||||
queryParts.push(`${key}={PAGE_NUMBER}`);
|
|
||||||
} else {
|
|
||||||
queryParts.push(`${key}=${value}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${baseUrl}?${queryParts.join('&')}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we still couldn't determine the pattern, return both URLs as examples
|
|
||||||
return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`;
|
|
||||||
}''', original_url, next_page_url)
|
|
||||||
|
|
||||||
# Try to extract last page number from the next page if we didn't find it on the first page
|
|
||||||
if not pagination_info.get('lastPage'):
|
|
||||||
last_page_from_next = await page.evaluate('''() => {
|
|
||||||
// Get all links on the page
|
|
||||||
const links = Array.from(document.querySelectorAll('a'));
|
|
||||||
|
|
||||||
// Strategy 1: Find numeric links (page numbers)
|
|
||||||
const numericLinks = links.filter(link => {
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (numericLinks.length > 0) {
|
// Check for URL parameters that might indicate pagination
|
||||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
const currentUrl = window.location.href;
|
||||||
return Math.max(...numericValues);
|
const urlParams = new URL(currentUrl).searchParams;
|
||||||
|
|
||||||
|
// Common pagination parameters
|
||||||
|
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
|
||||||
|
|
||||||
|
for (const param of paginationParams) {
|
||||||
|
if (urlParams.has(param)) {
|
||||||
|
data.detectedParameter = {
|
||||||
|
name: param,
|
||||||
|
value: urlParams.get(param)
|
||||||
|
};
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Strategy 2: Analyze all URLs for page numbers
|
return data;
|
||||||
const pageNumbersFromUrls = [];
|
}''')
|
||||||
links.forEach(link => {
|
|
||||||
if (!link.href || link.href === '#') return;
|
|
||||||
|
|
||||||
// Check for common pagination URL patterns
|
# If pagination is detected, try to navigate to the next page by clicking
|
||||||
const patterns = [
|
next_page_url = None
|
||||||
/[?&]page=(\d+)/,
|
pagination_parameter = None
|
||||||
/[?&]p=(\d+)/,
|
url_template = None
|
||||||
/[?&]pg=(\d+)/,
|
|
||||||
/\/page\/(\d+)/,
|
|
||||||
/\/p\/(\d+)/,
|
|
||||||
/\/paged\/(\d+)/,
|
|
||||||
/\/pages\/(\d+)/
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const pattern of patterns) {
|
if pagination_data['hasPagination']:
|
||||||
const match = link.href.match(pattern);
|
print("Pagination detected, attempting to click on a pagination element")
|
||||||
if (match && match[1]) {
|
|
||||||
pageNumbersFromUrls.push(parseInt(match[1]));
|
# Always try to click on a pagination element, regardless of type
|
||||||
}
|
clicked = False
|
||||||
|
|
||||||
|
# First try to click on a numbered link (preferably "2" if we're on page 1)
|
||||||
|
try:
|
||||||
|
clicked = await page.evaluate('''() => {
|
||||||
|
// First try to find and click on a "2" link or button
|
||||||
|
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||||
|
.filter(el => {
|
||||||
|
// Check for text content "2"
|
||||||
|
if (el.innerText.trim() === '2') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for href with page=2 or similar (for anchor elements)
|
||||||
|
if (el.tagName === 'A' && el.href) {
|
||||||
|
try {
|
||||||
|
const url = new URL(el.href, window.location.origin);
|
||||||
|
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
|
||||||
|
'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||||
|
|
||||||
|
for (const param of paginationParams) {
|
||||||
|
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for path-based pagination like /page/2/
|
||||||
|
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
||||||
|
if (pathMatch) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for data attributes that might indicate pagination
|
||||||
|
if (el.getAttribute('data-page') === '2' ||
|
||||||
|
el.getAttribute('data-pagenumber') === '2' ||
|
||||||
|
el.getAttribute('data-page-number') === '2') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (page2Elements.length > 0) {
|
||||||
|
console.log("Clicking on page 2 element");
|
||||||
|
page2Elements[0].click();
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
|
||||||
if (pageNumbersFromUrls.length > 0) {
|
// If no "2" link found, try any numbered link or button
|
||||||
return Math.max(...pageNumbersFromUrls);
|
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||||
|
.filter(el => /^\d+$/.test(el.innerText.trim()));
|
||||||
|
|
||||||
|
if (numberedElements.length > 0) {
|
||||||
|
// Sort by number and get the second one (likely page 2)
|
||||||
|
const sorted = numberedElements.sort((a, b) => {
|
||||||
|
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get the second element if available (page 2), otherwise the first one
|
||||||
|
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
|
||||||
|
console.log("Clicking on numbered element: " + elementToClick.innerText);
|
||||||
|
elementToClick.click();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no numbered links, try next button
|
||||||
|
const nextTexts = ['next', '›', '»', '→'];
|
||||||
|
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||||
|
.filter(el => {
|
||||||
|
const text = el.textContent.trim().toLowerCase();
|
||||||
|
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||||
|
return nextTexts.some(t => text.includes(t)) ||
|
||||||
|
ariaLabel.includes('next') ||
|
||||||
|
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||||
|
});
|
||||||
|
|
||||||
|
if (nextElements.length > 0) {
|
||||||
|
console.log("Clicking on next button");
|
||||||
|
nextElements[0].click();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}''')
|
||||||
|
|
||||||
|
if clicked:
|
||||||
|
print("Successfully clicked on pagination element")
|
||||||
|
# Wait for navigation to complete
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
next_page_url = page.url
|
||||||
|
else:
|
||||||
|
print("No clickable pagination element found")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error clicking on pagination element: {e}")
|
||||||
|
|
||||||
|
# If we successfully navigated to the next page, analyze the URL difference
|
||||||
|
if next_page_url:
|
||||||
|
print('Searching for pagination parameter')
|
||||||
|
# Parse both URLs
|
||||||
|
original_parsed = await page.evaluate(f'''(originalUrl) => {{
|
||||||
|
const original = new URL(originalUrl);
|
||||||
|
const current = new URL(window.location.href);
|
||||||
|
|
||||||
|
// Check for differences in query parameters
|
||||||
|
let paramDiff = null;
|
||||||
|
|
||||||
|
// Common pagination parameters to check
|
||||||
|
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||||
|
|
||||||
|
for (const param of paginationParams) {{
|
||||||
|
const originalValue = original.searchParams.get(param);
|
||||||
|
const currentValue = current.searchParams.get(param);
|
||||||
|
|
||||||
|
if (originalValue !== currentValue && currentValue !== null) {{
|
||||||
|
paramDiff = {{
|
||||||
|
name: param,
|
||||||
|
originalValue: originalValue,
|
||||||
|
currentValue: currentValue
|
||||||
|
}};
|
||||||
|
break;
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
|
||||||
|
// Check for path differences (like /page/1 vs /page/2)
|
||||||
|
const originalPath = original.pathname;
|
||||||
|
const currentPath = current.pathname;
|
||||||
|
|
||||||
|
let pathDiff = null;
|
||||||
|
if (originalPath !== currentPath) {{
|
||||||
|
const originalSegments = originalPath.split('/').filter(s => s);
|
||||||
|
const currentSegments = currentPath.split('/').filter(s => s);
|
||||||
|
|
||||||
|
// Find the segment that changed
|
||||||
|
if (originalSegments.length === currentSegments.length) {{
|
||||||
|
for (let i = 0; i < originalSegments.length; i++) {{
|
||||||
|
if (originalSegments[i] !== currentSegments[i]) {{
|
||||||
|
// Check if the difference is numeric
|
||||||
|
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
||||||
|
pathDiff = {{
|
||||||
|
index: i,
|
||||||
|
originalValue: originalSegments[i],
|
||||||
|
currentValue: currentSegments[i]
|
||||||
|
}};
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
|
||||||
|
return {{
|
||||||
|
paramDiff,
|
||||||
|
pathDiff,
|
||||||
|
originalUrl: originalUrl,
|
||||||
|
currentUrl: window.location.href
|
||||||
|
}};
|
||||||
|
}}''', original_url)
|
||||||
|
|
||||||
|
# Determine the pagination parameter and create URL template
|
||||||
|
if original_parsed['paramDiff']:
|
||||||
|
param_name = original_parsed['paramDiff']['name']
|
||||||
|
pagination_parameter = {
|
||||||
|
'type': 'query',
|
||||||
|
'name': param_name,
|
||||||
|
'value': original_parsed['paramDiff']['currentValue']
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
# Create URL template for query parameter
|
||||||
}''')
|
url_obj = await page.evaluate(f'''(url, paramName) => {{
|
||||||
|
const urlObj = new URL(url);
|
||||||
|
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||||
|
return urlObj.toString();
|
||||||
|
}}''', original_url, param_name)
|
||||||
|
|
||||||
if last_page_from_next:
|
url_template = url_obj
|
||||||
pagination_info['lastPage'] = last_page_from_next
|
|
||||||
|
|
||||||
# Check if the pattern is unclear
|
elif original_parsed['pathDiff']:
|
||||||
has_pagination = True
|
path_index = original_parsed['pathDiff']['index']
|
||||||
url_string_template = str(url_template)
|
pagination_parameter = {
|
||||||
if url_string_template and (url_string_template.startswith("Pattern unclear") or url_string_template.startswith("AJAX pagination")):
|
'type': 'path',
|
||||||
has_pagination = False
|
'index': path_index,
|
||||||
url_template = None
|
'value': original_parsed['pathDiff']['currentValue']
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create URL template for path parameter
|
||||||
|
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
||||||
|
const urlObj = new URL(url);
|
||||||
|
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||||
|
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||||
|
urlObj.pathname = '/' + pathSegments.join('/');
|
||||||
|
return urlObj.toString();
|
||||||
|
}}''', original_url, path_index)
|
||||||
|
|
||||||
|
if url_template:
|
||||||
|
# Decode URL-encoded characters in the template
|
||||||
|
import urllib.parse
|
||||||
|
url_template = urllib.parse.unquote(url_template)
|
||||||
|
|
||||||
|
# If we couldn't determine the URL template from navigation, try to infer it
|
||||||
|
if not url_template and pagination_data['detectedParameter']:
|
||||||
|
param_name = pagination_data['detectedParameter']['name']
|
||||||
|
url_template = await page.evaluate(f'''(url, paramName) => {{
|
||||||
|
const urlObj = new URL(url);
|
||||||
|
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||||
|
return urlObj.toString();
|
||||||
|
}}''', original_url, param_name)
|
||||||
|
|
||||||
|
# Return the pagination detection results with a simplified structure
|
||||||
result = {
|
result = {
|
||||||
"status": "success",
|
"status": "success",
|
||||||
"url": decoded_url,
|
"hasPagination": pagination_data['hasPagination'],
|
||||||
"hasPagination": has_pagination,
|
|
||||||
"urlTemplate": url_template,
|
"urlTemplate": url_template,
|
||||||
"lastPage": pagination_info.get('lastPage'),
|
"lastPage": pagination_data['lastPageNumber']
|
||||||
"originalUrl": original_url,
|
|
||||||
"nextPageUrl": next_page_url
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@@ -553,11 +541,10 @@ async def detect_pagination_service(decoded_url):
|
|||||||
print(f"Error during pagination detection: {e}")
|
print(f"Error during pagination detection: {e}")
|
||||||
return {
|
return {
|
||||||
"status": "error",
|
"status": "error",
|
||||||
"url": decoded_url,
|
|
||||||
"error": str(e),
|
|
||||||
"hasPagination": False,
|
"hasPagination": False,
|
||||||
"urlTemplate": None,
|
"urlTemplate": None,
|
||||||
"lastPage": None
|
"lastPage": None,
|
||||||
|
"error": str(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
# Perform the operation
|
# Perform the operation
|
||||||
|
|||||||
Reference in New Issue
Block a user