This commit is contained in:
@@ -153,153 +153,173 @@ async def detect_pagination_service(decoded_url):
|
||||
async def pagination_operation(page):
|
||||
try:
|
||||
# Navigate to the URL
|
||||
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||
original_url = page.url
|
||||
try:
|
||||
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||
except Exception as e:
|
||||
print(f"Error navigating to URL: {e}")
|
||||
# Try to get the current URL even if navigation failed
|
||||
try:
|
||||
original_url = page.url
|
||||
except:
|
||||
original_url = decoded_url
|
||||
else:
|
||||
original_url = page.url
|
||||
|
||||
# Check for common pagination indicators
|
||||
pagination_data = await page.evaluate('''() => {
|
||||
const data = {
|
||||
hasPagination: false,
|
||||
paginationType: null,
|
||||
paginationElements: [],
|
||||
detectedParameter: null,
|
||||
lastPageNumber: null
|
||||
};
|
||||
try:
|
||||
pagination_data = await page.evaluate('''() => {
|
||||
const data = {
|
||||
hasPagination: false,
|
||||
paginationType: null,
|
||||
paginationElements: [],
|
||||
detectedParameter: null,
|
||||
lastPageNumber: null
|
||||
};
|
||||
|
||||
// Look for numbered pagination links (1, 2, 3...)
|
||||
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
|
||||
.filter(el => {
|
||||
const text = el.innerText.trim();
|
||||
// Look for numbered pagination links (1, 2, 3...)
|
||||
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
|
||||
.filter(el => {
|
||||
const text = el.innerText.trim();
|
||||
|
||||
// check for data-page attribute
|
||||
const dataPage = el.getAttribute('data-page');
|
||||
if (dataPage) {
|
||||
return /^[0-9]+$/.test(dataPage);
|
||||
}
|
||||
// check for data-page attribute
|
||||
const dataPage = el.getAttribute('data-page');
|
||||
if (dataPage) {
|
||||
return /^[0-9]+$/.test(dataPage);
|
||||
}
|
||||
|
||||
return /^[0-9]+$/.test(text) &&
|
||||
(el.tagName === 'A' || el.onclick ||
|
||||
el.closest('button, [role="button"]'));
|
||||
});
|
||||
|
||||
console.log(numberedLinks);
|
||||
|
||||
// Look for next/prev buttons
|
||||
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||
.filter(el => {
|
||||
const text = el.innerText.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||
|
||||
return text.includes('next') ||
|
||||
text.includes('›') ||
|
||||
text.includes('»') ||
|
||||
text.includes('→') ||
|
||||
ariaLabel.includes('next') ||
|
||||
hasNextIcon;
|
||||
});
|
||||
|
||||
// Check for pagination containers
|
||||
const paginationContainers = Array.from(document.querySelectorAll(
|
||||
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
|
||||
));
|
||||
|
||||
// Collect all potential pagination elements
|
||||
if (numberedLinks.length > 0) {
|
||||
data.hasPagination = true;
|
||||
data.paginationType = 'numbered';
|
||||
|
||||
// Get href attributes or other identifiers from numbered links
|
||||
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
|
||||
return {
|
||||
text: el.innerText.trim(),
|
||||
dataPage: el.getAttribute('data-page'),
|
||||
href: el.tagName === 'A' ? el.href : null,
|
||||
classes: el.className,
|
||||
id: el.id
|
||||
};
|
||||
});
|
||||
|
||||
// Try to find the last page number
|
||||
const pageNumbers = numberedLinks
|
||||
.map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
|
||||
.filter(num => !isNaN(num));
|
||||
|
||||
if (pageNumbers.length > 0) {
|
||||
data.lastPageNumber = Math.max(...pageNumbers);
|
||||
}
|
||||
|
||||
// Also look for a "last page" element that might have text like "Last" or "»"
|
||||
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
|
||||
.find(el => {
|
||||
const text = el.innerText.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
return text.includes('last') ||
|
||||
text === '»' ||
|
||||
ariaLabel.includes('last page');
|
||||
return /^[0-9]+$/.test(text) &&
|
||||
(el.tagName === 'A' || el.onclick ||
|
||||
el.closest('button, [role="button"]'));
|
||||
});
|
||||
|
||||
if (lastPageElement && lastPageElement.href) {
|
||||
// Try to extract page number from the URL
|
||||
try {
|
||||
const url = new URL(lastPageElement.href);
|
||||
// Check common pagination parameters
|
||||
['page', 'p', 'pg'].forEach(param => {
|
||||
if (url.searchParams.has(param)) {
|
||||
const value = parseInt(url.searchParams.get(param));
|
||||
console.log(numberedLinks);
|
||||
|
||||
// Look for next/prev buttons
|
||||
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||
.filter(el => {
|
||||
const text = el.innerText.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||
|
||||
return text.includes('next') ||
|
||||
text.includes('›') ||
|
||||
text.includes('»') ||
|
||||
text.includes('→') ||
|
||||
ariaLabel.includes('next') ||
|
||||
hasNextIcon;
|
||||
});
|
||||
|
||||
// Check for pagination containers
|
||||
const paginationContainers = Array.from(document.querySelectorAll(
|
||||
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
|
||||
));
|
||||
|
||||
// Collect all potential pagination elements
|
||||
if (numberedLinks.length > 0) {
|
||||
data.hasPagination = true;
|
||||
data.paginationType = 'numbered';
|
||||
|
||||
// Get href attributes or other identifiers from numbered links
|
||||
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
|
||||
return {
|
||||
text: el.innerText.trim(),
|
||||
dataPage: el.getAttribute('data-page'),
|
||||
href: el.tagName === 'A' ? el.href : null,
|
||||
classes: el.className,
|
||||
id: el.id
|
||||
};
|
||||
});
|
||||
|
||||
// Try to find the last page number
|
||||
const pageNumbers = numberedLinks
|
||||
.map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
|
||||
.filter(num => !isNaN(num));
|
||||
|
||||
if (pageNumbers.length > 0) {
|
||||
data.lastPageNumber = Math.max(...pageNumbers);
|
||||
}
|
||||
|
||||
// Also look for a "last page" element that might have text like "Last" or "»"
|
||||
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
|
||||
.find(el => {
|
||||
const text = el.innerText.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
return text.includes('last') ||
|
||||
text === '»' ||
|
||||
ariaLabel.includes('last page');
|
||||
});
|
||||
|
||||
if (lastPageElement && lastPageElement.href) {
|
||||
// Try to extract page number from the URL
|
||||
try {
|
||||
const url = new URL(lastPageElement.href);
|
||||
// Check common pagination parameters
|
||||
['page', 'p', 'pg'].forEach(param => {
|
||||
if (url.searchParams.has(param)) {
|
||||
const value = parseInt(url.searchParams.get(param));
|
||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||
data.lastPageNumber = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Check for path-based pagination (like /page/10)
|
||||
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
|
||||
if (pathMatch && pathMatch[1]) {
|
||||
const value = parseInt(pathMatch[1]);
|
||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||
data.lastPageNumber = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Check for path-based pagination (like /page/10)
|
||||
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
|
||||
if (pathMatch && pathMatch[1]) {
|
||||
const value = parseInt(pathMatch[1]);
|
||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||
data.lastPageNumber = value;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Error parsing last page URL:", e);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Error parsing last page URL:", e);
|
||||
}
|
||||
} else if (nextButtons.length > 0) {
|
||||
data.hasPagination = true;
|
||||
data.paginationType = 'next-prev';
|
||||
|
||||
// Get information about next buttons
|
||||
data.paginationElements = nextButtons.slice(0, 3).map(el => {
|
||||
return {
|
||||
text: el.innerText.trim(),
|
||||
href: el.tagName === 'A' ? el.href : null,
|
||||
classes: el.className,
|
||||
id: el.id
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// Check for URL parameters that might indicate pagination
|
||||
const currentUrl = window.location.href;
|
||||
const urlParams = new URL(currentUrl).searchParams;
|
||||
|
||||
// Common pagination parameters
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
|
||||
|
||||
for (const param of paginationParams) {
|
||||
if (urlParams.has(param)) {
|
||||
data.detectedParameter = {
|
||||
name: param,
|
||||
value: urlParams.get(param)
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (nextButtons.length > 0) {
|
||||
data.hasPagination = true;
|
||||
data.paginationType = 'next-prev';
|
||||
|
||||
// Get information about next buttons
|
||||
data.paginationElements = nextButtons.slice(0, 3).map(el => {
|
||||
return {
|
||||
text: el.innerText.trim(),
|
||||
href: el.tagName === 'A' ? el.href : null,
|
||||
classes: el.className,
|
||||
id: el.id
|
||||
};
|
||||
});
|
||||
return data;
|
||||
}''')
|
||||
except Exception as e:
|
||||
print(f"Error during JavaScript evaluation for pagination detection: {e}")
|
||||
# Return a safe default if JavaScript evaluation fails
|
||||
pagination_data = {
|
||||
'hasPagination': False,
|
||||
'paginationType': None,
|
||||
'paginationElements': [],
|
||||
'detectedParameter': None,
|
||||
'lastPageNumber': None
|
||||
}
|
||||
|
||||
// Check for URL parameters that might indicate pagination
|
||||
const currentUrl = window.location.href;
|
||||
const urlParams = new URL(currentUrl).searchParams;
|
||||
|
||||
// Common pagination parameters
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
|
||||
|
||||
for (const param of paginationParams) {
|
||||
if (urlParams.has(param)) {
|
||||
data.detectedParameter = {
|
||||
name: param,
|
||||
value: urlParams.get(param)
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}''')
|
||||
|
||||
# If pagination is detected, try to navigate to the next page by clicking
|
||||
next_page_url = None
|
||||
pagination_parameter = None
|
||||
@@ -315,86 +335,91 @@ async def detect_pagination_service(decoded_url):
|
||||
# First try to click on a numbered link (preferably "2" if we're on page 1)
|
||||
try:
|
||||
clicked = await page.evaluate('''() => {
|
||||
// First try to find and click on a "2" link or button
|
||||
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||
.filter(el => {
|
||||
// Check for text content "2"
|
||||
if (el.innerText.trim() === '2') {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
// First try to find and click on a "2" link or button
|
||||
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||
.filter(el => {
|
||||
// Check for text content "2"
|
||||
if (el.innerText.trim() === '2') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for href with page=2 or similar (for anchor elements)
|
||||
if (el.tagName === 'A' && el.href) {
|
||||
try {
|
||||
const url = new URL(el.href, window.location.origin);
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
|
||||
'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||
// Check for href with page=2 or similar (for anchor elements)
|
||||
if (el.tagName === 'A' && el.href) {
|
||||
try {
|
||||
const url = new URL(el.href, window.location.origin);
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
|
||||
'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||
|
||||
for (const param of paginationParams) {
|
||||
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
|
||||
for (const param of paginationParams) {
|
||||
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for path-based pagination like /page/2/
|
||||
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
||||
if (pathMatch) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
// Check for path-based pagination like /page/2/
|
||||
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
||||
if (pathMatch) {
|
||||
return true;
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
// Check for data attributes that might indicate pagination
|
||||
if (el.getAttribute('data-page') === '2' ||
|
||||
el.getAttribute('data-pagenumber') === '2' ||
|
||||
el.getAttribute('data-page-number') === '2') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for data attributes that might indicate pagination
|
||||
if (el.getAttribute('data-page') === '2' ||
|
||||
el.getAttribute('data-pagenumber') === '2' ||
|
||||
el.getAttribute('data-page-number') === '2') {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
return false;
|
||||
});
|
||||
if (page2Elements.length > 0) {
|
||||
console.log("Clicking on page 2 element");
|
||||
page2Elements[0].click();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (page2Elements.length > 0) {
|
||||
console.log("Clicking on page 2 element");
|
||||
page2Elements[0].click();
|
||||
return true;
|
||||
// If no "2" link found, try any numbered link or button
|
||||
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||
.filter(el => /^\d+$/.test(el.innerText.trim()));
|
||||
|
||||
if (numberedElements.length > 0) {
|
||||
// Sort by number and get the second one (likely page 2)
|
||||
const sorted = numberedElements.sort((a, b) => {
|
||||
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
|
||||
});
|
||||
|
||||
// Get the second element if available (page 2), otherwise the first one
|
||||
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
|
||||
console.log("Clicking on numbered element: " + elementToClick.innerText);
|
||||
elementToClick.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
// If no numbered links, try next button
|
||||
const nextTexts = ['next', '›', '»', '→'];
|
||||
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||
.filter(el => {
|
||||
const text = el.textContent.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
return nextTexts.some(t => text.includes(t)) ||
|
||||
ariaLabel.includes('next') ||
|
||||
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||
});
|
||||
|
||||
if (nextElements.length > 0) {
|
||||
console.log("Clicking on next button");
|
||||
nextElements[0].click();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.error("Error during pagination click operation:", error);
|
||||
return false;
|
||||
}
|
||||
|
||||
// If no "2" link found, try any numbered link or button
|
||||
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||
.filter(el => /^\d+$/.test(el.innerText.trim()));
|
||||
|
||||
if (numberedElements.length > 0) {
|
||||
// Sort by number and get the second one (likely page 2)
|
||||
const sorted = numberedElements.sort((a, b) => {
|
||||
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
|
||||
});
|
||||
|
||||
// Get the second element if available (page 2), otherwise the first one
|
||||
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
|
||||
console.log("Clicking on numbered element: " + elementToClick.innerText);
|
||||
elementToClick.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
// If no numbered links, try next button
|
||||
const nextTexts = ['next', '›', '»', '→'];
|
||||
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||
.filter(el => {
|
||||
const text = el.textContent.trim().toLowerCase();
|
||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||
return nextTexts.some(t => text.includes(t)) ||
|
||||
ariaLabel.includes('next') ||
|
||||
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||
});
|
||||
|
||||
if (nextElements.length > 0) {
|
||||
console.log("Clicking on next button");
|
||||
nextElements[0].click();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}''')
|
||||
|
||||
if clicked:
|
||||
@@ -407,126 +432,166 @@ async def detect_pagination_service(decoded_url):
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error clicking on pagination element: {e}")
|
||||
# Continue with the process even if clicking fails
|
||||
|
||||
# If we successfully navigated to the next page, analyze the URL difference
|
||||
if next_page_url:
|
||||
print('Searching for pagination parameter')
|
||||
# Parse both URLs
|
||||
original_parsed = await page.evaluate(f'''(originalUrl) => {{
|
||||
const original = new URL(originalUrl);
|
||||
const current = new URL(window.location.href);
|
||||
try:
|
||||
original_parsed = await page.evaluate(f'''(originalUrl) => {{
|
||||
try {{
|
||||
const original = new URL(originalUrl);
|
||||
const current = new URL(window.location.href);
|
||||
|
||||
// Check for differences in query parameters
|
||||
let paramDiff = null;
|
||||
// Check for differences in query parameters
|
||||
let paramDiff = null;
|
||||
|
||||
// Common pagination parameters to check
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||
// Common pagination parameters to check
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||
|
||||
for (const param of paginationParams) {{
|
||||
const originalValue = original.searchParams.get(param);
|
||||
const currentValue = current.searchParams.get(param);
|
||||
for (const param of paginationParams) {{
|
||||
const originalValue = original.searchParams.get(param);
|
||||
const currentValue = current.searchParams.get(param);
|
||||
|
||||
if (originalValue !== currentValue && currentValue !== null) {{
|
||||
paramDiff = {{
|
||||
name: param,
|
||||
originalValue: originalValue,
|
||||
currentValue: currentValue
|
||||
}};
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
if (originalValue !== currentValue && currentValue !== null) {{
|
||||
paramDiff = {{
|
||||
name: param,
|
||||
originalValue: originalValue,
|
||||
currentValue: currentValue
|
||||
}};
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
|
||||
// Check for path differences (like /page/1 vs /page/2)
|
||||
const originalPath = original.pathname;
|
||||
const currentPath = current.pathname;
|
||||
// Check for path differences (like /page/1 vs /page/2)
|
||||
const originalPath = original.pathname;
|
||||
const currentPath = current.pathname;
|
||||
|
||||
let pathDiff = null;
|
||||
if (originalPath !== currentPath) {{
|
||||
const originalSegments = originalPath.split('/').filter(s => s);
|
||||
const currentSegments = currentPath.split('/').filter(s => s);
|
||||
let pathDiff = null;
|
||||
if (originalPath !== currentPath) {{
|
||||
const originalSegments = originalPath.split('/').filter(s => s);
|
||||
const currentSegments = currentPath.split('/').filter(s => s);
|
||||
|
||||
// Find the segment that changed
|
||||
if (originalSegments.length === currentSegments.length) {{
|
||||
for (let i = 0; i < originalSegments.length; i++) {{
|
||||
if (originalSegments[i] !== currentSegments[i]) {{
|
||||
// Check if the difference is numeric
|
||||
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
||||
pathDiff = {{
|
||||
index: i,
|
||||
originalValue: originalSegments[i],
|
||||
currentValue: currentSegments[i]
|
||||
}};
|
||||
// Find the segment that changed
|
||||
if (originalSegments.length === currentSegments.length) {{
|
||||
for (let i = 0; i < originalSegments.length; i++) {{
|
||||
if (originalSegments[i] !== currentSegments[i]) {{
|
||||
// Check if the difference is numeric
|
||||
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
||||
pathDiff = {{
|
||||
index: i,
|
||||
originalValue: originalSegments[i],
|
||||
currentValue: currentSegments[i]
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
return {{
|
||||
paramDiff,
|
||||
pathDiff,
|
||||
originalUrl: originalUrl,
|
||||
currentUrl: window.location.href
|
||||
}};
|
||||
}} catch (error) {{
|
||||
console.error("Error during URL analysis:", error);
|
||||
return {{
|
||||
paramDiff: null,
|
||||
pathDiff: null,
|
||||
originalUrl: originalUrl,
|
||||
currentUrl: window.location.href,
|
||||
error: error.message
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
|
||||
return {{
|
||||
paramDiff,
|
||||
pathDiff,
|
||||
originalUrl: originalUrl,
|
||||
currentUrl: window.location.href
|
||||
}};
|
||||
}}''', original_url)
|
||||
|
||||
# Determine the pagination parameter and create URL template
|
||||
print(original_parsed)
|
||||
if original_parsed['paramDiff']:
|
||||
param_name = original_parsed['paramDiff']['name']
|
||||
pagination_parameter = {
|
||||
'type': 'query',
|
||||
'name': param_name,
|
||||
'value': original_parsed['paramDiff']['currentValue']
|
||||
}}''', original_url)
|
||||
except Exception as e:
|
||||
print(f"Error during URL analysis: {e}")
|
||||
original_parsed = {
|
||||
'paramDiff': None,
|
||||
'pathDiff': None,
|
||||
'originalUrl': original_url,
|
||||
'currentUrl': next_page_url,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
|
||||
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
|
||||
curr_val = original_parsed['paramDiff']['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
# Determine the pagination parameter and create URL template
|
||||
print(original_parsed)
|
||||
if original_parsed['paramDiff']:
|
||||
param_name = original_parsed['paramDiff']['name']
|
||||
pagination_parameter = {
|
||||
'type': 'query',
|
||||
'name': param_name,
|
||||
'value': original_parsed['paramDiff']['currentValue']
|
||||
}
|
||||
|
||||
# Create URL template for query parameter
|
||||
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
|
||||
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
|
||||
curr_val = original_parsed['paramDiff']['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for query parameter
|
||||
try:
|
||||
url_obj = await page.evaluate(f'''(url, paramName) => {{
|
||||
const urlObj = new URL(url);
|
||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||
return urlObj.toString();
|
||||
try {{
|
||||
const urlObj = new URL(url);
|
||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
console.error("Error creating URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, param_name)
|
||||
|
||||
url_template = url_obj
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for query parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
elif original_parsed['pathDiff']:
|
||||
path_index = original_parsed['pathDiff']['index']
|
||||
pagination_parameter = {
|
||||
'type': 'path',
|
||||
'index': path_index,
|
||||
'value': original_parsed['pathDiff']['currentValue']
|
||||
}
|
||||
elif original_parsed['pathDiff']:
|
||||
path_index = original_parsed['pathDiff']['index']
|
||||
pagination_parameter = {
|
||||
'type': 'path',
|
||||
'index': path_index,
|
||||
'value': original_parsed['pathDiff']['currentValue']
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
||||
orig_val = original_parsed['pathDiff']['originalValue']
|
||||
curr_val = original_parsed['pathDiff']['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
||||
orig_val = original_parsed['pathDiff']['originalValue']
|
||||
curr_val = original_parsed['pathDiff']['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for path parameter
|
||||
# Create URL template for path parameter
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
||||
const urlObj = new URL(url);
|
||||
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||
urlObj.pathname = '/' + pathSegments.join('/');
|
||||
return urlObj.toString();
|
||||
try {{
|
||||
const urlObj = new URL(url);
|
||||
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||
urlObj.pathname = '/' + pathSegments.join('/');
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
console.error("Error creating path URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, path_index)
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for path parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
if url_template:
|
||||
# Decode URL-encoded characters in the template
|
||||
@@ -536,11 +601,20 @@ async def detect_pagination_service(decoded_url):
|
||||
# If we couldn't determine the URL template from navigation, try to infer it
|
||||
if not url_template and pagination_data['detectedParameter']:
|
||||
param_name = pagination_data['detectedParameter']['name']
|
||||
url_template = await page.evaluate(f'''(url, paramName) => {{
|
||||
const urlObj = new URL(url);
|
||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||
return urlObj.toString();
|
||||
}}''', original_url, param_name)
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, paramName) => {{
|
||||
try {{
|
||||
const urlObj = new URL(url);
|
||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
console.error("Error creating inferred URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, param_name)
|
||||
except Exception as e:
|
||||
print(f"Error creating inferred URL template: {e}")
|
||||
url_template = None
|
||||
|
||||
# Return the pagination detection results with a simplified structure
|
||||
result = {
|
||||
|
||||
Reference in New Issue
Block a user