This commit is contained in:
@@ -153,153 +153,173 @@ async def detect_pagination_service(decoded_url):
|
|||||||
async def pagination_operation(page):
|
async def pagination_operation(page):
|
||||||
try:
|
try:
|
||||||
# Navigate to the URL
|
# Navigate to the URL
|
||||||
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
try:
|
||||||
original_url = page.url
|
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error navigating to URL: {e}")
|
||||||
|
# Try to get the current URL even if navigation failed
|
||||||
|
try:
|
||||||
|
original_url = page.url
|
||||||
|
except:
|
||||||
|
original_url = decoded_url
|
||||||
|
else:
|
||||||
|
original_url = page.url
|
||||||
|
|
||||||
# Check for common pagination indicators
|
# Check for common pagination indicators
|
||||||
pagination_data = await page.evaluate('''() => {
|
try:
|
||||||
const data = {
|
pagination_data = await page.evaluate('''() => {
|
||||||
hasPagination: false,
|
const data = {
|
||||||
paginationType: null,
|
hasPagination: false,
|
||||||
paginationElements: [],
|
paginationType: null,
|
||||||
detectedParameter: null,
|
paginationElements: [],
|
||||||
lastPageNumber: null
|
detectedParameter: null,
|
||||||
};
|
lastPageNumber: null
|
||||||
|
};
|
||||||
|
|
||||||
// Look for numbered pagination links (1, 2, 3...)
|
// Look for numbered pagination links (1, 2, 3...)
|
||||||
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
|
const numberedLinks = Array.from(document.querySelectorAll('a, button, span'))
|
||||||
.filter(el => {
|
.filter(el => {
|
||||||
const text = el.innerText.trim();
|
const text = el.innerText.trim();
|
||||||
|
|
||||||
// check for data-page attribute
|
// check for data-page attribute
|
||||||
const dataPage = el.getAttribute('data-page');
|
const dataPage = el.getAttribute('data-page');
|
||||||
if (dataPage) {
|
if (dataPage) {
|
||||||
return /^[0-9]+$/.test(dataPage);
|
return /^[0-9]+$/.test(dataPage);
|
||||||
}
|
}
|
||||||
|
|
||||||
return /^[0-9]+$/.test(text) &&
|
return /^[0-9]+$/.test(text) &&
|
||||||
(el.tagName === 'A' || el.onclick ||
|
(el.tagName === 'A' || el.onclick ||
|
||||||
el.closest('button, [role="button"]'));
|
el.closest('button, [role="button"]'));
|
||||||
});
|
|
||||||
|
|
||||||
console.log(numberedLinks);
|
|
||||||
|
|
||||||
// Look for next/prev buttons
|
|
||||||
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
|
||||||
.filter(el => {
|
|
||||||
const text = el.innerText.trim().toLowerCase();
|
|
||||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
|
||||||
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
|
||||||
|
|
||||||
return text.includes('next') ||
|
|
||||||
text.includes('›') ||
|
|
||||||
text.includes('»') ||
|
|
||||||
text.includes('→') ||
|
|
||||||
ariaLabel.includes('next') ||
|
|
||||||
hasNextIcon;
|
|
||||||
});
|
|
||||||
|
|
||||||
// Check for pagination containers
|
|
||||||
const paginationContainers = Array.from(document.querySelectorAll(
|
|
||||||
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
|
|
||||||
));
|
|
||||||
|
|
||||||
// Collect all potential pagination elements
|
|
||||||
if (numberedLinks.length > 0) {
|
|
||||||
data.hasPagination = true;
|
|
||||||
data.paginationType = 'numbered';
|
|
||||||
|
|
||||||
// Get href attributes or other identifiers from numbered links
|
|
||||||
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
|
|
||||||
return {
|
|
||||||
text: el.innerText.trim(),
|
|
||||||
dataPage: el.getAttribute('data-page'),
|
|
||||||
href: el.tagName === 'A' ? el.href : null,
|
|
||||||
classes: el.className,
|
|
||||||
id: el.id
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
// Try to find the last page number
|
|
||||||
const pageNumbers = numberedLinks
|
|
||||||
.map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
|
|
||||||
.filter(num => !isNaN(num));
|
|
||||||
|
|
||||||
if (pageNumbers.length > 0) {
|
|
||||||
data.lastPageNumber = Math.max(...pageNumbers);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also look for a "last page" element that might have text like "Last" or "»"
|
|
||||||
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
|
|
||||||
.find(el => {
|
|
||||||
const text = el.innerText.trim().toLowerCase();
|
|
||||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
|
||||||
return text.includes('last') ||
|
|
||||||
text === '»' ||
|
|
||||||
ariaLabel.includes('last page');
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (lastPageElement && lastPageElement.href) {
|
console.log(numberedLinks);
|
||||||
// Try to extract page number from the URL
|
|
||||||
try {
|
// Look for next/prev buttons
|
||||||
const url = new URL(lastPageElement.href);
|
const nextButtons = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||||
// Check common pagination parameters
|
.filter(el => {
|
||||||
['page', 'p', 'pg'].forEach(param => {
|
const text = el.innerText.trim().toLowerCase();
|
||||||
if (url.searchParams.has(param)) {
|
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||||
const value = parseInt(url.searchParams.get(param));
|
const hasNextIcon = el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||||
|
|
||||||
|
return text.includes('next') ||
|
||||||
|
text.includes('›') ||
|
||||||
|
text.includes('»') ||
|
||||||
|
text.includes('→') ||
|
||||||
|
ariaLabel.includes('next') ||
|
||||||
|
hasNextIcon;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Check for pagination containers
|
||||||
|
const paginationContainers = Array.from(document.querySelectorAll(
|
||||||
|
'.pagination, [class*="pagination"], [class*="pager"], nav[aria-label*="pagination"], [role="navigation"]'
|
||||||
|
));
|
||||||
|
|
||||||
|
// Collect all potential pagination elements
|
||||||
|
if (numberedLinks.length > 0) {
|
||||||
|
data.hasPagination = true;
|
||||||
|
data.paginationType = 'numbered';
|
||||||
|
|
||||||
|
// Get href attributes or other identifiers from numbered links
|
||||||
|
data.paginationElements = numberedLinks.slice(0, 5).map(el => {
|
||||||
|
return {
|
||||||
|
text: el.innerText.trim(),
|
||||||
|
dataPage: el.getAttribute('data-page'),
|
||||||
|
href: el.tagName === 'A' ? el.href : null,
|
||||||
|
classes: el.className,
|
||||||
|
id: el.id
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Try to find the last page number
|
||||||
|
const pageNumbers = numberedLinks
|
||||||
|
.map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
|
||||||
|
.filter(num => !isNaN(num));
|
||||||
|
|
||||||
|
if (pageNumbers.length > 0) {
|
||||||
|
data.lastPageNumber = Math.max(...pageNumbers);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also look for a "last page" element that might have text like "Last" or "»"
|
||||||
|
const lastPageElement = Array.from(document.querySelectorAll('a, button'))
|
||||||
|
.find(el => {
|
||||||
|
const text = el.innerText.trim().toLowerCase();
|
||||||
|
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||||
|
return text.includes('last') ||
|
||||||
|
text === '»' ||
|
||||||
|
ariaLabel.includes('last page');
|
||||||
|
});
|
||||||
|
|
||||||
|
if (lastPageElement && lastPageElement.href) {
|
||||||
|
// Try to extract page number from the URL
|
||||||
|
try {
|
||||||
|
const url = new URL(lastPageElement.href);
|
||||||
|
// Check common pagination parameters
|
||||||
|
['page', 'p', 'pg'].forEach(param => {
|
||||||
|
if (url.searchParams.has(param)) {
|
||||||
|
const value = parseInt(url.searchParams.get(param));
|
||||||
|
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||||
|
data.lastPageNumber = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Check for path-based pagination (like /page/10)
|
||||||
|
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
|
||||||
|
if (pathMatch && pathMatch[1]) {
|
||||||
|
const value = parseInt(pathMatch[1]);
|
||||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
||||||
data.lastPageNumber = value;
|
data.lastPageNumber = value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
} catch (e) {
|
||||||
|
console.error("Error parsing last page URL:", e);
|
||||||
// Check for path-based pagination (like /page/10)
|
|
||||||
const pathMatch = url.pathname.match(/\/(?:page|p)\/(\d+)/i);
|
|
||||||
if (pathMatch && pathMatch[1]) {
|
|
||||||
const value = parseInt(pathMatch[1]);
|
|
||||||
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
|
|
||||||
data.lastPageNumber = value;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (e) {
|
}
|
||||||
console.error("Error parsing last page URL:", e);
|
} else if (nextButtons.length > 0) {
|
||||||
|
data.hasPagination = true;
|
||||||
|
data.paginationType = 'next-prev';
|
||||||
|
|
||||||
|
// Get information about next buttons
|
||||||
|
data.paginationElements = nextButtons.slice(0, 3).map(el => {
|
||||||
|
return {
|
||||||
|
text: el.innerText.trim(),
|
||||||
|
href: el.tagName === 'A' ? el.href : null,
|
||||||
|
classes: el.className,
|
||||||
|
id: el.id
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for URL parameters that might indicate pagination
|
||||||
|
const currentUrl = window.location.href;
|
||||||
|
const urlParams = new URL(currentUrl).searchParams;
|
||||||
|
|
||||||
|
// Common pagination parameters
|
||||||
|
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
|
||||||
|
|
||||||
|
for (const param of paginationParams) {
|
||||||
|
if (urlParams.has(param)) {
|
||||||
|
data.detectedParameter = {
|
||||||
|
name: param,
|
||||||
|
value: urlParams.get(param)
|
||||||
|
};
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (nextButtons.length > 0) {
|
|
||||||
data.hasPagination = true;
|
|
||||||
data.paginationType = 'next-prev';
|
|
||||||
|
|
||||||
// Get information about next buttons
|
return data;
|
||||||
data.paginationElements = nextButtons.slice(0, 3).map(el => {
|
}''')
|
||||||
return {
|
except Exception as e:
|
||||||
text: el.innerText.trim(),
|
print(f"Error during JavaScript evaluation for pagination detection: {e}")
|
||||||
href: el.tagName === 'A' ? el.href : null,
|
# Return a safe default if JavaScript evaluation fails
|
||||||
classes: el.className,
|
pagination_data = {
|
||||||
id: el.id
|
'hasPagination': False,
|
||||||
};
|
'paginationType': None,
|
||||||
});
|
'paginationElements': [],
|
||||||
|
'detectedParameter': None,
|
||||||
|
'lastPageNumber': None
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for URL parameters that might indicate pagination
|
|
||||||
const currentUrl = window.location.href;
|
|
||||||
const urlParams = new URL(currentUrl).searchParams;
|
|
||||||
|
|
||||||
// Common pagination parameters
|
|
||||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit'];
|
|
||||||
|
|
||||||
for (const param of paginationParams) {
|
|
||||||
if (urlParams.has(param)) {
|
|
||||||
data.detectedParameter = {
|
|
||||||
name: param,
|
|
||||||
value: urlParams.get(param)
|
|
||||||
};
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return data;
|
|
||||||
}''')
|
|
||||||
|
|
||||||
# If pagination is detected, try to navigate to the next page by clicking
|
# If pagination is detected, try to navigate to the next page by clicking
|
||||||
next_page_url = None
|
next_page_url = None
|
||||||
pagination_parameter = None
|
pagination_parameter = None
|
||||||
@@ -315,86 +335,91 @@ async def detect_pagination_service(decoded_url):
|
|||||||
# First try to click on a numbered link (preferably "2" if we're on page 1)
|
# First try to click on a numbered link (preferably "2" if we're on page 1)
|
||||||
try:
|
try:
|
||||||
clicked = await page.evaluate('''() => {
|
clicked = await page.evaluate('''() => {
|
||||||
// First try to find and click on a "2" link or button
|
try {
|
||||||
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
// First try to find and click on a "2" link or button
|
||||||
.filter(el => {
|
const page2Elements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||||
// Check for text content "2"
|
.filter(el => {
|
||||||
if (el.innerText.trim() === '2') {
|
// Check for text content "2"
|
||||||
return true;
|
if (el.innerText.trim() === '2') {
|
||||||
}
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
// Check for href with page=2 or similar (for anchor elements)
|
// Check for href with page=2 or similar (for anchor elements)
|
||||||
if (el.tagName === 'A' && el.href) {
|
if (el.tagName === 'A' && el.href) {
|
||||||
try {
|
try {
|
||||||
const url = new URL(el.href, window.location.origin);
|
const url = new URL(el.href, window.location.origin);
|
||||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
|
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit',
|
||||||
'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||||
|
|
||||||
for (const param of paginationParams) {
|
for (const param of paginationParams) {
|
||||||
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
|
if (url.searchParams.has(param) && url.searchParams.get(param) === '2') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for path-based pagination like /page/2/
|
||||||
|
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
||||||
|
if (pathMatch) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
// Check for path-based pagination like /page/2/
|
// Check for data attributes that might indicate pagination
|
||||||
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
if (el.getAttribute('data-page') === '2' ||
|
||||||
if (pathMatch) {
|
el.getAttribute('data-pagenumber') === '2' ||
|
||||||
return true;
|
el.getAttribute('data-page-number') === '2') {
|
||||||
}
|
return true;
|
||||||
} catch (e) {}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Check for data attributes that might indicate pagination
|
return false;
|
||||||
if (el.getAttribute('data-page') === '2' ||
|
});
|
||||||
el.getAttribute('data-pagenumber') === '2' ||
|
|
||||||
el.getAttribute('data-page-number') === '2') {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
if (page2Elements.length > 0) {
|
||||||
});
|
console.log("Clicking on page 2 element");
|
||||||
|
page2Elements[0].click();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
if (page2Elements.length > 0) {
|
// If no "2" link found, try any numbered link or button
|
||||||
console.log("Clicking on page 2 element");
|
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
||||||
page2Elements[0].click();
|
.filter(el => /^\d+$/.test(el.innerText.trim()));
|
||||||
return true;
|
|
||||||
|
if (numberedElements.length > 0) {
|
||||||
|
// Sort by number and get the second one (likely page 2)
|
||||||
|
const sorted = numberedElements.sort((a, b) => {
|
||||||
|
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get the second element if available (page 2), otherwise the first one
|
||||||
|
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
|
||||||
|
console.log("Clicking on numbered element: " + elementToClick.innerText);
|
||||||
|
elementToClick.click();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no numbered links, try next button
|
||||||
|
const nextTexts = ['next', '›', '»', '→'];
|
||||||
|
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
||||||
|
.filter(el => {
|
||||||
|
const text = el.textContent.trim().toLowerCase();
|
||||||
|
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
||||||
|
return nextTexts.some(t => text.includes(t)) ||
|
||||||
|
ariaLabel.includes('next') ||
|
||||||
|
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
||||||
|
});
|
||||||
|
|
||||||
|
if (nextElements.length > 0) {
|
||||||
|
console.log("Clicking on next button");
|
||||||
|
nextElements[0].click();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error during pagination click operation:", error);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// If no "2" link found, try any numbered link or button
|
|
||||||
const numberedElements = Array.from(document.querySelectorAll('a[href], button, [role="button"]'))
|
|
||||||
.filter(el => /^\d+$/.test(el.innerText.trim()));
|
|
||||||
|
|
||||||
if (numberedElements.length > 0) {
|
|
||||||
// Sort by number and get the second one (likely page 2)
|
|
||||||
const sorted = numberedElements.sort((a, b) => {
|
|
||||||
return parseInt(a.innerText.trim()) - parseInt(b.innerText.trim());
|
|
||||||
});
|
|
||||||
|
|
||||||
// Get the second element if available (page 2), otherwise the first one
|
|
||||||
const elementToClick = sorted.length > 1 ? sorted[1] : sorted[0];
|
|
||||||
console.log("Clicking on numbered element: " + elementToClick.innerText);
|
|
||||||
elementToClick.click();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no numbered links, try next button
|
|
||||||
const nextTexts = ['next', '›', '»', '→'];
|
|
||||||
const nextElements = Array.from(document.querySelectorAll('a, button, [role="button"]'))
|
|
||||||
.filter(el => {
|
|
||||||
const text = el.textContent.trim().toLowerCase();
|
|
||||||
const ariaLabel = el.getAttribute('aria-label')?.toLowerCase() || '';
|
|
||||||
return nextTexts.some(t => text.includes(t)) ||
|
|
||||||
ariaLabel.includes('next') ||
|
|
||||||
el.querySelector('i.fa-chevron-right, i.fa-arrow-right, svg[class*="arrow"], svg[class*="next"]');
|
|
||||||
});
|
|
||||||
|
|
||||||
if (nextElements.length > 0) {
|
|
||||||
console.log("Clicking on next button");
|
|
||||||
nextElements[0].click();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}''')
|
}''')
|
||||||
|
|
||||||
if clicked:
|
if clicked:
|
||||||
@@ -407,126 +432,166 @@ async def detect_pagination_service(decoded_url):
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error clicking on pagination element: {e}")
|
print(f"Error clicking on pagination element: {e}")
|
||||||
|
# Continue with the process even if clicking fails
|
||||||
|
|
||||||
# If we successfully navigated to the next page, analyze the URL difference
|
# If we successfully navigated to the next page, analyze the URL difference
|
||||||
if next_page_url:
|
if next_page_url:
|
||||||
print('Searching for pagination parameter')
|
print('Searching for pagination parameter')
|
||||||
# Parse both URLs
|
# Parse both URLs
|
||||||
original_parsed = await page.evaluate(f'''(originalUrl) => {{
|
try:
|
||||||
const original = new URL(originalUrl);
|
original_parsed = await page.evaluate(f'''(originalUrl) => {{
|
||||||
const current = new URL(window.location.href);
|
try {{
|
||||||
|
const original = new URL(originalUrl);
|
||||||
|
const current = new URL(window.location.href);
|
||||||
|
|
||||||
// Check for differences in query parameters
|
// Check for differences in query parameters
|
||||||
let paramDiff = null;
|
let paramDiff = null;
|
||||||
|
|
||||||
// Common pagination parameters to check
|
// Common pagination parameters to check
|
||||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||||
|
|
||||||
for (const param of paginationParams) {{
|
for (const param of paginationParams) {{
|
||||||
const originalValue = original.searchParams.get(param);
|
const originalValue = original.searchParams.get(param);
|
||||||
const currentValue = current.searchParams.get(param);
|
const currentValue = current.searchParams.get(param);
|
||||||
|
|
||||||
if (originalValue !== currentValue && currentValue !== null) {{
|
if (originalValue !== currentValue && currentValue !== null) {{
|
||||||
paramDiff = {{
|
paramDiff = {{
|
||||||
name: param,
|
name: param,
|
||||||
originalValue: originalValue,
|
originalValue: originalValue,
|
||||||
currentValue: currentValue
|
currentValue: currentValue
|
||||||
}};
|
}};
|
||||||
break;
|
break;
|
||||||
}}
|
}}
|
||||||
}}
|
}}
|
||||||
|
|
||||||
// Check for path differences (like /page/1 vs /page/2)
|
// Check for path differences (like /page/1 vs /page/2)
|
||||||
const originalPath = original.pathname;
|
const originalPath = original.pathname;
|
||||||
const currentPath = current.pathname;
|
const currentPath = current.pathname;
|
||||||
|
|
||||||
let pathDiff = null;
|
let pathDiff = null;
|
||||||
if (originalPath !== currentPath) {{
|
if (originalPath !== currentPath) {{
|
||||||
const originalSegments = originalPath.split('/').filter(s => s);
|
const originalSegments = originalPath.split('/').filter(s => s);
|
||||||
const currentSegments = currentPath.split('/').filter(s => s);
|
const currentSegments = currentPath.split('/').filter(s => s);
|
||||||
|
|
||||||
// Find the segment that changed
|
// Find the segment that changed
|
||||||
if (originalSegments.length === currentSegments.length) {{
|
if (originalSegments.length === currentSegments.length) {{
|
||||||
for (let i = 0; i < originalSegments.length; i++) {{
|
for (let i = 0; i < originalSegments.length; i++) {{
|
||||||
if (originalSegments[i] !== currentSegments[i]) {{
|
if (originalSegments[i] !== currentSegments[i]) {{
|
||||||
// Check if the difference is numeric
|
// Check if the difference is numeric
|
||||||
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
||||||
pathDiff = {{
|
pathDiff = {{
|
||||||
index: i,
|
index: i,
|
||||||
originalValue: originalSegments[i],
|
originalValue: originalSegments[i],
|
||||||
currentValue: currentSegments[i]
|
currentValue: currentSegments[i]
|
||||||
}};
|
}};
|
||||||
|
}}
|
||||||
|
}}
|
||||||
}}
|
}}
|
||||||
}}
|
}}
|
||||||
}}
|
}}
|
||||||
|
|
||||||
|
return {{
|
||||||
|
paramDiff,
|
||||||
|
pathDiff,
|
||||||
|
originalUrl: originalUrl,
|
||||||
|
currentUrl: window.location.href
|
||||||
|
}};
|
||||||
|
}} catch (error) {{
|
||||||
|
console.error("Error during URL analysis:", error);
|
||||||
|
return {{
|
||||||
|
paramDiff: null,
|
||||||
|
pathDiff: null,
|
||||||
|
originalUrl: originalUrl,
|
||||||
|
currentUrl: window.location.href,
|
||||||
|
error: error.message
|
||||||
|
}};
|
||||||
}}
|
}}
|
||||||
}}
|
}}''', original_url)
|
||||||
|
except Exception as e:
|
||||||
return {{
|
print(f"Error during URL analysis: {e}")
|
||||||
paramDiff,
|
original_parsed = {
|
||||||
pathDiff,
|
'paramDiff': None,
|
||||||
originalUrl: originalUrl,
|
'pathDiff': None,
|
||||||
currentUrl: window.location.href
|
'originalUrl': original_url,
|
||||||
}};
|
'currentUrl': next_page_url,
|
||||||
}}''', original_url)
|
'error': str(e)
|
||||||
|
|
||||||
# Determine the pagination parameter and create URL template
|
|
||||||
print(original_parsed)
|
|
||||||
if original_parsed['paramDiff']:
|
|
||||||
param_name = original_parsed['paramDiff']['name']
|
|
||||||
pagination_parameter = {
|
|
||||||
'type': 'query',
|
|
||||||
'name': param_name,
|
|
||||||
'value': original_parsed['paramDiff']['currentValue']
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
|
# Determine the pagination parameter and create URL template
|
||||||
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
|
print(original_parsed)
|
||||||
curr_val = original_parsed['paramDiff']['currentValue']
|
if original_parsed['paramDiff']:
|
||||||
try:
|
param_name = original_parsed['paramDiff']['name']
|
||||||
if orig_val is not None and curr_val is not None:
|
pagination_parameter = {
|
||||||
orig_num = int(orig_val)
|
'type': 'query',
|
||||||
curr_num = int(curr_val)
|
'name': param_name,
|
||||||
step_size = abs(curr_num - orig_num)
|
'value': original_parsed['paramDiff']['currentValue']
|
||||||
except Exception:
|
}
|
||||||
step_size = None
|
|
||||||
|
|
||||||
# Create URL template for query parameter
|
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
|
||||||
|
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
|
||||||
|
curr_val = original_parsed['paramDiff']['currentValue']
|
||||||
|
try:
|
||||||
|
if orig_val is not None and curr_val is not None:
|
||||||
|
orig_num = int(orig_val)
|
||||||
|
curr_num = int(curr_val)
|
||||||
|
step_size = abs(curr_num - orig_num)
|
||||||
|
except Exception:
|
||||||
|
step_size = None
|
||||||
|
|
||||||
|
# Create URL template for query parameter
|
||||||
|
try:
|
||||||
url_obj = await page.evaluate(f'''(url, paramName) => {{
|
url_obj = await page.evaluate(f'''(url, paramName) => {{
|
||||||
const urlObj = new URL(url);
|
try {{
|
||||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
const urlObj = new URL(url);
|
||||||
return urlObj.toString();
|
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||||
|
return urlObj.toString();
|
||||||
|
}} catch (error) {{
|
||||||
|
console.error("Error creating URL template:", error);
|
||||||
|
return null;
|
||||||
|
}}
|
||||||
}}''', original_url, param_name)
|
}}''', original_url, param_name)
|
||||||
|
|
||||||
url_template = url_obj
|
url_template = url_obj
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error creating URL template for query parameter: {e}")
|
||||||
|
url_template = None
|
||||||
|
|
||||||
elif original_parsed['pathDiff']:
|
elif original_parsed['pathDiff']:
|
||||||
path_index = original_parsed['pathDiff']['index']
|
path_index = original_parsed['pathDiff']['index']
|
||||||
pagination_parameter = {
|
pagination_parameter = {
|
||||||
'type': 'path',
|
'type': 'path',
|
||||||
'index': path_index,
|
'index': path_index,
|
||||||
'value': original_parsed['pathDiff']['currentValue']
|
'value': original_parsed['pathDiff']['currentValue']
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
||||||
orig_val = original_parsed['pathDiff']['originalValue']
|
orig_val = original_parsed['pathDiff']['originalValue']
|
||||||
curr_val = original_parsed['pathDiff']['currentValue']
|
curr_val = original_parsed['pathDiff']['currentValue']
|
||||||
try:
|
try:
|
||||||
if orig_val is not None and curr_val is not None:
|
if orig_val is not None and curr_val is not None:
|
||||||
orig_num = int(orig_val)
|
orig_num = int(orig_val)
|
||||||
curr_num = int(curr_val)
|
curr_num = int(curr_val)
|
||||||
step_size = abs(curr_num - orig_num)
|
step_size = abs(curr_num - orig_num)
|
||||||
except Exception:
|
except Exception:
|
||||||
step_size = None
|
step_size = None
|
||||||
|
|
||||||
# Create URL template for path parameter
|
# Create URL template for path parameter
|
||||||
|
try:
|
||||||
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
||||||
const urlObj = new URL(url);
|
try {{
|
||||||
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
const urlObj = new URL(url);
|
||||||
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||||
urlObj.pathname = '/' + pathSegments.join('/');
|
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||||
return urlObj.toString();
|
urlObj.pathname = '/' + pathSegments.join('/');
|
||||||
|
return urlObj.toString();
|
||||||
|
}} catch (error) {{
|
||||||
|
console.error("Error creating path URL template:", error);
|
||||||
|
return null;
|
||||||
|
}}
|
||||||
}}''', original_url, path_index)
|
}}''', original_url, path_index)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error creating URL template for path parameter: {e}")
|
||||||
|
url_template = None
|
||||||
|
|
||||||
if url_template:
|
if url_template:
|
||||||
# Decode URL-encoded characters in the template
|
# Decode URL-encoded characters in the template
|
||||||
@@ -536,11 +601,20 @@ async def detect_pagination_service(decoded_url):
|
|||||||
# If we couldn't determine the URL template from navigation, try to infer it
|
# If we couldn't determine the URL template from navigation, try to infer it
|
||||||
if not url_template and pagination_data['detectedParameter']:
|
if not url_template and pagination_data['detectedParameter']:
|
||||||
param_name = pagination_data['detectedParameter']['name']
|
param_name = pagination_data['detectedParameter']['name']
|
||||||
url_template = await page.evaluate(f'''(url, paramName) => {{
|
try:
|
||||||
const urlObj = new URL(url);
|
url_template = await page.evaluate(f'''(url, paramName) => {{
|
||||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
try {{
|
||||||
return urlObj.toString();
|
const urlObj = new URL(url);
|
||||||
}}''', original_url, param_name)
|
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||||
|
return urlObj.toString();
|
||||||
|
}} catch (error) {{
|
||||||
|
console.error("Error creating inferred URL template:", error);
|
||||||
|
return null;
|
||||||
|
}}
|
||||||
|
}}''', original_url, param_name)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error creating inferred URL template: {e}")
|
||||||
|
url_template = None
|
||||||
|
|
||||||
# Return the pagination detection results with a simplified structure
|
# Return the pagination detection results with a simplified structure
|
||||||
result = {
|
result = {
|
||||||
|
|||||||
+18
-463
@@ -655,475 +655,30 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
|
|||||||
if not x_api_key or x_api_key != API_KEY:
|
if not x_api_key or x_api_key != API_KEY:
|
||||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||||
|
|
||||||
conn = sqlite3.connect(DB_PATH)
|
|
||||||
cursor = conn.cursor()
|
|
||||||
|
|
||||||
# Get total entries
|
|
||||||
cursor.execute("SELECT COUNT(*) FROM cache")
|
|
||||||
total_entries = cursor.fetchone()[0]
|
|
||||||
|
|
||||||
# Get entries by route
|
|
||||||
cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route")
|
|
||||||
routes = {route: count for route, count in cursor.fetchall()}
|
|
||||||
|
|
||||||
# Get recent entries (last 24 hours)
|
|
||||||
recent_timestamp = int(time.time()) - (24 * 60 * 60)
|
|
||||||
cursor.execute("SELECT COUNT(*) FROM cache WHERE timestamp > ?", (recent_timestamp,))
|
|
||||||
recent_entries = cursor.fetchone()[0]
|
|
||||||
|
|
||||||
# Get oldest entry timestamp
|
|
||||||
cursor.execute("SELECT MIN(timestamp) FROM cache")
|
|
||||||
oldest_timestamp = cursor.fetchone()[0]
|
|
||||||
oldest_date = datetime.fromtimestamp(oldest_timestamp).isoformat() if oldest_timestamp else None
|
|
||||||
|
|
||||||
# Get newest entry timestamp
|
|
||||||
cursor.execute("SELECT MAX(timestamp) FROM cache")
|
|
||||||
newest_timestamp = cursor.fetchone()[0]
|
|
||||||
newest_date = datetime.fromtimestamp(newest_timestamp).isoformat() if newest_timestamp else None
|
|
||||||
|
|
||||||
conn.close()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"status": "success",
|
|
||||||
"stats": {
|
|
||||||
"total_entries": total_entries,
|
|
||||||
"entries_by_route": routes,
|
|
||||||
"recent_entries": recent_entries,
|
|
||||||
"oldest_entry": oldest_date,
|
|
||||||
"newest_entry": newest_date,
|
|
||||||
"cache_expiry_hours": CACHE_EXPIRY_HOURS,
|
|
||||||
"cleanup_schedule": CLEANUP_CRON
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@app.get("/pagination")
|
|
||||||
async def detect_pagination(url: str, x_api_key: Optional[str] = Header(None)):
|
|
||||||
"""Detect pagination on a website and determine the pagination pattern"""
|
|
||||||
# Validate API key
|
|
||||||
if not x_api_key or x_api_key != API_KEY:
|
|
||||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
|
||||||
|
|
||||||
# Decode URL if it's encoded
|
|
||||||
decoded_url = unquote(url)
|
|
||||||
|
|
||||||
# Check cache first
|
|
||||||
cached_result = get_cached_data(decoded_url, "pagination")
|
|
||||||
if cached_result:
|
|
||||||
return cached_result
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
print(f"Detecting pagination on: {decoded_url}")
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
# Define the operation to perform with the browser
|
# Get total count
|
||||||
async def pagination_operation(page):
|
cursor.execute("SELECT COUNT(*) FROM cache")
|
||||||
try:
|
total_count = cursor.fetchone()[0]
|
||||||
# Navigate to the URL
|
|
||||||
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
|
||||||
original_url = page.url
|
|
||||||
|
|
||||||
print(f"Successfully loaded page: {original_url}")
|
# Get count by route
|
||||||
|
cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route")
|
||||||
|
route_counts = dict(cursor.fetchall())
|
||||||
|
|
||||||
# Analyze the page for pagination information
|
# Get oldest and newest entries
|
||||||
pagination_info = await page.evaluate('''() => {
|
cursor.execute("SELECT MIN(timestamp), MAX(timestamp) FROM cache")
|
||||||
// Find the last page number if available
|
min_time, max_time = cursor.fetchone()
|
||||||
const findLastPageNumber = () => {
|
|
||||||
// Get all links on the page
|
|
||||||
const links = Array.from(document.querySelectorAll('a'));
|
|
||||||
|
|
||||||
// Strategy 1: Find numeric links (page numbers)
|
conn.close()
|
||||||
const numericLinks = links.filter(link => {
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
|
||||||
});
|
|
||||||
|
|
||||||
if (numericLinks.length > 0) {
|
|
||||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
|
||||||
return Math.max(...numericValues);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strategy 2: Look for "last page" link
|
|
||||||
const lastLinks = links.filter(link => {
|
|
||||||
const text = link.innerText.trim().toLowerCase();
|
|
||||||
const classes = (link.className || '').toLowerCase();
|
|
||||||
const ariaLabel = (link.getAttribute('aria-label') || '').toLowerCase();
|
|
||||||
|
|
||||||
return (text === 'last' ||
|
|
||||||
classes.includes('last') ||
|
|
||||||
ariaLabel.includes('last') ||
|
|
||||||
link.getAttribute('rel') === 'last');
|
|
||||||
});
|
|
||||||
|
|
||||||
if (lastLinks.length > 0) {
|
|
||||||
const lastLink = lastLinks[0];
|
|
||||||
const href = lastLink.href;
|
|
||||||
|
|
||||||
// Common patterns: page=X, /page/X, etc.
|
|
||||||
const pagePatterns = [
|
|
||||||
/[?&]page=(\d+)/,
|
|
||||||
/[?&]p=(\d+)/,
|
|
||||||
/[?&]pg=(\d+)/,
|
|
||||||
/\/page\/(\d+)/,
|
|
||||||
/\/p\/(\d+)/,
|
|
||||||
/\/paged\/(\d+)/
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const pattern of pagePatterns) {
|
|
||||||
const match = href.match(pattern);
|
|
||||||
if (match && match[1]) {
|
|
||||||
return parseInt(match[1]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strategy 3: Analyze all URLs for page numbers
|
|
||||||
const pageNumbersFromUrls = [];
|
|
||||||
links.forEach(link => {
|
|
||||||
if (!link.href || link.href === '#') return;
|
|
||||||
|
|
||||||
// Check for common pagination URL patterns
|
|
||||||
const patterns = [
|
|
||||||
/[?&]page=(\d+)/,
|
|
||||||
/[?&]p=(\d+)/,
|
|
||||||
/[?&]pg=(\d+)/,
|
|
||||||
/\/page\/(\d+)/,
|
|
||||||
/\/p\/(\d+)/,
|
|
||||||
/\/paged\/(\d+)/,
|
|
||||||
/\/pages\/(\d+)/
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const pattern of patterns) {
|
|
||||||
const match = link.href.match(pattern);
|
|
||||||
if (match && match[1]) {
|
|
||||||
pageNumbersFromUrls.push(parseInt(match[1]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (pageNumbersFromUrls.length > 0) {
|
|
||||||
return Math.max(...pageNumbersFromUrls);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Find a pagination link to click
|
|
||||||
const findPaginationLink = () => {
|
|
||||||
const links = Array.from(document.querySelectorAll('a'));
|
|
||||||
|
|
||||||
// Try to find a page "2" link first (most reliable)
|
|
||||||
const page2Link = links.find(link => {
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
return text === '2' && link.href && link.href !== '#';
|
|
||||||
});
|
|
||||||
|
|
||||||
if (page2Link) {
|
|
||||||
return { element: page2Link, href: page2Link.href, type: 'numeric' };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try common "next page" selectors
|
|
||||||
const nextSelectors = [
|
|
||||||
'a.next',
|
|
||||||
'a.page-next',
|
|
||||||
'a[rel="next"]',
|
|
||||||
'a[aria-label="Next page"]',
|
|
||||||
'a[aria-label="next"]'
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const selector of nextSelectors) {
|
|
||||||
const element = document.querySelector(selector);
|
|
||||||
if (element && element.href && element.href !== '#') {
|
|
||||||
return { element, href: element.href, type: 'next' };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Look for any link that might be pagination
|
|
||||||
const paginationLinks = links.filter(link => {
|
|
||||||
if (!link.href || link.href === '#') return false;
|
|
||||||
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
const href = link.href;
|
|
||||||
|
|
||||||
// Check for numeric text or next/prev indicators
|
|
||||||
const isNumeric = /^[0-9]+$/.test(text) && text !== '1';
|
|
||||||
const isNextPrev = /next|prev|previous|older|newer/i.test(text) ||
|
|
||||||
/[»«‹›<>]/.test(text);
|
|
||||||
|
|
||||||
// Check for page parameter in URL
|
|
||||||
const hasPageParam = /[?&]page=|[?&]p=|[?&]pg=|\/page\/|\/p\//.test(href);
|
|
||||||
|
|
||||||
return (isNumeric || isNextPrev || hasPageParam);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (paginationLinks.length > 0) {
|
|
||||||
const link = paginationLinks[0];
|
|
||||||
return { element: link, href: link.href, type: 'other' };
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
const lastPage = findLastPageNumber();
|
|
||||||
const paginationLink = findPaginationLink();
|
|
||||||
|
|
||||||
if (paginationLink) {
|
|
||||||
// Click the link
|
|
||||||
paginationLink.element.click();
|
|
||||||
return {
|
|
||||||
clicked: true,
|
|
||||||
href: paginationLink.href,
|
|
||||||
type: paginationLink.type,
|
|
||||||
lastPage
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return { clicked: false, lastPage };
|
|
||||||
}''')
|
|
||||||
|
|
||||||
# If no pagination was found or clicked
|
|
||||||
if not pagination_info.get('clicked', False):
|
|
||||||
return {
|
|
||||||
"status": "success",
|
|
||||||
"url": decoded_url,
|
|
||||||
"hasPagination": False,
|
|
||||||
"urlTemplate": None,
|
|
||||||
"lastPage": pagination_info.get('lastPage')
|
|
||||||
}
|
|
||||||
|
|
||||||
# Wait for navigation to complete after the click
|
|
||||||
try:
|
|
||||||
await page.waitForNavigation({'timeout': 10000, 'waitUntil': 'networkidle2'})
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Navigation timeout: {e}")
|
|
||||||
|
|
||||||
# Get the new URL after clicking
|
|
||||||
next_page_url = page.url
|
|
||||||
|
|
||||||
# If URL didn't change, pagination might be handled by AJAX
|
|
||||||
if next_page_url == original_url:
|
|
||||||
return {
|
|
||||||
"status": "success",
|
|
||||||
"url": decoded_url,
|
|
||||||
"hasPagination": True,
|
|
||||||
"urlTemplate": "AJAX pagination (URL doesn't change)",
|
|
||||||
"lastPage": pagination_info.get('lastPage')
|
|
||||||
}
|
|
||||||
|
|
||||||
print(f"Navigation successful: {original_url} -> {next_page_url}")
|
|
||||||
|
|
||||||
# Analyze the URL structure to determine pagination pattern
|
|
||||||
url_template = await page.evaluate('''(originalUrl, nextPageUrl) => {
|
|
||||||
// Helper function to parse URL query parameters
|
|
||||||
const parseQueryParams = (url) => {
|
|
||||||
const params = {};
|
|
||||||
if (url.includes('?')) {
|
|
||||||
const queryString = url.split('?')[1].split('#')[0];
|
|
||||||
queryString.split('&').forEach(param => {
|
|
||||||
if (param.includes('=')) {
|
|
||||||
const [key, value] = param.split('=', 2);
|
|
||||||
params[key] = value;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return params;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Check for query parameter based pagination
|
|
||||||
if (nextPageUrl.includes('?')) {
|
|
||||||
const originalParams = parseQueryParams(originalUrl);
|
|
||||||
const nextParams = parseQueryParams(nextPageUrl);
|
|
||||||
|
|
||||||
// Find parameters that changed or were added
|
|
||||||
let paginationParam = null;
|
|
||||||
|
|
||||||
// First check for common pagination parameter names
|
|
||||||
const commonPaginationParams = ['page', 'p', 'pg', 'paged', 'current_page', 'pagenum', 'pageNumber'];
|
|
||||||
|
|
||||||
for (const key of commonPaginationParams) {
|
|
||||||
if (key in nextParams &&
|
|
||||||
(!(key in originalParams) || originalParams[key] !== nextParams[key])) {
|
|
||||||
if (/^\d+$/.test(nextParams[key]) && parseInt(nextParams[key]) > 1) {
|
|
||||||
paginationParam = key;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no common parameter found, check all parameters
|
|
||||||
if (!paginationParam) {
|
|
||||||
for (const [key, value] of Object.entries(nextParams)) {
|
|
||||||
// Check if parameter is new or changed
|
|
||||||
if (!(key in originalParams) || originalParams[key] !== value) {
|
|
||||||
// Check if the value is numeric and could be a page number
|
|
||||||
if (/^\d+$/.test(value) && parseInt(value) > 1) {
|
|
||||||
paginationParam = key;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we found a pagination parameter
|
|
||||||
if (paginationParam) {
|
|
||||||
const baseUrl = nextPageUrl.split('?')[0];
|
|
||||||
|
|
||||||
// Reconstruct the URL template with all parameters
|
|
||||||
const queryParts = [];
|
|
||||||
for (const [key, value] of Object.entries(nextParams)) {
|
|
||||||
if (key === paginationParam) {
|
|
||||||
queryParts.push(`${key}={PAGE_NUMBER}`);
|
|
||||||
} else {
|
|
||||||
queryParts.push(`${key}=${value}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${baseUrl}?${queryParts.join('&')}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for path-based pagination
|
|
||||||
const pathPatterns = ['/page/', '/p/', '/paged/', '/pages/'];
|
|
||||||
for (const pattern of pathPatterns) {
|
|
||||||
if (nextPageUrl.includes(pattern)) {
|
|
||||||
const parts = nextPageUrl.split(pattern);
|
|
||||||
let template = `${parts[0]}${pattern}{PAGE_NUMBER}`;
|
|
||||||
|
|
||||||
// Add any suffix after the page number
|
|
||||||
if (parts.length > 1 && parts[1].includes('/')) {
|
|
||||||
const suffix = parts[1].split('/', 1)[1];
|
|
||||||
if (suffix) {
|
|
||||||
template += `/${suffix}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return template;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we couldn't determine the pattern, try to make an educated guess
|
|
||||||
// For the specific case where a parameter like current_page=2 is added
|
|
||||||
const originalUrlObj = new URL(originalUrl);
|
|
||||||
const nextUrlObj = new URL(nextPageUrl);
|
|
||||||
|
|
||||||
// Check if the paths are the same but query params differ
|
|
||||||
if (originalUrlObj.pathname === nextUrlObj.pathname) {
|
|
||||||
const originalParams = parseQueryParams(originalUrl);
|
|
||||||
const nextParams = parseQueryParams(nextPageUrl);
|
|
||||||
|
|
||||||
// Find parameters that exist in next but not in original
|
|
||||||
const newParams = Object.keys(nextParams).filter(key => !(key in originalParams));
|
|
||||||
|
|
||||||
// If there's exactly one new parameter and it has a numeric value
|
|
||||||
if (newParams.length === 1 && /^\d+$/.test(nextParams[newParams[0]])) {
|
|
||||||
const paginationParam = newParams[0];
|
|
||||||
const baseUrl = nextPageUrl.split('?')[0];
|
|
||||||
|
|
||||||
// Reconstruct the URL template
|
|
||||||
const queryParts = [];
|
|
||||||
for (const [key, value] of Object.entries(nextParams)) {
|
|
||||||
if (key === paginationParam) {
|
|
||||||
queryParts.push(`${key}={PAGE_NUMBER}`);
|
|
||||||
} else {
|
|
||||||
queryParts.push(`${key}=${value}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${baseUrl}?${queryParts.join('&')}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we still couldn't determine the pattern, return both URLs as examples
|
|
||||||
return `Pattern unclear. Example: ${originalUrl} → ${nextPageUrl}`;
|
|
||||||
}''', original_url, next_page_url)
|
|
||||||
|
|
||||||
# Try to extract last page number from the next page if we didn't find it on the first page
|
|
||||||
if not pagination_info.get('lastPage'):
|
|
||||||
last_page_from_next = await page.evaluate('''() => {
|
|
||||||
// Get all links on the page
|
|
||||||
const links = Array.from(document.querySelectorAll('a'));
|
|
||||||
|
|
||||||
// Strategy 1: Find numeric links (page numbers)
|
|
||||||
const numericLinks = links.filter(link => {
|
|
||||||
const text = link.innerText.trim();
|
|
||||||
return /^[0-9]+$/.test(text) && link.href && link.href !== '#';
|
|
||||||
});
|
|
||||||
|
|
||||||
if (numericLinks.length > 0) {
|
|
||||||
const numericValues = numericLinks.map(link => parseInt(link.innerText.trim()));
|
|
||||||
return Math.max(...numericValues);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strategy 2: Analyze all URLs for page numbers
|
|
||||||
const pageNumbersFromUrls = [];
|
|
||||||
links.forEach(link => {
|
|
||||||
if (!link.href || link.href === '#') return;
|
|
||||||
|
|
||||||
// Check for common pagination URL patterns
|
|
||||||
const patterns = [
|
|
||||||
/[?&]page=(\d+)/,
|
|
||||||
/[?&]p=(\d+)/,
|
|
||||||
/[?&]pg=(\d+)/,
|
|
||||||
/\/page\/(\d+)/,
|
|
||||||
/\/p\/(\d+)/,
|
|
||||||
/\/paged\/(\d+)/,
|
|
||||||
/\/pages\/(\d+)/
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const pattern of patterns) {
|
|
||||||
const match = link.href.match(pattern);
|
|
||||||
if (match && match[1]) {
|
|
||||||
pageNumbersFromUrls.push(parseInt(match[1]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (pageNumbersFromUrls.length > 0) {
|
|
||||||
return Math.max(...pageNumbersFromUrls);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}''')
|
|
||||||
|
|
||||||
if last_page_from_next:
|
|
||||||
pagination_info['lastPage'] = last_page_from_next
|
|
||||||
|
|
||||||
# Check if the pattern is unclear
|
|
||||||
has_pagination = True
|
|
||||||
url_string_template = str(url_template)
|
|
||||||
if url_string_template and url_string_template.startswith("Pattern unclear"):
|
|
||||||
has_pagination = False
|
|
||||||
url_template = None
|
|
||||||
|
|
||||||
result = {
|
|
||||||
"status": "success",
|
|
||||||
"url": decoded_url,
|
|
||||||
"hasPagination": has_pagination,
|
|
||||||
"urlTemplate": url_template,
|
|
||||||
"lastPage": pagination_info.get('lastPage'),
|
|
||||||
"originalUrl": original_url,
|
|
||||||
"nextPageUrl": next_page_url
|
|
||||||
}
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Error during pagination detection: {e}")
|
|
||||||
return {
|
|
||||||
"status": "error",
|
|
||||||
"url": decoded_url,
|
|
||||||
"error": str(e),
|
|
||||||
"hasPagination": False,
|
|
||||||
"urlTemplate": None,
|
|
||||||
"lastPage": None
|
|
||||||
}
|
|
||||||
|
|
||||||
# Perform the operation
|
|
||||||
result = await safe_browser_operation(decoded_url, pagination_operation)
|
|
||||||
|
|
||||||
# Save to cache
|
|
||||||
save_to_cache(decoded_url, "pagination", result)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total_entries": total_count,
|
||||||
|
"route_counts": route_counts,
|
||||||
|
"oldest_entry": min_time,
|
||||||
|
"newest_entry": max_time
|
||||||
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Test script for the pagination route to verify error handling and browser cleanup.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import aiohttp
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
async def test_pagination(url, api_key):
|
||||||
|
"""Test the pagination endpoint with a given URL"""
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
headers = {"X-API-Key": api_key}
|
||||||
|
|
||||||
|
# Test URL with pagination
|
||||||
|
test_url = f"http://localhost:8000/pagination?url={url}"
|
||||||
|
|
||||||
|
print(f"Testing pagination for: {url}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with session.get(test_url, headers=headers) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
result = await response.json()
|
||||||
|
print(f"✅ Success: {json.dumps(result, indent=2)}")
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
error_text = await response.text()
|
||||||
|
print(f"❌ Error {response.status}: {error_text}")
|
||||||
|
return False
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ Exception: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
"""Main test function"""
|
||||||
|
# Test URLs - some with pagination, some without
|
||||||
|
test_urls = [
|
||||||
|
"https://example.com", # No pagination
|
||||||
|
"https://httpbin.org/get", # No pagination
|
||||||
|
"https://news.ycombinator.com", # Has pagination
|
||||||
|
]
|
||||||
|
|
||||||
|
api_key = "test-key" # Replace with your actual API key
|
||||||
|
|
||||||
|
print("Testing pagination route...")
|
||||||
|
print("=" * 50)
|
||||||
|
|
||||||
|
for url in test_urls:
|
||||||
|
success = await test_pagination(url, api_key)
|
||||||
|
print("-" * 30)
|
||||||
|
|
||||||
|
# Small delay between tests
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
|
print("Test completed!")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
Reference in New Issue
Block a user