This commit is contained in:
@@ -358,11 +358,18 @@ async def detect_pagination_service(decoded_url):
|
||||
}
|
||||
}
|
||||
|
||||
// Check for path-based pagination like /page/2/
|
||||
// Check for path-based pagination like /page/2/ or /vacatures/page/2
|
||||
const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
|
||||
if (pathMatch) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for path-based pagination where /page/2 is appended to the current path
|
||||
const currentPath = window.location.pathname;
|
||||
const expectedPath = currentPath.replace(/\/$/, '') + '/page/2';
|
||||
if (url.pathname === expectedPath) {
|
||||
return true;
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
@@ -426,7 +433,7 @@ async def detect_pagination_service(decoded_url):
|
||||
if clicked:
|
||||
print("Successfully clicked on pagination element")
|
||||
# Wait for navigation to complete
|
||||
await asyncio.sleep(1)
|
||||
await page.waitForNavigation(waitUntil='networkidle2', timeout=10000)
|
||||
next_page_url = page.url
|
||||
else:
|
||||
print("No clickable pagination element found")
|
||||
@@ -465,7 +472,7 @@ async def detect_pagination_service(decoded_url):
|
||||
}}
|
||||
}}
|
||||
|
||||
// Check for path differences (like /page/1 vs /page/2)
|
||||
// Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2)
|
||||
const originalPath = original.pathname;
|
||||
const currentPath = current.pathname;
|
||||
|
||||
@@ -474,13 +481,14 @@ async def detect_pagination_service(decoded_url):
|
||||
const originalSegments = originalPath.split('/').filter(s => s);
|
||||
const currentSegments = currentPath.split('/').filter(s => s);
|
||||
|
||||
// Find the segment that changed
|
||||
// Case 1: Same number of segments - find the one that changed
|
||||
if (originalSegments.length === currentSegments.length) {{
|
||||
for (let i = 0; i < originalSegments.length; i++) {{
|
||||
if (originalSegments[i] !== currentSegments[i]) {{
|
||||
// Check if the difference is numeric
|
||||
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
||||
pathDiff = {{
|
||||
type: 'replace',
|
||||
index: i,
|
||||
originalValue: originalSegments[i],
|
||||
currentValue: currentSegments[i]
|
||||
@@ -489,6 +497,41 @@ async def detect_pagination_service(decoded_url):
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
// Case 2: Current path has more segments - check for added pagination segments
|
||||
else if (currentSegments.length > originalSegments.length) {{
|
||||
// Look for patterns like /page/NUMBER or /p/NUMBER at the end
|
||||
const pagePattern = /^(page|p)\/(\d+)$/i;
|
||||
|
||||
// Check the last two segments of the current path
|
||||
if (currentSegments.length >= 2) {{
|
||||
const lastTwoSegments = currentSegments.slice(-2).join('/');
|
||||
const match = lastTwoSegments.match(pagePattern);
|
||||
|
||||
if (match) {{
|
||||
pathDiff = {{
|
||||
type: 'append',
|
||||
pageSegment: match[1], // 'page' or 'p'
|
||||
pageNumber: match[2], // the actual number
|
||||
originalSegments: originalSegments,
|
||||
currentSegments: currentSegments
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
|
||||
// If no pattern match, check if the last segment is numeric
|
||||
if (!pathDiff && currentSegments.length > 0) {{
|
||||
const lastSegment = currentSegments[currentSegments.length - 1];
|
||||
if (!isNaN(lastSegment)) {{
|
||||
pathDiff = {{
|
||||
type: 'append',
|
||||
pageSegment: null,
|
||||
pageNumber: lastSegment,
|
||||
originalSegments: originalSegments,
|
||||
currentSegments: currentSegments
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
return {{
|
||||
@@ -559,41 +602,93 @@ async def detect_pagination_service(decoded_url):
|
||||
url_template = None
|
||||
|
||||
elif original_parsed.get('pathDiff'):
|
||||
path_index = original_parsed['pathDiff']['index']
|
||||
pagination_parameter = {
|
||||
'type': 'path',
|
||||
'index': path_index,
|
||||
'value': original_parsed['pathDiff']['currentValue']
|
||||
}
|
||||
path_diff = original_parsed['pathDiff']
|
||||
|
||||
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
||||
orig_val = original_parsed['pathDiff']['originalValue']
|
||||
curr_val = original_parsed['pathDiff']['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
if path_diff.get('type') == 'replace':
|
||||
# Handle existing path segment replacement
|
||||
path_index = path_diff['index']
|
||||
pagination_parameter = {
|
||||
'type': 'path',
|
||||
'index': path_index,
|
||||
'value': path_diff['currentValue']
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR PATH PARAM ---
|
||||
orig_val = path_diff['originalValue']
|
||||
curr_val = path_diff['currentValue']
|
||||
try:
|
||||
if orig_val is not None and curr_val is not None:
|
||||
orig_num = int(orig_val)
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for path parameter replacement
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
||||
try {{
|
||||
const urlObj = new URL(url);
|
||||
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||
urlObj.pathname = '/' + pathSegments.join('/');
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
console.error("Error creating path URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, path_index)
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for path parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
elif path_diff.get('type') == 'append':
|
||||
# Handle new pagination segments being appended
|
||||
pagination_parameter = {
|
||||
'type': 'path_append',
|
||||
'pageSegment': path_diff.get('pageSegment'),
|
||||
'pageNumber': path_diff['pageNumber']
|
||||
}
|
||||
|
||||
# --- STEP SIZE DETECTION FOR APPENDED PATH PARAM ---
|
||||
try:
|
||||
curr_val = path_diff['pageNumber']
|
||||
# For appended pagination, assume we started from page 1 (implicit)
|
||||
orig_num = 1
|
||||
curr_num = int(curr_val)
|
||||
step_size = abs(curr_num - orig_num)
|
||||
except Exception:
|
||||
step_size = None
|
||||
except Exception:
|
||||
step_size = None
|
||||
|
||||
# Create URL template for path parameter
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
||||
try {{
|
||||
const urlObj = new URL(url);
|
||||
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||
urlObj.pathname = '/' + pathSegments.join('/');
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
console.error("Error creating path URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, path_index)
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for path parameter: {e}")
|
||||
url_template = None
|
||||
# Create URL template for appended path parameter
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, pageSegment) => {{
|
||||
try {{
|
||||
const urlObj = new URL(url);
|
||||
let newPath = urlObj.pathname;
|
||||
|
||||
// Remove trailing slash if present
|
||||
if (newPath.endsWith('/')) {{
|
||||
newPath = newPath.slice(0, -1);
|
||||
}}
|
||||
|
||||
// Append the pagination segment
|
||||
if (pageSegment) {{
|
||||
newPath += '/' + pageSegment + '/{{PAGE_NUMBER}}';
|
||||
}} else {{
|
||||
newPath += '/{{PAGE_NUMBER}}';
|
||||
}}
|
||||
|
||||
urlObj.pathname = newPath;
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
console.error("Error creating appended path URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, path_diff.get('pageSegment'))
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for appended path parameter: {e}")
|
||||
url_template = None
|
||||
|
||||
if url_template:
|
||||
# Decode URL-encoded characters in the template
|
||||
@@ -618,6 +713,40 @@ async def detect_pagination_service(decoded_url):
|
||||
print(f"Error creating inferred URL template: {e}")
|
||||
url_template = None
|
||||
|
||||
# If still no template and we have pagination elements, try to infer from the current URL structure
|
||||
if not url_template and pagination_data['hasPagination']:
|
||||
try:
|
||||
url_template = await page.evaluate('''(originalUrl) => {
|
||||
try {
|
||||
const urlObj = new URL(originalUrl);
|
||||
let path = urlObj.pathname;
|
||||
|
||||
// Remove trailing slash if present
|
||||
if (path.endsWith('/')) {
|
||||
path = path.slice(0, -1);
|
||||
}
|
||||
|
||||
// Check if the current URL already has a pagination pattern
|
||||
const pagePattern = /\/(page|p)\/\d+$/i;
|
||||
if (pagePattern.test(path)) {
|
||||
// Replace the existing page number with placeholder
|
||||
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{{PAGE_NUMBER}}');
|
||||
} else {
|
||||
// Add pagination pattern
|
||||
path += '/page/{{PAGE_NUMBER}}';
|
||||
}
|
||||
|
||||
urlObj.pathname = path;
|
||||
return urlObj.toString();
|
||||
} catch (error) {
|
||||
console.error("Error creating fallback URL template:", error);
|
||||
return null;
|
||||
}
|
||||
}''', original_url)
|
||||
except Exception as e:
|
||||
print(f"Error creating fallback URL template: {e}")
|
||||
url_template = None
|
||||
|
||||
# Return the pagination detection results with a simplified structure
|
||||
result = {
|
||||
"status": "success",
|
||||
|
||||
Reference in New Issue
Block a user