Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

transiation code from js to python
Build and Push Docker Images / build-and-push (push) Successful in 22s
Details

Browse Source
This commit is contained in:
Bram
2025-07-04 14:19:28 +02:00
parent 2da4b2b5d1
commit 6410660dc0
1 changed files with 160 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+160 -169
View File
@@ -330,6 +330,9 @@ async def detect_pagination_service(decoded_url):
if pagination_data['hasPagination']:
print("Pagination detected, attempting to click on a pagination element")
# Capture the original URL before any navigation
original_url_before_navigation = page.url
# Always try to click on a pagination element, regardless of type
clicked = False
@@ -445,118 +448,101 @@ async def detect_pagination_service(decoded_url):
# If we successfully navigated to the next page, analyze the URL difference
if next_page_url:
print('Searching for pagination parameter')
# Parse both URLs
# Parse both URLs using Python instead of JavaScript to avoid execution context issues
try:
original_parsed = await page.evaluate('''([originalUrl]) => {
try {
const original = new URL(originalUrl);
const current = new URL(window.location.href);
from urllib.parse import urlparse, parse_qs
// Check for differences in query parameters
let paramDiff = null;
original_parsed_url = urlparse(original_url_before_navigation)
current_parsed_url = urlparse(next_page_url)
// Common pagination parameters to check
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
# Check for differences in query parameters
param_diff = None
original_params = parse_qs(original_parsed_url.query)
current_params = parse_qs(current_parsed_url.query)
for (const param of paginationParams) {
const originalValue = original.searchParams.get(param);
const currentValue = current.searchParams.get(param);
# Common pagination parameters to check
pagination_params = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']
if (originalValue !== currentValue && currentValue !== null) {
paramDiff = {
name: param,
originalValue: originalValue,
currentValue: currentValue
};
break;
}
for param in pagination_params:
original_value = original_params.get(param, [None])[0]
current_value = current_params.get(param, [None])[0]
if original_value != current_value and current_value is not None:
param_diff = {
'name': param,
'originalValue': original_value,
'currentValue': current_value
}
break
// Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2)
const originalPath = original.pathname;
const currentPath = current.pathname;
# Check for path differences
path_diff = None
original_path = original_parsed_url.path
current_path = current_parsed_url.path
let pathDiff = null;
if (originalPath !== currentPath) {
const originalSegments = originalPath.split('/').filter(s => s);
const currentSegments = currentPath.split('/').filter(s => s);
if original_path != current_path:
original_segments = [s for s in original_path.split('/') if s]
current_segments = [s for s in current_path.split('/') if s]
// Case 1: Same number of segments - find the one that changed
if (originalSegments.length === currentSegments.length) {
for (let i = 0; i < originalSegments.length; i++) {
if (originalSegments[i] !== currentSegments[i]) {
// Check if the difference is numeric
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {
pathDiff = {
type: 'replace',
index: i,
originalValue: originalSegments[i],
currentValue: currentSegments[i]
};
}
# Case 1: Same number of segments - find the one that changed
if len(original_segments) == len(current_segments):
for i, (orig_seg, curr_seg) in enumerate(zip(original_segments, current_segments)):
if orig_seg != curr_seg:
# Check if the difference is numeric
if orig_seg.isdigit() and curr_seg.isdigit():
path_diff = {
'type': 'replace',
'index': i,
'originalValue': orig_seg,
'currentValue': curr_seg
}
}
}
// Case 2: Current path has more segments - check for added pagination segments
else if (currentSegments.length > originalSegments.length) {
// Look for patterns like /page/NUMBER or /p/NUMBER at the end
const pagePattern = /^(page|p)\/(\d+)$/i;
break
// Check the last two segments of the current path
if (currentSegments.length >= 2) {
const lastTwoSegments = currentSegments.slice(-2).join('/');
const match = lastTwoSegments.match(pagePattern);
# Case 2: Current path has more segments - check for added pagination segments
elif len(current_segments) > len(original_segments):
# Look for patterns like /page/NUMBER or /p/NUMBER at the end
import re
page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE)
if (match) {
pathDiff = {
type: 'append',
pageSegment: match[1], // 'page' or 'p'
pageNumber: match[2], // the actual number
originalSegments: originalSegments,
currentSegments: currentSegments
};
}
# Check the last two segments of the current path
if len(current_segments) >= 2:
last_two_segments = '/'.join(current_segments[-2:])
match = page_pattern.match(last_two_segments)
if match:
path_diff = {
'type': 'append',
'pageSegment': match.group(1), # 'page' or 'p'
'pageNumber': match.group(2), # the actual number
'originalSegments': original_segments,
'currentSegments': current_segments
}
// If no pattern match, check if the last segment is numeric
if (!pathDiff && currentSegments.length > 0) {
const lastSegment = currentSegments[currentSegments.length - 1];
if (!isNaN(lastSegment)) {
pathDiff = {
type: 'append',
pageSegment: null,
pageNumber: lastSegment,
originalSegments: originalSegments,
currentSegments: currentSegments
};
}
# If no pattern match, check if the last segment is numeric
if not path_diff and current_segments:
last_segment = current_segments[-1]
if last_segment.isdigit():
path_diff = {
'type': 'append',
'pageSegment': None,
'pageNumber': last_segment,
'originalSegments': original_segments,
'currentSegments': current_segments
}
}
}
return {
paramDiff,
pathDiff,
originalUrl: originalUrl,
currentUrl: window.location.href
};
} catch (error) {
console.error("Error during URL analysis:", error);
return {
paramDiff: null,
pathDiff: null,
originalUrl: originalUrl,
currentUrl: window.location.href,
error: error.message
};
}
}''', [original_url])
original_parsed = {
'paramDiff': param_diff,
'pathDiff': path_diff,
'originalUrl': original_url_before_navigation,
'currentUrl': next_page_url
}
except Exception as e:
print(f"Error during URL analysis: {e}")
original_parsed = {
'paramDiff': None,
'pathDiff': None,
'originalUrl': original_url,
'originalUrl': original_url_before_navigation,
'currentUrl': next_page_url,
'error': str(e)
}
@@ -585,18 +571,20 @@ async def detect_pagination_service(decoded_url):
# Create URL template for query parameter
try:
url_obj = await page.evaluate('''([url, paramName]) => {
try {
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
return urlObj.toString();
} catch (error) {
console.error("Error creating URL template:", error);
return null;
}
}''', [original_url, param_name])
from urllib.parse import urlparse, urlencode, parse_qs
parsed_url = urlparse(original_url_before_navigation)
params = parse_qs(parsed_url.query)
params[param_name] = ['{PAGE_NUMBER}']
# Reconstruct the URL
new_query = urlencode(params, doseq=True)
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
if new_query:
url_template += f"?{new_query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
url_template = url_obj
except Exception as e:
print(f"Error creating URL template for query parameter: {e}")
url_template = None
@@ -626,18 +614,20 @@ async def detect_pagination_service(decoded_url):
# Create URL template for path parameter replacement
try:
url_template = await page.evaluate('''([url, pathIndex]) => {
try {
const urlObj = new URL(url);
const pathSegments = urlObj.pathname.split('/').filter(s => s);
pathSegments[pathIndex] = "{PAGE_NUMBER}";
urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString();
} catch (error) {
console.error("Error creating path URL template:", error);
return null;
}
}''', [original_url, path_index])
from urllib.parse import urlparse
parsed_url = urlparse(original_url_before_navigation)
path_segments = [s for s in parsed_url.path.split('/') if s]
path_segments[path_index] = '{PAGE_NUMBER}'
# Reconstruct the URL
new_path = '/' + '/'.join(path_segments)
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}"
if parsed_url.query:
url_template += f"?{parsed_url.query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
except Exception as e:
print(f"Error creating URL template for path parameter: {e}")
url_template = None
@@ -662,30 +652,28 @@ async def detect_pagination_service(decoded_url):
# Create URL template for appended path parameter
try:
url_template = await page.evaluate('''([url, pageSegment]) => {
try {
const urlObj = new URL(url);
let newPath = urlObj.pathname;
from urllib.parse import urlparse
// Remove trailing slash if present
if (newPath.endsWith('/')) {
newPath = newPath.slice(0, -1);
}
parsed_url = urlparse(original_url_before_navigation)
new_path = parsed_url.path
// Append the pagination segment
if (pageSegment) {
newPath += '/' + pageSegment + '/{PAGE_NUMBER}';
} else {
newPath += '/{PAGE_NUMBER}';
}
# Remove trailing slash if present
if new_path.endswith('/'):
new_path = new_path[:-1]
# Append the pagination segment
if path_diff.get('pageSegment'):
new_path += f"/{path_diff['pageSegment']}/{{PAGE_NUMBER}}"
else:
new_path += "/{PAGE_NUMBER}"
# Reconstruct the URL
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}"
if parsed_url.query:
url_template += f"?{parsed_url.query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
urlObj.pathname = newPath;
return urlObj.toString();
} catch (error) {
console.error("Error creating appended path URL template:", error);
return null;
}
}''', [original_url, path_diff.get('pageSegment')])
except Exception as e:
print(f"Error creating URL template for appended path parameter: {e}")
url_template = None
@@ -699,16 +687,20 @@ async def detect_pagination_service(decoded_url):
if not url_template and pagination_data['detectedParameter']:
param_name = pagination_data['detectedParameter']['name']
try:
url_template = await page.evaluate('''([url, paramName]) => {
try {
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
return urlObj.toString();
} catch (error) {
console.error("Error creating inferred URL template:", error);
return null;
}
}''', [original_url, param_name])
from urllib.parse import urlparse, urlencode, parse_qs
parsed_url = urlparse(original_url_before_navigation)
params = parse_qs(parsed_url.query)
params[param_name] = ['{PAGE_NUMBER}']
# Reconstruct the URL
new_query = urlencode(params, doseq=True)
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
if new_query:
url_template += f"?{new_query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
except Exception as e:
print(f"Error creating inferred URL template: {e}")
url_template = None
@@ -716,33 +708,32 @@ async def detect_pagination_service(decoded_url):
# If still no template and we have pagination elements, try to infer from the current URL structure
if not url_template and pagination_data['hasPagination']:
try:
url_template = await page.evaluate('''([originalUrl]) => {
try {
const urlObj = new URL(originalUrl);
let path = urlObj.pathname;
from urllib.parse import urlparse
import re
// Remove trailing slash if present
if (path.endsWith('/')) {
path = path.slice(0, -1);
}
parsed_url = urlparse(original_url_before_navigation)
path = parsed_url.path
// Check if the current URL already has a pagination pattern
const pagePattern = /\/(page|p)\/\d+$/i;
if (pagePattern.test(path)) {
// Replace the existing page number with placeholder
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{PAGE_NUMBER}');
} else {
// Add pagination pattern
path += '/page/{PAGE_NUMBER}';
}
# Remove trailing slash if present
if path.endswith('/'):
path = path[:-1]
# Check if the current URL already has a pagination pattern
page_pattern = re.compile(r'/(page|p)/\d+$', re.IGNORECASE)
if page_pattern.search(path):
# Replace the existing page number with placeholder
path = page_pattern.sub(r'/\1/{PAGE_NUMBER}', path)
else:
# Add pagination pattern
path += '/page/{PAGE_NUMBER}'
# Reconstruct the URL
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{path}"
if parsed_url.query:
url_template += f"?{parsed_url.query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
urlObj.pathname = path;
return urlObj.toString();
} catch (error) {
console.error("Error creating fallback URL template:", error);
return null;
}
}''', [original_url])
except Exception as e:
print(f"Error creating fallback URL template: {e}")
url_template = None