Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

transiation code from js to python
Build and Push Docker Images / build-and-push (push) Successful in 22s
Details

Browse Source
This commit is contained in:
Bram
2025-07-04 14:19:28 +02:00
parent 2da4b2b5d1
commit 6410660dc0
1 changed files with 160 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+160 -169
View File
@@ -330,6 +330,9 @@ async def detect_pagination_service(decoded_url):
if pagination_data['hasPagination']: if pagination_data['hasPagination']:
print("Pagination detected, attempting to click on a pagination element") print("Pagination detected, attempting to click on a pagination element")
# Capture the original URL before any navigation
original_url_before_navigation = page.url
# Always try to click on a pagination element, regardless of type # Always try to click on a pagination element, regardless of type
clicked = False clicked = False
@@ -445,118 +448,101 @@ async def detect_pagination_service(decoded_url):
# If we successfully navigated to the next page, analyze the URL difference # If we successfully navigated to the next page, analyze the URL difference
if next_page_url: if next_page_url:
print('Searching for pagination parameter') print('Searching for pagination parameter')
# Parse both URLs # Parse both URLs using Python instead of JavaScript to avoid execution context issues
try: try:
original_parsed = await page.evaluate('''([originalUrl]) => { from urllib.parse import urlparse, parse_qs
try {
const original = new URL(originalUrl);
const current = new URL(window.location.href);
// Check for differences in query parameters original_parsed_url = urlparse(original_url_before_navigation)
let paramDiff = null; current_parsed_url = urlparse(next_page_url)
// Common pagination parameters to check # Check for differences in query parameters
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']; param_diff = None
original_params = parse_qs(original_parsed_url.query)
current_params = parse_qs(current_parsed_url.query)
for (const param of paginationParams) { # Common pagination parameters to check
const originalValue = original.searchParams.get(param); pagination_params = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']
const currentValue = current.searchParams.get(param);
if (originalValue !== currentValue && currentValue !== null) { for param in pagination_params:
paramDiff = { original_value = original_params.get(param, [None])[0]
name: param, current_value = current_params.get(param, [None])[0]
originalValue: originalValue,
currentValue: currentValue if original_value != current_value and current_value is not None:
}; param_diff = {
break; 'name': param,
} 'originalValue': original_value,
'currentValue': current_value
} }
break
// Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2) # Check for path differences
const originalPath = original.pathname; path_diff = None
const currentPath = current.pathname; original_path = original_parsed_url.path
current_path = current_parsed_url.path
let pathDiff = null; if original_path != current_path:
if (originalPath !== currentPath) { original_segments = [s for s in original_path.split('/') if s]
const originalSegments = originalPath.split('/').filter(s => s); current_segments = [s for s in current_path.split('/') if s]
const currentSegments = currentPath.split('/').filter(s => s);
// Case 1: Same number of segments - find the one that changed # Case 1: Same number of segments - find the one that changed
if (originalSegments.length === currentSegments.length) { if len(original_segments) == len(current_segments):
for (let i = 0; i < originalSegments.length; i++) { for i, (orig_seg, curr_seg) in enumerate(zip(original_segments, current_segments)):
if (originalSegments[i] !== currentSegments[i]) { if orig_seg != curr_seg:
// Check if the difference is numeric # Check if the difference is numeric
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) { if orig_seg.isdigit() and curr_seg.isdigit():
pathDiff = { path_diff = {
type: 'replace', 'type': 'replace',
index: i, 'index': i,
originalValue: originalSegments[i], 'originalValue': orig_seg,
currentValue: currentSegments[i] 'currentValue': curr_seg
};
}
} }
} break
}
// Case 2: Current path has more segments - check for added pagination segments
else if (currentSegments.length > originalSegments.length) {
// Look for patterns like /page/NUMBER or /p/NUMBER at the end
const pagePattern = /^(page|p)\/(\d+)$/i;
// Check the last two segments of the current path # Case 2: Current path has more segments - check for added pagination segments
if (currentSegments.length >= 2) { elif len(current_segments) > len(original_segments):
const lastTwoSegments = currentSegments.slice(-2).join('/'); # Look for patterns like /page/NUMBER or /p/NUMBER at the end
const match = lastTwoSegments.match(pagePattern); import re
page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE)
if (match) { # Check the last two segments of the current path
pathDiff = { if len(current_segments) >= 2:
type: 'append', last_two_segments = '/'.join(current_segments[-2:])
pageSegment: match[1], // 'page' or 'p' match = page_pattern.match(last_two_segments)
pageNumber: match[2], // the actual number
originalSegments: originalSegments, if match:
currentSegments: currentSegments path_diff = {
}; 'type': 'append',
} 'pageSegment': match.group(1), # 'page' or 'p'
'pageNumber': match.group(2), # the actual number
'originalSegments': original_segments,
'currentSegments': current_segments
} }
// If no pattern match, check if the last segment is numeric # If no pattern match, check if the last segment is numeric
if (!pathDiff && currentSegments.length > 0) { if not path_diff and current_segments:
const lastSegment = currentSegments[currentSegments.length - 1]; last_segment = current_segments[-1]
if (!isNaN(lastSegment)) { if last_segment.isdigit():
pathDiff = { path_diff = {
type: 'append', 'type': 'append',
pageSegment: null, 'pageSegment': None,
pageNumber: lastSegment, 'pageNumber': last_segment,
originalSegments: originalSegments, 'originalSegments': original_segments,
currentSegments: currentSegments 'currentSegments': current_segments
};
}
} }
}
}
return { original_parsed = {
paramDiff, 'paramDiff': param_diff,
pathDiff, 'pathDiff': path_diff,
originalUrl: originalUrl, 'originalUrl': original_url_before_navigation,
currentUrl: window.location.href 'currentUrl': next_page_url
}; }
} catch (error) {
console.error("Error during URL analysis:", error);
return {
paramDiff: null,
pathDiff: null,
originalUrl: originalUrl,
currentUrl: window.location.href,
error: error.message
};
}
}''', [original_url])
except Exception as e: except Exception as e:
print(f"Error during URL analysis: {e}") print(f"Error during URL analysis: {e}")
original_parsed = { original_parsed = {
'paramDiff': None, 'paramDiff': None,
'pathDiff': None, 'pathDiff': None,
'originalUrl': original_url, 'originalUrl': original_url_before_navigation,
'currentUrl': next_page_url, 'currentUrl': next_page_url,
'error': str(e) 'error': str(e)
} }
@@ -585,18 +571,20 @@ async def detect_pagination_service(decoded_url):
# Create URL template for query parameter # Create URL template for query parameter
try: try:
url_obj = await page.evaluate('''([url, paramName]) => { from urllib.parse import urlparse, urlencode, parse_qs
try {
const urlObj = new URL(url); parsed_url = urlparse(original_url_before_navigation)
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}"); params = parse_qs(parsed_url.query)
return urlObj.toString(); params[param_name] = ['{PAGE_NUMBER}']
} catch (error) {
console.error("Error creating URL template:", error); # Reconstruct the URL
return null; new_query = urlencode(params, doseq=True)
} url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
}''', [original_url, param_name]) if new_query:
url_template += f"?{new_query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
url_template = url_obj
except Exception as e: except Exception as e:
print(f"Error creating URL template for query parameter: {e}") print(f"Error creating URL template for query parameter: {e}")
url_template = None url_template = None
@@ -626,18 +614,20 @@ async def detect_pagination_service(decoded_url):
# Create URL template for path parameter replacement # Create URL template for path parameter replacement
try: try:
url_template = await page.evaluate('''([url, pathIndex]) => { from urllib.parse import urlparse
try {
const urlObj = new URL(url); parsed_url = urlparse(original_url_before_navigation)
const pathSegments = urlObj.pathname.split('/').filter(s => s); path_segments = [s for s in parsed_url.path.split('/') if s]
pathSegments[pathIndex] = "{PAGE_NUMBER}"; path_segments[path_index] = '{PAGE_NUMBER}'
urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString(); # Reconstruct the URL
} catch (error) { new_path = '/' + '/'.join(path_segments)
console.error("Error creating path URL template:", error); url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}"
return null; if parsed_url.query:
} url_template += f"?{parsed_url.query}"
}''', [original_url, path_index]) if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
except Exception as e: except Exception as e:
print(f"Error creating URL template for path parameter: {e}") print(f"Error creating URL template for path parameter: {e}")
url_template = None url_template = None
@@ -662,30 +652,28 @@ async def detect_pagination_service(decoded_url):
# Create URL template for appended path parameter # Create URL template for appended path parameter
try: try:
url_template = await page.evaluate('''([url, pageSegment]) => { from urllib.parse import urlparse
try {
const urlObj = new URL(url);
let newPath = urlObj.pathname;
// Remove trailing slash if present parsed_url = urlparse(original_url_before_navigation)
if (newPath.endsWith('/')) { new_path = parsed_url.path
newPath = newPath.slice(0, -1);
}
// Append the pagination segment # Remove trailing slash if present
if (pageSegment) { if new_path.endswith('/'):
newPath += '/' + pageSegment + '/{PAGE_NUMBER}'; new_path = new_path[:-1]
} else {
newPath += '/{PAGE_NUMBER}'; # Append the pagination segment
} if path_diff.get('pageSegment'):
new_path += f"/{path_diff['pageSegment']}/{{PAGE_NUMBER}}"
else:
new_path += "/{PAGE_NUMBER}"
# Reconstruct the URL
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}"
if parsed_url.query:
url_template += f"?{parsed_url.query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
urlObj.pathname = newPath;
return urlObj.toString();
} catch (error) {
console.error("Error creating appended path URL template:", error);
return null;
}
}''', [original_url, path_diff.get('pageSegment')])
except Exception as e: except Exception as e:
print(f"Error creating URL template for appended path parameter: {e}") print(f"Error creating URL template for appended path parameter: {e}")
url_template = None url_template = None
@@ -699,16 +687,20 @@ async def detect_pagination_service(decoded_url):
if not url_template and pagination_data['detectedParameter']: if not url_template and pagination_data['detectedParameter']:
param_name = pagination_data['detectedParameter']['name'] param_name = pagination_data['detectedParameter']['name']
try: try:
url_template = await page.evaluate('''([url, paramName]) => { from urllib.parse import urlparse, urlencode, parse_qs
try {
const urlObj = new URL(url); parsed_url = urlparse(original_url_before_navigation)
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}"); params = parse_qs(parsed_url.query)
return urlObj.toString(); params[param_name] = ['{PAGE_NUMBER}']
} catch (error) {
console.error("Error creating inferred URL template:", error); # Reconstruct the URL
return null; new_query = urlencode(params, doseq=True)
} url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
}''', [original_url, param_name]) if new_query:
url_template += f"?{new_query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
except Exception as e: except Exception as e:
print(f"Error creating inferred URL template: {e}") print(f"Error creating inferred URL template: {e}")
url_template = None url_template = None
@@ -716,33 +708,32 @@ async def detect_pagination_service(decoded_url):
# If still no template and we have pagination elements, try to infer from the current URL structure # If still no template and we have pagination elements, try to infer from the current URL structure
if not url_template and pagination_data['hasPagination']: if not url_template and pagination_data['hasPagination']:
try: try:
url_template = await page.evaluate('''([originalUrl]) => { from urllib.parse import urlparse
try { import re
const urlObj = new URL(originalUrl);
let path = urlObj.pathname;
// Remove trailing slash if present parsed_url = urlparse(original_url_before_navigation)
if (path.endsWith('/')) { path = parsed_url.path
path = path.slice(0, -1);
}
// Check if the current URL already has a pagination pattern # Remove trailing slash if present
const pagePattern = /\/(page|p)\/\d+$/i; if path.endswith('/'):
if (pagePattern.test(path)) { path = path[:-1]
// Replace the existing page number with placeholder
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{PAGE_NUMBER}'); # Check if the current URL already has a pagination pattern
} else { page_pattern = re.compile(r'/(page|p)/\d+$', re.IGNORECASE)
// Add pagination pattern if page_pattern.search(path):
path += '/page/{PAGE_NUMBER}'; # Replace the existing page number with placeholder
} path = page_pattern.sub(r'/\1/{PAGE_NUMBER}', path)
else:
# Add pagination pattern
path += '/page/{PAGE_NUMBER}'
# Reconstruct the URL
url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{path}"
if parsed_url.query:
url_template += f"?{parsed_url.query}"
if parsed_url.fragment:
url_template += f"#{parsed_url.fragment}"
urlObj.pathname = path;
return urlObj.toString();
} catch (error) {
console.error("Error creating fallback URL template:", error);
return null;
}
}''', [original_url])
except Exception as e: except Exception as e:
print(f"Error creating fallback URL template: {e}") print(f"Error creating fallback URL template: {e}")
url_template = None url_template = None