From 6410660dc0e58bdb3ea671856702679be9fae7c8 Mon Sep 17 00:00:00 2001 From: Bram Date: Fri, 4 Jul 2025 14:19:28 +0200 Subject: [PATCH] transiation code from js to python --- Dockers/puppeteer-api/app/services/browser.py | 329 +++++++++--------- 1 file changed, 160 insertions(+), 169 deletions(-) diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 704dc74..e1b3197 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -330,6 +330,9 @@ async def detect_pagination_service(decoded_url): if pagination_data['hasPagination']: print("Pagination detected, attempting to click on a pagination element") + # Capture the original URL before any navigation + original_url_before_navigation = page.url + # Always try to click on a pagination element, regardless of type clicked = False @@ -445,118 +448,101 @@ async def detect_pagination_service(decoded_url): # If we successfully navigated to the next page, analyze the URL difference if next_page_url: print('Searching for pagination parameter') - # Parse both URLs + # Parse both URLs using Python instead of JavaScript to avoid execution context issues try: - original_parsed = await page.evaluate('''([originalUrl]) => { - try { - const original = new URL(originalUrl); - const current = new URL(window.location.href); + from urllib.parse import urlparse, parse_qs - // Check for differences in query parameters - let paramDiff = null; + original_parsed_url = urlparse(original_url_before_navigation) + current_parsed_url = urlparse(next_page_url) - // Common pagination parameters to check - const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged']; + # Check for differences in query parameters + param_diff = None + original_params = parse_qs(original_parsed_url.query) + current_params = parse_qs(current_parsed_url.query) - for (const param of paginationParams) { - const originalValue = original.searchParams.get(param); - const currentValue = current.searchParams.get(param); + # Common pagination parameters to check + pagination_params = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'] - if (originalValue !== currentValue && currentValue !== null) { - paramDiff = { - name: param, - originalValue: originalValue, - currentValue: currentValue - }; - break; - } + for param in pagination_params: + original_value = original_params.get(param, [None])[0] + current_value = current_params.get(param, [None])[0] + + if original_value != current_value and current_value is not None: + param_diff = { + 'name': param, + 'originalValue': original_value, + 'currentValue': current_value } + break - // Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2) - const originalPath = original.pathname; - const currentPath = current.pathname; + # Check for path differences + path_diff = None + original_path = original_parsed_url.path + current_path = current_parsed_url.path - let pathDiff = null; - if (originalPath !== currentPath) { - const originalSegments = originalPath.split('/').filter(s => s); - const currentSegments = currentPath.split('/').filter(s => s); + if original_path != current_path: + original_segments = [s for s in original_path.split('/') if s] + current_segments = [s for s in current_path.split('/') if s] - // Case 1: Same number of segments - find the one that changed - if (originalSegments.length === currentSegments.length) { - for (let i = 0; i < originalSegments.length; i++) { - if (originalSegments[i] !== currentSegments[i]) { - // Check if the difference is numeric - if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) { - pathDiff = { - type: 'replace', - index: i, - originalValue: originalSegments[i], - currentValue: currentSegments[i] - }; - } + # Case 1: Same number of segments - find the one that changed + if len(original_segments) == len(current_segments): + for i, (orig_seg, curr_seg) in enumerate(zip(original_segments, current_segments)): + if orig_seg != curr_seg: + # Check if the difference is numeric + if orig_seg.isdigit() and curr_seg.isdigit(): + path_diff = { + 'type': 'replace', + 'index': i, + 'originalValue': orig_seg, + 'currentValue': curr_seg } - } - } - // Case 2: Current path has more segments - check for added pagination segments - else if (currentSegments.length > originalSegments.length) { - // Look for patterns like /page/NUMBER or /p/NUMBER at the end - const pagePattern = /^(page|p)\/(\d+)$/i; + break - // Check the last two segments of the current path - if (currentSegments.length >= 2) { - const lastTwoSegments = currentSegments.slice(-2).join('/'); - const match = lastTwoSegments.match(pagePattern); + # Case 2: Current path has more segments - check for added pagination segments + elif len(current_segments) > len(original_segments): + # Look for patterns like /page/NUMBER or /p/NUMBER at the end + import re + page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE) - if (match) { - pathDiff = { - type: 'append', - pageSegment: match[1], // 'page' or 'p' - pageNumber: match[2], // the actual number - originalSegments: originalSegments, - currentSegments: currentSegments - }; - } + # Check the last two segments of the current path + if len(current_segments) >= 2: + last_two_segments = '/'.join(current_segments[-2:]) + match = page_pattern.match(last_two_segments) + + if match: + path_diff = { + 'type': 'append', + 'pageSegment': match.group(1), # 'page' or 'p' + 'pageNumber': match.group(2), # the actual number + 'originalSegments': original_segments, + 'currentSegments': current_segments } - // If no pattern match, check if the last segment is numeric - if (!pathDiff && currentSegments.length > 0) { - const lastSegment = currentSegments[currentSegments.length - 1]; - if (!isNaN(lastSegment)) { - pathDiff = { - type: 'append', - pageSegment: null, - pageNumber: lastSegment, - originalSegments: originalSegments, - currentSegments: currentSegments - }; - } + # If no pattern match, check if the last segment is numeric + if not path_diff and current_segments: + last_segment = current_segments[-1] + if last_segment.isdigit(): + path_diff = { + 'type': 'append', + 'pageSegment': None, + 'pageNumber': last_segment, + 'originalSegments': original_segments, + 'currentSegments': current_segments } - } - } - return { - paramDiff, - pathDiff, - originalUrl: originalUrl, - currentUrl: window.location.href - }; - } catch (error) { - console.error("Error during URL analysis:", error); - return { - paramDiff: null, - pathDiff: null, - originalUrl: originalUrl, - currentUrl: window.location.href, - error: error.message - }; - } - }''', [original_url]) + original_parsed = { + 'paramDiff': param_diff, + 'pathDiff': path_diff, + 'originalUrl': original_url_before_navigation, + 'currentUrl': next_page_url + } + except Exception as e: print(f"Error during URL analysis: {e}") original_parsed = { 'paramDiff': None, 'pathDiff': None, - 'originalUrl': original_url, + 'originalUrl': original_url_before_navigation, 'currentUrl': next_page_url, 'error': str(e) } @@ -585,18 +571,20 @@ async def detect_pagination_service(decoded_url): # Create URL template for query parameter try: - url_obj = await page.evaluate('''([url, paramName]) => { - try { - const urlObj = new URL(url); - urlObj.searchParams.set(paramName, "{PAGE_NUMBER}"); - return urlObj.toString(); - } catch (error) { - console.error("Error creating URL template:", error); - return null; - } - }''', [original_url, param_name]) + from urllib.parse import urlparse, urlencode, parse_qs + + parsed_url = urlparse(original_url_before_navigation) + params = parse_qs(parsed_url.query) + params[param_name] = ['{PAGE_NUMBER}'] + + # Reconstruct the URL + new_query = urlencode(params, doseq=True) + url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}" + if new_query: + url_template += f"?{new_query}" + if parsed_url.fragment: + url_template += f"#{parsed_url.fragment}" - url_template = url_obj except Exception as e: print(f"Error creating URL template for query parameter: {e}") url_template = None @@ -626,18 +614,20 @@ async def detect_pagination_service(decoded_url): # Create URL template for path parameter replacement try: - url_template = await page.evaluate('''([url, pathIndex]) => { - try { - const urlObj = new URL(url); - const pathSegments = urlObj.pathname.split('/').filter(s => s); - pathSegments[pathIndex] = "{PAGE_NUMBER}"; - urlObj.pathname = '/' + pathSegments.join('/'); - return urlObj.toString(); - } catch (error) { - console.error("Error creating path URL template:", error); - return null; - } - }''', [original_url, path_index]) + from urllib.parse import urlparse + + parsed_url = urlparse(original_url_before_navigation) + path_segments = [s for s in parsed_url.path.split('/') if s] + path_segments[path_index] = '{PAGE_NUMBER}' + + # Reconstruct the URL + new_path = '/' + '/'.join(path_segments) + url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}" + if parsed_url.query: + url_template += f"?{parsed_url.query}" + if parsed_url.fragment: + url_template += f"#{parsed_url.fragment}" + except Exception as e: print(f"Error creating URL template for path parameter: {e}") url_template = None @@ -662,30 +652,28 @@ async def detect_pagination_service(decoded_url): # Create URL template for appended path parameter try: - url_template = await page.evaluate('''([url, pageSegment]) => { - try { - const urlObj = new URL(url); - let newPath = urlObj.pathname; + from urllib.parse import urlparse - // Remove trailing slash if present - if (newPath.endsWith('/')) { - newPath = newPath.slice(0, -1); - } + parsed_url = urlparse(original_url_before_navigation) + new_path = parsed_url.path - // Append the pagination segment - if (pageSegment) { - newPath += '/' + pageSegment + '/{PAGE_NUMBER}'; - } else { - newPath += '/{PAGE_NUMBER}'; - } + # Remove trailing slash if present + if new_path.endswith('/'): + new_path = new_path[:-1] + + # Append the pagination segment + if path_diff.get('pageSegment'): + new_path += f"/{path_diff['pageSegment']}/{{PAGE_NUMBER}}" + else: + new_path += "/{PAGE_NUMBER}" + + # Reconstruct the URL + url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{new_path}" + if parsed_url.query: + url_template += f"?{parsed_url.query}" + if parsed_url.fragment: + url_template += f"#{parsed_url.fragment}" - urlObj.pathname = newPath; - return urlObj.toString(); - } catch (error) { - console.error("Error creating appended path URL template:", error); - return null; - } - }''', [original_url, path_diff.get('pageSegment')]) except Exception as e: print(f"Error creating URL template for appended path parameter: {e}") url_template = None @@ -699,16 +687,20 @@ async def detect_pagination_service(decoded_url): if not url_template and pagination_data['detectedParameter']: param_name = pagination_data['detectedParameter']['name'] try: - url_template = await page.evaluate('''([url, paramName]) => { - try { - const urlObj = new URL(url); - urlObj.searchParams.set(paramName, "{PAGE_NUMBER}"); - return urlObj.toString(); - } catch (error) { - console.error("Error creating inferred URL template:", error); - return null; - } - }''', [original_url, param_name]) + from urllib.parse import urlparse, urlencode, parse_qs + + parsed_url = urlparse(original_url_before_navigation) + params = parse_qs(parsed_url.query) + params[param_name] = ['{PAGE_NUMBER}'] + + # Reconstruct the URL + new_query = urlencode(params, doseq=True) + url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}" + if new_query: + url_template += f"?{new_query}" + if parsed_url.fragment: + url_template += f"#{parsed_url.fragment}" + except Exception as e: print(f"Error creating inferred URL template: {e}") url_template = None @@ -716,33 +708,32 @@ async def detect_pagination_service(decoded_url): # If still no template and we have pagination elements, try to infer from the current URL structure if not url_template and pagination_data['hasPagination']: try: - url_template = await page.evaluate('''([originalUrl]) => { - try { - const urlObj = new URL(originalUrl); - let path = urlObj.pathname; + from urllib.parse import urlparse + import re - // Remove trailing slash if present - if (path.endsWith('/')) { - path = path.slice(0, -1); - } + parsed_url = urlparse(original_url_before_navigation) + path = parsed_url.path - // Check if the current URL already has a pagination pattern - const pagePattern = /\/(page|p)\/\d+$/i; - if (pagePattern.test(path)) { - // Replace the existing page number with placeholder - path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{PAGE_NUMBER}'); - } else { - // Add pagination pattern - path += '/page/{PAGE_NUMBER}'; - } + # Remove trailing slash if present + if path.endswith('/'): + path = path[:-1] + + # Check if the current URL already has a pagination pattern + page_pattern = re.compile(r'/(page|p)/\d+$', re.IGNORECASE) + if page_pattern.search(path): + # Replace the existing page number with placeholder + path = page_pattern.sub(r'/\1/{PAGE_NUMBER}', path) + else: + # Add pagination pattern + path += '/page/{PAGE_NUMBER}' + + # Reconstruct the URL + url_template = f"{parsed_url.scheme}://{parsed_url.netloc}{path}" + if parsed_url.query: + url_template += f"?{parsed_url.query}" + if parsed_url.fragment: + url_template += f"#{parsed_url.fragment}" - urlObj.pathname = path; - return urlObj.toString(); - } catch (error) { - console.error("Error creating fallback URL template:", error); - return null; - } - }''', [original_url]) except Exception as e: print(f"Error creating fallback URL template: {e}") url_template = None