From 0525f42ab8358e640c71f628889f252a64a19c46 Mon Sep 17 00:00:00 2001 From: Bram Date: Thu, 3 Jul 2025 13:20:21 +0200 Subject: [PATCH] add some pagination shizzle --- Dockers/puppeteer-api/app/services/browser.py | 199 +++++++++++++++--- Dockers/puppeteer-api/test_pagination_fix.py | 101 +++++++++ Dockers/puppeteer-healthcheck/README.md | 7 +- 3 files changed, 267 insertions(+), 40 deletions(-) create mode 100644 Dockers/puppeteer-api/test_pagination_fix.py diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index ff59349..2a2ff6c 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -358,11 +358,18 @@ async def detect_pagination_service(decoded_url): } } - // Check for path-based pagination like /page/2/ + // Check for path-based pagination like /page/2/ or /vacatures/page/2 const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i); if (pathMatch) { return true; } + + // Check for path-based pagination where /page/2 is appended to the current path + const currentPath = window.location.pathname; + const expectedPath = currentPath.replace(/\/$/, '') + '/page/2'; + if (url.pathname === expectedPath) { + return true; + } } catch (e) {} } @@ -426,7 +433,7 @@ async def detect_pagination_service(decoded_url): if clicked: print("Successfully clicked on pagination element") # Wait for navigation to complete - await asyncio.sleep(1) + await page.waitForNavigation(waitUntil='networkidle2', timeout=10000) next_page_url = page.url else: print("No clickable pagination element found") @@ -465,7 +472,7 @@ async def detect_pagination_service(decoded_url): }} }} - // Check for path differences (like /page/1 vs /page/2) + // Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2) const originalPath = original.pathname; const currentPath = current.pathname; @@ -474,13 +481,14 @@ async def detect_pagination_service(decoded_url): const originalSegments = originalPath.split('/').filter(s => s); const currentSegments = currentPath.split('/').filter(s => s); - // Find the segment that changed + // Case 1: Same number of segments - find the one that changed if (originalSegments.length === currentSegments.length) {{ for (let i = 0; i < originalSegments.length; i++) {{ if (originalSegments[i] !== currentSegments[i]) {{ // Check if the difference is numeric if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{ pathDiff = {{ + type: 'replace', index: i, originalValue: originalSegments[i], currentValue: currentSegments[i] @@ -489,6 +497,41 @@ async def detect_pagination_service(decoded_url): }} }} }} + // Case 2: Current path has more segments - check for added pagination segments + else if (currentSegments.length > originalSegments.length) {{ + // Look for patterns like /page/NUMBER or /p/NUMBER at the end + const pagePattern = /^(page|p)\/(\d+)$/i; + + // Check the last two segments of the current path + if (currentSegments.length >= 2) {{ + const lastTwoSegments = currentSegments.slice(-2).join('/'); + const match = lastTwoSegments.match(pagePattern); + + if (match) {{ + pathDiff = {{ + type: 'append', + pageSegment: match[1], // 'page' or 'p' + pageNumber: match[2], // the actual number + originalSegments: originalSegments, + currentSegments: currentSegments + }}; + }} + }} + + // If no pattern match, check if the last segment is numeric + if (!pathDiff && currentSegments.length > 0) {{ + const lastSegment = currentSegments[currentSegments.length - 1]; + if (!isNaN(lastSegment)) {{ + pathDiff = {{ + type: 'append', + pageSegment: null, + pageNumber: lastSegment, + originalSegments: originalSegments, + currentSegments: currentSegments + }}; + }} + }} + }} }} return {{ @@ -559,41 +602,93 @@ async def detect_pagination_service(decoded_url): url_template = None elif original_parsed.get('pathDiff'): - path_index = original_parsed['pathDiff']['index'] - pagination_parameter = { - 'type': 'path', - 'index': path_index, - 'value': original_parsed['pathDiff']['currentValue'] - } + path_diff = original_parsed['pathDiff'] - # --- STEP SIZE DETECTION FOR PATH PARAM --- - orig_val = original_parsed['pathDiff']['originalValue'] - curr_val = original_parsed['pathDiff']['currentValue'] - try: - if orig_val is not None and curr_val is not None: - orig_num = int(orig_val) + if path_diff.get('type') == 'replace': + # Handle existing path segment replacement + path_index = path_diff['index'] + pagination_parameter = { + 'type': 'path', + 'index': path_index, + 'value': path_diff['currentValue'] + } + + # --- STEP SIZE DETECTION FOR PATH PARAM --- + orig_val = path_diff['originalValue'] + curr_val = path_diff['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None + + # Create URL template for path parameter replacement + try: + url_template = await page.evaluate(f'''(url, pathIndex) => {{ + try {{ + const urlObj = new URL(url); + const pathSegments = urlObj.pathname.split('/').filter(s => s); + pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; + urlObj.pathname = '/' + pathSegments.join('/'); + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating path URL template:", error); + return null; + }} + }}''', original_url, path_index) + except Exception as e: + print(f"Error creating URL template for path parameter: {e}") + url_template = None + + elif path_diff.get('type') == 'append': + # Handle new pagination segments being appended + pagination_parameter = { + 'type': 'path_append', + 'pageSegment': path_diff.get('pageSegment'), + 'pageNumber': path_diff['pageNumber'] + } + + # --- STEP SIZE DETECTION FOR APPENDED PATH PARAM --- + try: + curr_val = path_diff['pageNumber'] + # For appended pagination, assume we started from page 1 (implicit) + orig_num = 1 curr_num = int(curr_val) step_size = abs(curr_num - orig_num) - except Exception: - step_size = None + except Exception: + step_size = None - # Create URL template for path parameter - try: - url_template = await page.evaluate(f'''(url, pathIndex) => {{ - try {{ - const urlObj = new URL(url); - const pathSegments = urlObj.pathname.split('/').filter(s => s); - pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; - urlObj.pathname = '/' + pathSegments.join('/'); - return urlObj.toString(); - }} catch (error) {{ - console.error("Error creating path URL template:", error); - return null; - }} - }}''', original_url, path_index) - except Exception as e: - print(f"Error creating URL template for path parameter: {e}") - url_template = None + # Create URL template for appended path parameter + try: + url_template = await page.evaluate(f'''(url, pageSegment) => {{ + try {{ + const urlObj = new URL(url); + let newPath = urlObj.pathname; + + // Remove trailing slash if present + if (newPath.endsWith('/')) {{ + newPath = newPath.slice(0, -1); + }} + + // Append the pagination segment + if (pageSegment) {{ + newPath += '/' + pageSegment + '/{{PAGE_NUMBER}}'; + }} else {{ + newPath += '/{{PAGE_NUMBER}}'; + }} + + urlObj.pathname = newPath; + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating appended path URL template:", error); + return null; + }} + }}''', original_url, path_diff.get('pageSegment')) + except Exception as e: + print(f"Error creating URL template for appended path parameter: {e}") + url_template = None if url_template: # Decode URL-encoded characters in the template @@ -618,6 +713,40 @@ async def detect_pagination_service(decoded_url): print(f"Error creating inferred URL template: {e}") url_template = None + # If still no template and we have pagination elements, try to infer from the current URL structure + if not url_template and pagination_data['hasPagination']: + try: + url_template = await page.evaluate('''(originalUrl) => { + try { + const urlObj = new URL(originalUrl); + let path = urlObj.pathname; + + // Remove trailing slash if present + if (path.endsWith('/')) { + path = path.slice(0, -1); + } + + // Check if the current URL already has a pagination pattern + const pagePattern = /\/(page|p)\/\d+$/i; + if (pagePattern.test(path)) { + // Replace the existing page number with placeholder + path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{{PAGE_NUMBER}}'); + } else { + // Add pagination pattern + path += '/page/{{PAGE_NUMBER}}'; + } + + urlObj.pathname = path; + return urlObj.toString(); + } catch (error) { + console.error("Error creating fallback URL template:", error); + return null; + } + }''', original_url) + except Exception as e: + print(f"Error creating fallback URL template: {e}") + url_template = None + # Return the pagination detection results with a simplified structure result = { "status": "success", diff --git a/Dockers/puppeteer-api/test_pagination_fix.py b/Dockers/puppeteer-api/test_pagination_fix.py new file mode 100644 index 0000000..6b56d75 --- /dev/null +++ b/Dockers/puppeteer-api/test_pagination_fix.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import asyncio +import json + +# Mock the pagination detection logic to test the URL template generation +async def test_url_template_generation(): + """Test the URL template generation for the specific case""" + + # Test case: /vacatures -> /vacatures/page/2 + original_url = "https://www.werkenbijabnamro.nl/vacatures" + current_url = "https://www.werkenbijabnamro.nl/vacatures/page/2" + + # Simulate the JavaScript logic for path difference detection + def detect_path_diff(original_url, current_url): + from urllib.parse import urlparse + + original_parsed = urlparse(original_url) + current_parsed = urlparse(current_url) + + original_path = original_parsed.path + current_path = current_parsed.path + + if original_path != current_path: + original_segments = [s for s in original_path.split('/') if s] + current_segments = [s for s in current_path.split('/') if s] + + # Case 2: Current path has more segments - check for added pagination segments + if len(current_segments) > len(original_segments): + # Look for patterns like /page/NUMBER or /p/NUMBER at the end + import re + page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE) + + # Check the last two segments of the current path + if len(current_segments) >= 2: + last_two_segments = '/'.join(current_segments[-2:]) + match = page_pattern.match(last_two_segments) + + if match: + return { + 'type': 'append', + 'pageSegment': match.group(1), # 'page' or 'p' + 'pageNumber': match.group(2), # the actual number + 'originalSegments': original_segments, + 'currentSegments': current_segments + } + + # If no pattern match, check if the last segment is numeric + if len(current_segments) > 0: + last_segment = current_segments[-1] + if last_segment.isdigit(): + return { + 'type': 'append', + 'pageSegment': None, + 'pageNumber': last_segment, + 'originalSegments': original_segments, + 'currentSegments': current_segments + } + + return None + + # Test the path difference detection + path_diff = detect_path_diff(original_url, current_url) + print("Path difference detection:") + print(json.dumps(path_diff, indent=2)) + + # Test URL template generation + def create_url_template(original_url, path_diff): + from urllib.parse import urlparse, urlunparse + + if path_diff and path_diff.get('type') == 'append': + url_obj = urlparse(original_url) + new_path = url_obj.path + + # Remove trailing slash if present + if new_path.endswith('/'): + new_path = new_path[:-1] + + # Append the pagination segment + page_segment = path_diff.get('pageSegment') + if page_segment: + new_path += f'/{page_segment}/{{PAGE_NUMBER}}' + else: + new_path += '/{PAGE_NUMBER}' + + url_obj = url_obj._replace(pathname=new_path) + return urlunparse(url_obj) + + return None + + url_template = create_url_template(original_url, path_diff) + print(f"\nGenerated URL template: {url_template}") + + # Test with different page numbers + if url_template: + for page_num in [1, 2, 3, 10]: + test_url = url_template.replace('{PAGE_NUMBER}', str(page_num)) + print(f"Page {page_num}: {test_url}") + +if __name__ == "__main__": + asyncio.run(test_url_template_generation()) \ No newline at end of file diff --git a/Dockers/puppeteer-healthcheck/README.md b/Dockers/puppeteer-healthcheck/README.md index 9162662..a803d39 100644 --- a/Dockers/puppeteer-healthcheck/README.md +++ b/Dockers/puppeteer-healthcheck/README.md @@ -29,7 +29,6 @@ A Docker container that monitors the Puppeteer API and automatically restarts th docker run -d \ --name puppeteer-healthcheck \ -v /var/run/docker.sock:/var/run/docker.sock \ - --group-add $(getent group docker | cut -d: -f3) \ -e BASE_URL="https://puppeteer.workwithkora.com" \ -e TEST_URL="https://www.google.com" \ -e API_KEY="your-api-key" \ @@ -49,8 +48,6 @@ services: container_name: puppeteer-healthcheck volumes: - /var/run/docker.sock:/var/run/docker.sock - group_add: - - docker environment: - BASE_URL=https://puppeteer.workwithkora.com - TEST_URL=https://www.google.com @@ -83,8 +80,8 @@ The container logs all health check activities to both stdout and a log file (`/ - The container requires access to the Docker socket to restart other containers - Ensure proper API key management -- The container runs as a non-root user in the docker group for security -- Use `--group-add` or `group_add` to add the container to the docker group +- The container runs as root for Docker socket access (required for container management) +- Consider using Docker-in-Docker (DinD) or Docker socket proxy for enhanced security in production ## Troubleshooting