add some pagination shizzle

Dockers/puppeteer-api/app/services/browser.py

@@ -358,11 +358,18 @@ async def detect_pagination_service(decoded_url):
                                                 }
                                             }
 
-                                            // Check for path-based pagination like /page/2/
+                                            // Check for path-based pagination like /page/2/ or /vacatures/page/2
                                             const pathMatch = url.pathname.match(/\/(page|p)\/2\/?$/i);
                                             if (pathMatch) {
                                                 return true;
                                             }
+
+                                            // Check for path-based pagination where /page/2 is appended to the current path
+                                            const currentPath = window.location.pathname;
+                                            const expectedPath = currentPath.replace(/\/$/, '') + '/page/2';
+                                            if (url.pathname === expectedPath) {
+                                                return true;
+                                            }
                                         } catch (e) {}
                                     }
 
@@ -426,7 +433,7 @@ async def detect_pagination_service(decoded_url):
                     if clicked:
                         print("Successfully clicked on pagination element")
                         # Wait for navigation to complete
-                        await asyncio.sleep(1)
+                        await page.waitForNavigation(waitUntil='networkidle2', timeout=10000)
                         next_page_url = page.url
                     else:
                         print("No clickable pagination element found")
@@ -465,7 +472,7 @@ async def detect_pagination_service(decoded_url):
                                 }}
                             }}
 
-                            // Check for path differences (like /page/1 vs /page/2)
+                            // Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2)
                             const originalPath = original.pathname;
                             const currentPath = current.pathname;
 
@@ -474,13 +481,14 @@ async def detect_pagination_service(decoded_url):
                                 const originalSegments = originalPath.split('/').filter(s => s);
                                 const currentSegments = currentPath.split('/').filter(s => s);
 
-                                // Find the segment that changed
+                                // Case 1: Same number of segments - find the one that changed
                                 if (originalSegments.length === currentSegments.length) {{
                                     for (let i = 0; i < originalSegments.length; i++) {{
                                         if (originalSegments[i] !== currentSegments[i]) {{
                                             // Check if the difference is numeric
                                             if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
                                                 pathDiff = {{
+                                                    type: 'replace',
                                                     index: i,
                                                     originalValue: originalSegments[i],
                                                     currentValue: currentSegments[i]
@@ -489,6 +497,41 @@ async def detect_pagination_service(decoded_url):
                                         }}
                                     }}
                                 }}
+                                // Case 2: Current path has more segments - check for added pagination segments
+                                else if (currentSegments.length > originalSegments.length) {{
+                                    // Look for patterns like /page/NUMBER or /p/NUMBER at the end
+                                    const pagePattern = /^(page|p)\/(\d+)$/i;
+
+                                    // Check the last two segments of the current path
+                                    if (currentSegments.length >= 2) {{
+                                        const lastTwoSegments = currentSegments.slice(-2).join('/');
+                                        const match = lastTwoSegments.match(pagePattern);
+
+                                        if (match) {{
+                                            pathDiff = {{
+                                                type: 'append',
+                                                pageSegment: match[1], // 'page' or 'p'
+                                                pageNumber: match[2],  // the actual number
+                                                originalSegments: originalSegments,
+                                                currentSegments: currentSegments
+                                            }};
+                                        }}
+                                    }}
+
+                                    // If no pattern match, check if the last segment is numeric
+                                    if (!pathDiff && currentSegments.length > 0) {{
+                                        const lastSegment = currentSegments[currentSegments.length - 1];
+                                        if (!isNaN(lastSegment)) {{
+                                            pathDiff = {{
+                                                type: 'append',
+                                                pageSegment: null,
+                                                pageNumber: lastSegment,
+                                                originalSegments: originalSegments,
+                                                currentSegments: currentSegments
+                                            }};
+                                        }}
+                                    }}
+                                }}
                             }}
 
                             return {{
@@ -559,41 +602,93 @@ async def detect_pagination_service(decoded_url):
                         url_template = None
 
                 elif original_parsed.get('pathDiff'):
-                    path_index = original_parsed['pathDiff']['index']
+                    path_diff = original_parsed['pathDiff']
-                    pagination_parameter = {
-                        'type': 'path',
-                        'index': path_index,
-                        'value': original_parsed['pathDiff']['currentValue']
-                    }
 
-                    # --- STEP SIZE DETECTION FOR PATH PARAM ---
+                    if path_diff.get('type') == 'replace':
-                    orig_val = original_parsed['pathDiff']['originalValue']
+                        # Handle existing path segment replacement
-                    curr_val = original_parsed['pathDiff']['currentValue']
+                        path_index = path_diff['index']
-                    try:
+                        pagination_parameter = {
-                        if orig_val is not None and curr_val is not None:
+                            'type': 'path',
-                            orig_num = int(orig_val)
+                            'index': path_index,
+                            'value': path_diff['currentValue']
+                        }
+
+                        # --- STEP SIZE DETECTION FOR PATH PARAM ---
+                        orig_val = path_diff['originalValue']
+                        curr_val = path_diff['currentValue']
+                        try:
+                            if orig_val is not None and curr_val is not None:
+                                orig_num = int(orig_val)
+                                curr_num = int(curr_val)
+                                step_size = abs(curr_num - orig_num)
+                        except Exception:
+                            step_size = None
+
+                        # Create URL template for path parameter replacement
+                        try:
+                            url_template = await page.evaluate(f'''(url, pathIndex) => {{
+                                try {{
+                                    const urlObj = new URL(url);
+                                    const pathSegments = urlObj.pathname.split('/').filter(s => s);
+                                    pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
+                                    urlObj.pathname = '/' + pathSegments.join('/');
+                                    return urlObj.toString();
+                                }} catch (error) {{
+                                    console.error("Error creating path URL template:", error);
+                                    return null;
+                                }}
+                            }}''', original_url, path_index)
+                        except Exception as e:
+                            print(f"Error creating URL template for path parameter: {e}")
+                            url_template = None
+
+                    elif path_diff.get('type') == 'append':
+                        # Handle new pagination segments being appended
+                        pagination_parameter = {
+                            'type': 'path_append',
+                            'pageSegment': path_diff.get('pageSegment'),
+                            'pageNumber': path_diff['pageNumber']
+                        }
+
+                        # --- STEP SIZE DETECTION FOR APPENDED PATH PARAM ---
+                        try:
+                            curr_val = path_diff['pageNumber']
+                            # For appended pagination, assume we started from page 1 (implicit)
+                            orig_num = 1
                             curr_num = int(curr_val)
                             step_size = abs(curr_num - orig_num)
-                    except Exception:
+                        except Exception:
-                        step_size = None
+                            step_size = None
 
-                    # Create URL template for path parameter
+                        # Create URL template for appended path parameter
-                    try:
+                        try:
-                        url_template = await page.evaluate(f'''(url, pathIndex) => {{
+                            url_template = await page.evaluate(f'''(url, pageSegment) => {{
-                            try {{
+                                try {{
-                                const urlObj = new URL(url);
+                                    const urlObj = new URL(url);
-                                const pathSegments = urlObj.pathname.split('/').filter(s => s);
+                                    let newPath = urlObj.pathname;
-                                pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
+
-                                urlObj.pathname = '/' + pathSegments.join('/');
+                                    // Remove trailing slash if present
-                                return urlObj.toString();
+                                    if (newPath.endsWith('/')) {{
-                            }} catch (error) {{
+                                        newPath = newPath.slice(0, -1);
-                                console.error("Error creating path URL template:", error);
+                                    }}
-                                return null;
+
-                            }}
+                                    // Append the pagination segment
-                        }}''', original_url, path_index)
+                                    if (pageSegment) {{
-                    except Exception as e:
+                                        newPath += '/' + pageSegment + '/{{PAGE_NUMBER}}';
-                        print(f"Error creating URL template for path parameter: {e}")
+                                    }} else {{
-                        url_template = None
+                                        newPath += '/{{PAGE_NUMBER}}';
+                                    }}
+
+                                    urlObj.pathname = newPath;
+                                    return urlObj.toString();
+                                }} catch (error) {{
+                                    console.error("Error creating appended path URL template:", error);
+                                    return null;
+                                }}
+                            }}''', original_url, path_diff.get('pageSegment'))
+                        except Exception as e:
+                            print(f"Error creating URL template for appended path parameter: {e}")
+                            url_template = None
 
             if url_template:
                 # Decode URL-encoded characters in the template
@@ -618,6 +713,40 @@ async def detect_pagination_service(decoded_url):
                     print(f"Error creating inferred URL template: {e}")
                     url_template = None
 
+            # If still no template and we have pagination elements, try to infer from the current URL structure
+            if not url_template and pagination_data['hasPagination']:
+                try:
+                    url_template = await page.evaluate('''(originalUrl) => {
+                        try {
+                            const urlObj = new URL(originalUrl);
+                            let path = urlObj.pathname;
+
+                            // Remove trailing slash if present
+                            if (path.endsWith('/')) {
+                                path = path.slice(0, -1);
+                            }
+
+                            // Check if the current URL already has a pagination pattern
+                            const pagePattern = /\/(page|p)\/\d+$/i;
+                            if (pagePattern.test(path)) {
+                                // Replace the existing page number with placeholder
+                                path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{{PAGE_NUMBER}}');
+                            } else {
+                                // Add pagination pattern
+                                path += '/page/{{PAGE_NUMBER}}';
+                            }
+
+                            urlObj.pathname = path;
+                            return urlObj.toString();
+                        } catch (error) {
+                            console.error("Error creating fallback URL template:", error);
+                            return null;
+                        }
+                    }''', original_url)
+                except Exception as e:
+                    print(f"Error creating fallback URL template: {e}")
+                    url_template = None
+
             # Return the pagination detection results with a simplified structure
             result = {
                 "status": "success",

Dockers/puppeteer-api/test_pagination_fix.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+import asyncio
+import json
+
+# Mock the pagination detection logic to test the URL template generation
+async def test_url_template_generation():
+    """Test the URL template generation for the specific case"""
+
+    # Test case: /vacatures -> /vacatures/page/2
+    original_url = "https://www.werkenbijabnamro.nl/vacatures"
+    current_url = "https://www.werkenbijabnamro.nl/vacatures/page/2"
+
+    # Simulate the JavaScript logic for path difference detection
+    def detect_path_diff(original_url, current_url):
+        from urllib.parse import urlparse
+
+        original_parsed = urlparse(original_url)
+        current_parsed = urlparse(current_url)
+
+        original_path = original_parsed.path
+        current_path = current_parsed.path
+
+        if original_path != current_path:
+            original_segments = [s for s in original_path.split('/') if s]
+            current_segments = [s for s in current_path.split('/') if s]
+
+            # Case 2: Current path has more segments - check for added pagination segments
+            if len(current_segments) > len(original_segments):
+                # Look for patterns like /page/NUMBER or /p/NUMBER at the end
+                import re
+                page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE)
+
+                # Check the last two segments of the current path
+                if len(current_segments) >= 2:
+                    last_two_segments = '/'.join(current_segments[-2:])
+                    match = page_pattern.match(last_two_segments)
+
+                    if match:
+                        return {
+                            'type': 'append',
+                            'pageSegment': match.group(1),  # 'page' or 'p'
+                            'pageNumber': match.group(2),   # the actual number
+                            'originalSegments': original_segments,
+                            'currentSegments': current_segments
+                        }
+
+                # If no pattern match, check if the last segment is numeric
+                if len(current_segments) > 0:
+                    last_segment = current_segments[-1]
+                    if last_segment.isdigit():
+                        return {
+                            'type': 'append',
+                            'pageSegment': None,
+                            'pageNumber': last_segment,
+                            'originalSegments': original_segments,
+                            'currentSegments': current_segments
+                        }
+
+        return None
+
+    # Test the path difference detection
+    path_diff = detect_path_diff(original_url, current_url)
+    print("Path difference detection:")
+    print(json.dumps(path_diff, indent=2))
+
+    # Test URL template generation
+    def create_url_template(original_url, path_diff):
+        from urllib.parse import urlparse, urlunparse
+
+        if path_diff and path_diff.get('type') == 'append':
+            url_obj = urlparse(original_url)
+            new_path = url_obj.path
+
+            # Remove trailing slash if present
+            if new_path.endswith('/'):
+                new_path = new_path[:-1]
+
+            # Append the pagination segment
+            page_segment = path_diff.get('pageSegment')
+            if page_segment:
+                new_path += f'/{page_segment}/{{PAGE_NUMBER}}'
+            else:
+                new_path += '/{PAGE_NUMBER}'
+
+            url_obj = url_obj._replace(pathname=new_path)
+            return urlunparse(url_obj)
+
+        return None
+
+    url_template = create_url_template(original_url, path_diff)
+    print(f"\nGenerated URL template: {url_template}")
+
+    # Test with different page numbers
+    if url_template:
+        for page_num in [1, 2, 3, 10]:
+            test_url = url_template.replace('{PAGE_NUMBER}', str(page_num))
+            print(f"Page {page_num}: {test_url}")
+
+if __name__ == "__main__":
+    asyncio.run(test_url_template_generation())

Dockers/puppeteer-healthcheck/README.md

@@ -29,7 +29,6 @@ A Docker container that monitors the Puppeteer API and automatically restarts th
 docker run -d \
   --name puppeteer-healthcheck \
   -v /var/run/docker.sock:/var/run/docker.sock \
-  --group-add $(getent group docker | cut -d: -f3) \
   -e BASE_URL="https://puppeteer.workwithkora.com" \
   -e TEST_URL="https://www.google.com" \
   -e API_KEY="your-api-key" \
@@ -49,8 +48,6 @@ services:
     container_name: puppeteer-healthcheck
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-    group_add:
-      - docker
     environment:
       - BASE_URL=https://puppeteer.workwithkora.com
       - TEST_URL=https://www.google.com
@@ -83,8 +80,8 @@ The container logs all health check activities to both stdout and a log file (`/
 
 - The container requires access to the Docker socket to restart other containers
 - Ensure proper API key management
-- The container runs as a non-root user in the docker group for security
+- The container runs as root for Docker socket access (required for container management)
-- Use `--group-add` or `group_add` to add the container to the docker group
+- Consider using Docker-in-Docker (DinD) or Docker socket proxy for enhanced security in production
 
 ## Troubleshooting