add some pagination shizzle

Dockers/puppeteer-api/test_pagination_fix.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+import asyncio
+import json
+
+# Mock the pagination detection logic to test the URL template generation
+async def test_url_template_generation():
+    """Test the URL template generation for the specific case"""
+
+    # Test case: /vacatures -> /vacatures/page/2
+    original_url = "https://www.werkenbijabnamro.nl/vacatures"
+    current_url = "https://www.werkenbijabnamro.nl/vacatures/page/2"
+
+    # Simulate the JavaScript logic for path difference detection
+    def detect_path_diff(original_url, current_url):
+        from urllib.parse import urlparse
+
+        original_parsed = urlparse(original_url)
+        current_parsed = urlparse(current_url)
+
+        original_path = original_parsed.path
+        current_path = current_parsed.path
+
+        if original_path != current_path:
+            original_segments = [s for s in original_path.split('/') if s]
+            current_segments = [s for s in current_path.split('/') if s]
+
+            # Case 2: Current path has more segments - check for added pagination segments
+            if len(current_segments) > len(original_segments):
+                # Look for patterns like /page/NUMBER or /p/NUMBER at the end
+                import re
+                page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE)
+
+                # Check the last two segments of the current path
+                if len(current_segments) >= 2:
+                    last_two_segments = '/'.join(current_segments[-2:])
+                    match = page_pattern.match(last_two_segments)
+
+                    if match:
+                        return {
+                            'type': 'append',
+                            'pageSegment': match.group(1),  # 'page' or 'p'
+                            'pageNumber': match.group(2),   # the actual number
+                            'originalSegments': original_segments,
+                            'currentSegments': current_segments
+                        }
+
+                # If no pattern match, check if the last segment is numeric
+                if len(current_segments) > 0:
+                    last_segment = current_segments[-1]
+                    if last_segment.isdigit():
+                        return {
+                            'type': 'append',
+                            'pageSegment': None,
+                            'pageNumber': last_segment,
+                            'originalSegments': original_segments,
+                            'currentSegments': current_segments
+                        }
+
+        return None
+
+    # Test the path difference detection
+    path_diff = detect_path_diff(original_url, current_url)
+    print("Path difference detection:")
+    print(json.dumps(path_diff, indent=2))
+
+    # Test URL template generation
+    def create_url_template(original_url, path_diff):
+        from urllib.parse import urlparse, urlunparse
+
+        if path_diff and path_diff.get('type') == 'append':
+            url_obj = urlparse(original_url)
+            new_path = url_obj.path
+
+            # Remove trailing slash if present
+            if new_path.endswith('/'):
+                new_path = new_path[:-1]
+
+            # Append the pagination segment
+            page_segment = path_diff.get('pageSegment')
+            if page_segment:
+                new_path += f'/{page_segment}/{{PAGE_NUMBER}}'
+            else:
+                new_path += '/{PAGE_NUMBER}'
+
+            url_obj = url_obj._replace(pathname=new_path)
+            return urlunparse(url_obj)
+
+        return None
+
+    url_template = create_url_template(original_url, path_diff)
+    print(f"\nGenerated URL template: {url_template}")
+
+    # Test with different page numbers
+    if url_template:
+        for page_num in [1, 2, 3, 10]:
+            test_url = url_template.replace('{PAGE_NUMBER}', str(page_num))
+            print(f"Page {page_num}: {test_url}")
+
+if __name__ == "__main__":
+    asyncio.run(test_url_template_generation())