#!/usr/bin/env python3 import asyncio import json # Mock the pagination detection logic to test the URL template generation async def test_url_template_generation(): """Test the URL template generation for the specific case""" # Test case: /vacatures -> /vacatures/page/2 original_url = "https://www.werkenbijabnamro.nl/vacatures" current_url = "https://www.werkenbijabnamro.nl/vacatures/page/2" # Simulate the JavaScript logic for path difference detection def detect_path_diff(original_url, current_url): from urllib.parse import urlparse original_parsed = urlparse(original_url) current_parsed = urlparse(current_url) original_path = original_parsed.path current_path = current_parsed.path if original_path != current_path: original_segments = [s for s in original_path.split('/') if s] current_segments = [s for s in current_path.split('/') if s] # Case 2: Current path has more segments - check for added pagination segments if len(current_segments) > len(original_segments): # Look for patterns like /page/NUMBER or /p/NUMBER at the end import re page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE) # Check the last two segments of the current path if len(current_segments) >= 2: last_two_segments = '/'.join(current_segments[-2:]) match = page_pattern.match(last_two_segments) if match: return { 'type': 'append', 'pageSegment': match.group(1), # 'page' or 'p' 'pageNumber': match.group(2), # the actual number 'originalSegments': original_segments, 'currentSegments': current_segments } # If no pattern match, check if the last segment is numeric if len(current_segments) > 0: last_segment = current_segments[-1] if last_segment.isdigit(): return { 'type': 'append', 'pageSegment': None, 'pageNumber': last_segment, 'originalSegments': original_segments, 'currentSegments': current_segments } return None # Test the path difference detection path_diff = detect_path_diff(original_url, current_url) print("Path difference detection:") print(json.dumps(path_diff, indent=2)) # Test URL template generation def create_url_template(original_url, path_diff): from urllib.parse import urlparse, urlunparse if path_diff and path_diff.get('type') == 'append': url_obj = urlparse(original_url) new_path = url_obj.path # Remove trailing slash if present if new_path.endswith('/'): new_path = new_path[:-1] # Append the pagination segment page_segment = path_diff.get('pageSegment') if page_segment: new_path += f'/{page_segment}/{{PAGE_NUMBER}}' else: new_path += '/{PAGE_NUMBER}' url_obj = url_obj._replace(pathname=new_path) return urlunparse(url_obj) return None url_template = create_url_template(original_url, path_diff) print(f"\nGenerated URL template: {url_template}") # Test with different page numbers if url_template: for page_num in [1, 2, 3, 10]: test_url = url_template.replace('{PAGE_NUMBER}', str(page_num)) print(f"Page {page_num}: {test_url}") if __name__ == "__main__": asyncio.run(test_url_template_generation())