101 lines
3.9 KiB
Python
101 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
# Mock the pagination detection logic to test the URL template generation
|
|
async def test_url_template_generation():
|
|
"""Test the URL template generation for the specific case"""
|
|
|
|
# Test case: /vacatures -> /vacatures/page/2
|
|
original_url = "https://www.werkenbijabnamro.nl/vacatures"
|
|
current_url = "https://www.werkenbijabnamro.nl/vacatures/page/2"
|
|
|
|
# Simulate the JavaScript logic for path difference detection
|
|
def detect_path_diff(original_url, current_url):
|
|
from urllib.parse import urlparse
|
|
|
|
original_parsed = urlparse(original_url)
|
|
current_parsed = urlparse(current_url)
|
|
|
|
original_path = original_parsed.path
|
|
current_path = current_parsed.path
|
|
|
|
if original_path != current_path:
|
|
original_segments = [s for s in original_path.split('/') if s]
|
|
current_segments = [s for s in current_path.split('/') if s]
|
|
|
|
# Case 2: Current path has more segments - check for added pagination segments
|
|
if len(current_segments) > len(original_segments):
|
|
# Look for patterns like /page/NUMBER or /p/NUMBER at the end
|
|
import re
|
|
page_pattern = re.compile(r'^(page|p)/(\d+)$', re.IGNORECASE)
|
|
|
|
# Check the last two segments of the current path
|
|
if len(current_segments) >= 2:
|
|
last_two_segments = '/'.join(current_segments[-2:])
|
|
match = page_pattern.match(last_two_segments)
|
|
|
|
if match:
|
|
return {
|
|
'type': 'append',
|
|
'pageSegment': match.group(1), # 'page' or 'p'
|
|
'pageNumber': match.group(2), # the actual number
|
|
'originalSegments': original_segments,
|
|
'currentSegments': current_segments
|
|
}
|
|
|
|
# If no pattern match, check if the last segment is numeric
|
|
if len(current_segments) > 0:
|
|
last_segment = current_segments[-1]
|
|
if last_segment.isdigit():
|
|
return {
|
|
'type': 'append',
|
|
'pageSegment': None,
|
|
'pageNumber': last_segment,
|
|
'originalSegments': original_segments,
|
|
'currentSegments': current_segments
|
|
}
|
|
|
|
return None
|
|
|
|
# Test the path difference detection
|
|
path_diff = detect_path_diff(original_url, current_url)
|
|
print("Path difference detection:")
|
|
print(json.dumps(path_diff, indent=2))
|
|
|
|
# Test URL template generation
|
|
def create_url_template(original_url, path_diff):
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
if path_diff and path_diff.get('type') == 'append':
|
|
url_obj = urlparse(original_url)
|
|
new_path = url_obj.path
|
|
|
|
# Remove trailing slash if present
|
|
if new_path.endswith('/'):
|
|
new_path = new_path[:-1]
|
|
|
|
# Append the pagination segment
|
|
page_segment = path_diff.get('pageSegment')
|
|
if page_segment:
|
|
new_path += f'/{page_segment}/{{PAGE_NUMBER}}'
|
|
else:
|
|
new_path += '/{PAGE_NUMBER}'
|
|
|
|
url_obj = url_obj._replace(pathname=new_path)
|
|
return urlunparse(url_obj)
|
|
|
|
return None
|
|
|
|
url_template = create_url_template(original_url, path_diff)
|
|
print(f"\nGenerated URL template: {url_template}")
|
|
|
|
# Test with different page numbers
|
|
if url_template:
|
|
for page_num in [1, 2, 3, 10]:
|
|
test_url = url_template.replace('{PAGE_NUMBER}', str(page_num))
|
|
print(f"Page {page_num}: {test_url}")
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(test_url_template_generation()) |