diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 874e5fb..8bb42a7 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -231,7 +231,7 @@ async def detect_pagination_service(decoded_url): // Try to find the last page number const pageNumbers = numberedLinks - .map(el => parseInt(el.innerText.trim())) + .map(el => parseInt(el.innerText.replace(/[^\d]/g, ''))) .filter(num => !isNaN(num)); if (pageNumbers.length > 0) { @@ -313,6 +313,7 @@ async def detect_pagination_service(decoded_url): next_page_url = None pagination_parameter = None url_template = None + step_size = None if pagination_data['hasPagination']: print("Pagination detected, attempting to click on a pagination element") @@ -479,6 +480,7 @@ async def detect_pagination_service(decoded_url): }}''', original_url) # Determine the pagination parameter and create URL template + print(original_parsed) if original_parsed['paramDiff']: param_name = original_parsed['paramDiff']['name'] pagination_parameter = { @@ -487,6 +489,17 @@ async def detect_pagination_service(decoded_url): 'value': original_parsed['paramDiff']['currentValue'] } + # --- STEP SIZE DETECTION FOR QUERY PARAM --- + orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0 + curr_val = original_parsed['paramDiff']['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None + # Create URL template for query parameter url_obj = await page.evaluate(f'''(url, paramName) => {{ const urlObj = new URL(url); @@ -504,6 +517,17 @@ async def detect_pagination_service(decoded_url): 'value': original_parsed['pathDiff']['currentValue'] } + # --- STEP SIZE DETECTION FOR PATH PARAM --- + orig_val = original_parsed['pathDiff']['originalValue'] + curr_val = original_parsed['pathDiff']['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None + # Create URL template for path parameter url_template = await page.evaluate(f'''(url, pathIndex) => {{ const urlObj = new URL(url); @@ -532,7 +556,8 @@ async def detect_pagination_service(decoded_url): "status": "success", "hasPagination": pagination_data['hasPagination'], "urlTemplate": url_template, - "lastPage": pagination_data['lastPageNumber'] + "lastPage": pagination_data['lastPageNumber'], + "stepSize": step_size } return result @@ -544,6 +569,7 @@ async def detect_pagination_service(decoded_url): "hasPagination": False, "urlTemplate": None, "lastPage": None, + "stepSize": None, "error": str(e) }