offset detection

Dockers/puppeteer-api/app/services/browser.py

@@ -231,7 +231,7 @@ async def detect_pagination_service(decoded_url):
 
                     // Try to find the last page number
                     const pageNumbers = numberedLinks
-                        .map(el => parseInt(el.innerText.trim()))
+                        .map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
                         .filter(num => !isNaN(num));
 
                     if (pageNumbers.length > 0) {
@@ -313,6 +313,7 @@ async def detect_pagination_service(decoded_url):
             next_page_url = None
             pagination_parameter = None
             url_template = None
+            step_size = None
 
             if pagination_data['hasPagination']:
                 print("Pagination detected, attempting to click on a pagination element")
@@ -479,6 +480,7 @@ async def detect_pagination_service(decoded_url):
                 }}''', original_url)
 
                 # Determine the pagination parameter and create URL template
+                print(original_parsed)
                 if original_parsed['paramDiff']:
                     param_name = original_parsed['paramDiff']['name']
                     pagination_parameter = {
@@ -487,6 +489,17 @@ async def detect_pagination_service(decoded_url):
                         'value': original_parsed['paramDiff']['currentValue']
                     }
 
+                    # --- STEP SIZE DETECTION FOR QUERY PARAM ---
+                    orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
+                    curr_val = original_parsed['paramDiff']['currentValue']
+                    try:
+                        if orig_val is not None and curr_val is not None:
+                            orig_num = int(orig_val)
+                            curr_num = int(curr_val)
+                            step_size = abs(curr_num - orig_num)
+                    except Exception:
+                        step_size = None
+
                     # Create URL template for query parameter
                     url_obj = await page.evaluate(f'''(url, paramName) => {{
                         const urlObj = new URL(url);
@@ -504,6 +517,17 @@ async def detect_pagination_service(decoded_url):
                         'value': original_parsed['pathDiff']['currentValue']
                     }
 
+                    # --- STEP SIZE DETECTION FOR PATH PARAM ---
+                    orig_val = original_parsed['pathDiff']['originalValue']
+                    curr_val = original_parsed['pathDiff']['currentValue']
+                    try:
+                        if orig_val is not None and curr_val is not None:
+                            orig_num = int(orig_val)
+                            curr_num = int(curr_val)
+                            step_size = abs(curr_num - orig_num)
+                    except Exception:
+                        step_size = None
+
                     # Create URL template for path parameter
                     url_template = await page.evaluate(f'''(url, pathIndex) => {{
                         const urlObj = new URL(url);
@@ -532,7 +556,8 @@ async def detect_pagination_service(decoded_url):
                 "status": "success",
                 "hasPagination": pagination_data['hasPagination'],
                 "urlTemplate": url_template,
-                "lastPage": pagination_data['lastPageNumber']
+                "lastPage": pagination_data['lastPageNumber'],
+                "stepSize": step_size
             }
 
             return result
@@ -544,6 +569,7 @@ async def detect_pagination_service(decoded_url):
                 "hasPagination": False,
                 "urlTemplate": None,
                 "lastPage": None,
+                "stepSize": None,
                 "error": str(e)
             }