From bde19374c05752db1295c610c720a9725560e4cd Mon Sep 17 00:00:00 2001 From: Bram Date: Sat, 21 Jun 2025 20:37:26 +0200 Subject: [PATCH] fix original parsed err --- Dockers/puppeteer-api/app/services/browser.py | 138 +++++++++--------- 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 106fc63..ff59349 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -325,6 +325,7 @@ async def detect_pagination_service(decoded_url): pagination_parameter = None url_template = None step_size = None + original_parsed = None # Initialize the variable if pagination_data['hasPagination']: print("Pagination detected, attempting to click on a pagination element") @@ -518,80 +519,81 @@ async def detect_pagination_service(decoded_url): } # Determine the pagination parameter and create URL template - print(original_parsed) - if original_parsed['paramDiff']: - param_name = original_parsed['paramDiff']['name'] - pagination_parameter = { - 'type': 'query', - 'name': param_name, - 'value': original_parsed['paramDiff']['currentValue'] - } + if original_parsed: + print(original_parsed) + if original_parsed.get('paramDiff'): + param_name = original_parsed['paramDiff']['name'] + pagination_parameter = { + 'type': 'query', + 'name': param_name, + 'value': original_parsed['paramDiff']['currentValue'] + } - # --- STEP SIZE DETECTION FOR QUERY PARAM --- - orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0 - curr_val = original_parsed['paramDiff']['currentValue'] - try: - if orig_val is not None and curr_val is not None: - orig_num = int(orig_val) - curr_num = int(curr_val) - step_size = abs(curr_num - orig_num) - except Exception: - step_size = None + # --- STEP SIZE DETECTION FOR QUERY PARAM --- + orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0 + curr_val = original_parsed['paramDiff']['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None - # Create URL template for query parameter - try: - url_obj = await page.evaluate(f'''(url, paramName) => {{ - try {{ - const urlObj = new URL(url); - urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); - return urlObj.toString(); - }} catch (error) {{ - console.error("Error creating URL template:", error); - return null; - }} - }}''', original_url, param_name) + # Create URL template for query parameter + try: + url_obj = await page.evaluate(f'''(url, paramName) => {{ + try {{ + const urlObj = new URL(url); + urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating URL template:", error); + return null; + }} + }}''', original_url, param_name) - url_template = url_obj - except Exception as e: - print(f"Error creating URL template for query parameter: {e}") - url_template = None + url_template = url_obj + except Exception as e: + print(f"Error creating URL template for query parameter: {e}") + url_template = None - elif original_parsed['pathDiff']: - path_index = original_parsed['pathDiff']['index'] - pagination_parameter = { - 'type': 'path', - 'index': path_index, - 'value': original_parsed['pathDiff']['currentValue'] - } + elif original_parsed.get('pathDiff'): + path_index = original_parsed['pathDiff']['index'] + pagination_parameter = { + 'type': 'path', + 'index': path_index, + 'value': original_parsed['pathDiff']['currentValue'] + } - # --- STEP SIZE DETECTION FOR PATH PARAM --- - orig_val = original_parsed['pathDiff']['originalValue'] - curr_val = original_parsed['pathDiff']['currentValue'] - try: - if orig_val is not None and curr_val is not None: - orig_num = int(orig_val) - curr_num = int(curr_val) - step_size = abs(curr_num - orig_num) - except Exception: - step_size = None + # --- STEP SIZE DETECTION FOR PATH PARAM --- + orig_val = original_parsed['pathDiff']['originalValue'] + curr_val = original_parsed['pathDiff']['currentValue'] + try: + if orig_val is not None and curr_val is not None: + orig_num = int(orig_val) + curr_num = int(curr_val) + step_size = abs(curr_num - orig_num) + except Exception: + step_size = None - # Create URL template for path parameter - try: - url_template = await page.evaluate(f'''(url, pathIndex) => {{ - try {{ - const urlObj = new URL(url); - const pathSegments = urlObj.pathname.split('/').filter(s => s); - pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; - urlObj.pathname = '/' + pathSegments.join('/'); - return urlObj.toString(); - }} catch (error) {{ - console.error("Error creating path URL template:", error); - return null; - }} - }}''', original_url, path_index) - except Exception as e: - print(f"Error creating URL template for path parameter: {e}") - url_template = None + # Create URL template for path parameter + try: + url_template = await page.evaluate(f'''(url, pathIndex) => {{ + try {{ + const urlObj = new URL(url); + const pathSegments = urlObj.pathname.split('/').filter(s => s); + pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; + urlObj.pathname = '/' + pathSegments.join('/'); + return urlObj.toString(); + }} catch (error) {{ + console.error("Error creating path URL template:", error); + return null; + }} + }}''', original_url, path_index) + except Exception as e: + print(f"Error creating URL template for path parameter: {e}") + url_template = None if url_template: # Decode URL-encoded characters in the template