Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix original parsed err
Build and Push Docker Images / build-and-push (push) Successful in 28s
Details

Browse Source
This commit is contained in:
Bram
2025-06-21 20:37:26 +02:00
parent 6051e41cfa
commit bde19374c0
1 changed files with 70 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+70 -68
View File
@@ -325,6 +325,7 @@ async def detect_pagination_service(decoded_url):
pagination_parameter = None pagination_parameter = None
url_template = None url_template = None
step_size = None step_size = None
original_parsed = None # Initialize the variable
if pagination_data['hasPagination']: if pagination_data['hasPagination']:
print("Pagination detected, attempting to click on a pagination element") print("Pagination detected, attempting to click on a pagination element")
@@ -518,80 +519,81 @@ async def detect_pagination_service(decoded_url):
} }
# Determine the pagination parameter and create URL template # Determine the pagination parameter and create URL template
print(original_parsed) if original_parsed:
if original_parsed['paramDiff']: print(original_parsed)
param_name = original_parsed['paramDiff']['name'] if original_parsed.get('paramDiff'):
pagination_parameter = { param_name = original_parsed['paramDiff']['name']
'type': 'query', pagination_parameter = {
'name': param_name, 'type': 'query',
'value': original_parsed['paramDiff']['currentValue'] 'name': param_name,
} 'value': original_parsed['paramDiff']['currentValue']
}
# --- STEP SIZE DETECTION FOR QUERY PARAM --- # --- STEP SIZE DETECTION FOR QUERY PARAM ---
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0 orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
curr_val = original_parsed['paramDiff']['currentValue'] curr_val = original_parsed['paramDiff']['currentValue']
try: try:
if orig_val is not None and curr_val is not None: if orig_val is not None and curr_val is not None:
orig_num = int(orig_val) orig_num = int(orig_val)
curr_num = int(curr_val) curr_num = int(curr_val)
step_size = abs(curr_num - orig_num) step_size = abs(curr_num - orig_num)
except Exception: except Exception:
step_size = None step_size = None
# Create URL template for query parameter # Create URL template for query parameter
try: try:
url_obj = await page.evaluate(f'''(url, paramName) => {{ url_obj = await page.evaluate(f'''(url, paramName) => {{
try {{ try {{
const urlObj = new URL(url); const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}"); urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
return urlObj.toString(); return urlObj.toString();
}} catch (error) {{ }} catch (error) {{
console.error("Error creating URL template:", error); console.error("Error creating URL template:", error);
return null; return null;
}} }}
}}''', original_url, param_name) }}''', original_url, param_name)
url_template = url_obj url_template = url_obj
except Exception as e: except Exception as e:
print(f"Error creating URL template for query parameter: {e}") print(f"Error creating URL template for query parameter: {e}")
url_template = None url_template = None
elif original_parsed['pathDiff']: elif original_parsed.get('pathDiff'):
path_index = original_parsed['pathDiff']['index'] path_index = original_parsed['pathDiff']['index']
pagination_parameter = { pagination_parameter = {
'type': 'path', 'type': 'path',
'index': path_index, 'index': path_index,
'value': original_parsed['pathDiff']['currentValue'] 'value': original_parsed['pathDiff']['currentValue']
} }
# --- STEP SIZE DETECTION FOR PATH PARAM --- # --- STEP SIZE DETECTION FOR PATH PARAM ---
orig_val = original_parsed['pathDiff']['originalValue'] orig_val = original_parsed['pathDiff']['originalValue']
curr_val = original_parsed['pathDiff']['currentValue'] curr_val = original_parsed['pathDiff']['currentValue']
try: try:
if orig_val is not None and curr_val is not None: if orig_val is not None and curr_val is not None:
orig_num = int(orig_val) orig_num = int(orig_val)
curr_num = int(curr_val) curr_num = int(curr_val)
step_size = abs(curr_num - orig_num) step_size = abs(curr_num - orig_num)
except Exception: except Exception:
step_size = None step_size = None
# Create URL template for path parameter # Create URL template for path parameter
try: try:
url_template = await page.evaluate(f'''(url, pathIndex) => {{ url_template = await page.evaluate(f'''(url, pathIndex) => {{
try {{ try {{
const urlObj = new URL(url); const urlObj = new URL(url);
const pathSegments = urlObj.pathname.split('/').filter(s => s); const pathSegments = urlObj.pathname.split('/').filter(s => s);
pathSegments[pathIndex] = "{{PAGE_NUMBER}}"; pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
urlObj.pathname = '/' + pathSegments.join('/'); urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString(); return urlObj.toString();
}} catch (error) {{ }} catch (error) {{
console.error("Error creating path URL template:", error); console.error("Error creating path URL template:", error);
return null; return null;
}} }}
}}''', original_url, path_index) }}''', original_url, path_index)
except Exception as e: except Exception as e:
print(f"Error creating URL template for path parameter: {e}") print(f"Error creating URL template for path parameter: {e}")
url_template = None url_template = None
if url_template: if url_template:
# Decode URL-encoded characters in the template # Decode URL-encoded characters in the template