Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix original parsed err
Build and Push Docker Images / build-and-push (push) Successful in 28s
Details

Browse Source
This commit is contained in:
Bram
2025-06-21 20:37:26 +02:00
parent 6051e41cfa
commit bde19374c0
1 changed files with 70 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+70 -68
View File
@@ -325,6 +325,7 @@ async def detect_pagination_service(decoded_url):
pagination_parameter = None
url_template = None
step_size = None
original_parsed = None # Initialize the variable
if pagination_data['hasPagination']:
print("Pagination detected, attempting to click on a pagination element")
@@ -518,80 +519,81 @@ async def detect_pagination_service(decoded_url):
}
# Determine the pagination parameter and create URL template
print(original_parsed)
if original_parsed['paramDiff']:
param_name = original_parsed['paramDiff']['name']
pagination_parameter = {
'type': 'query',
'name': param_name,
'value': original_parsed['paramDiff']['currentValue']
}
if original_parsed:
print(original_parsed)
if original_parsed.get('paramDiff'):
param_name = original_parsed['paramDiff']['name']
pagination_parameter = {
'type': 'query',
'name': param_name,
'value': original_parsed['paramDiff']['currentValue']
}
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
curr_val = original_parsed['paramDiff']['currentValue']
try:
if orig_val is not None and curr_val is not None:
orig_num = int(orig_val)
curr_num = int(curr_val)
step_size = abs(curr_num - orig_num)
except Exception:
step_size = None
# --- STEP SIZE DETECTION FOR QUERY PARAM ---
orig_val = original_parsed['paramDiff']['originalValue'] if original_parsed['paramDiff']['originalValue'] is not None else 0
curr_val = original_parsed['paramDiff']['currentValue']
try:
if orig_val is not None and curr_val is not None:
orig_num = int(orig_val)
curr_num = int(curr_val)
step_size = abs(curr_num - orig_num)
except Exception:
step_size = None
# Create URL template for query parameter
try:
url_obj = await page.evaluate(f'''(url, paramName) => {{
try {{
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
return urlObj.toString();
}} catch (error) {{
console.error("Error creating URL template:", error);
return null;
}}
}}''', original_url, param_name)
# Create URL template for query parameter
try:
url_obj = await page.evaluate(f'''(url, paramName) => {{
try {{
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
return urlObj.toString();
}} catch (error) {{
console.error("Error creating URL template:", error);
return null;
}}
}}''', original_url, param_name)
url_template = url_obj
except Exception as e:
print(f"Error creating URL template for query parameter: {e}")
url_template = None
url_template = url_obj
except Exception as e:
print(f"Error creating URL template for query parameter: {e}")
url_template = None
elif original_parsed['pathDiff']:
path_index = original_parsed['pathDiff']['index']
pagination_parameter = {
'type': 'path',
'index': path_index,
'value': original_parsed['pathDiff']['currentValue']
}
elif original_parsed.get('pathDiff'):
path_index = original_parsed['pathDiff']['index']
pagination_parameter = {
'type': 'path',
'index': path_index,
'value': original_parsed['pathDiff']['currentValue']
}
# --- STEP SIZE DETECTION FOR PATH PARAM ---
orig_val = original_parsed['pathDiff']['originalValue']
curr_val = original_parsed['pathDiff']['currentValue']
try:
if orig_val is not None and curr_val is not None:
orig_num = int(orig_val)
curr_num = int(curr_val)
step_size = abs(curr_num - orig_num)
except Exception:
step_size = None
# --- STEP SIZE DETECTION FOR PATH PARAM ---
orig_val = original_parsed['pathDiff']['originalValue']
curr_val = original_parsed['pathDiff']['currentValue']
try:
if orig_val is not None and curr_val is not None:
orig_num = int(orig_val)
curr_num = int(curr_val)
step_size = abs(curr_num - orig_num)
except Exception:
step_size = None
# Create URL template for path parameter
try:
url_template = await page.evaluate(f'''(url, pathIndex) => {{
try {{
const urlObj = new URL(url);
const pathSegments = urlObj.pathname.split('/').filter(s => s);
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString();
}} catch (error) {{
console.error("Error creating path URL template:", error);
return null;
}}
}}''', original_url, path_index)
except Exception as e:
print(f"Error creating URL template for path parameter: {e}")
url_template = None
# Create URL template for path parameter
try:
url_template = await page.evaluate(f'''(url, pathIndex) => {{
try {{
const urlObj = new URL(url);
const pathSegments = urlObj.pathname.split('/').filter(s => s);
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString();
}} catch (error) {{
console.error("Error creating path URL template:", error);
return null;
}}
}}''', original_url, path_index)
except Exception as e:
print(f"Error creating URL template for path parameter: {e}")
url_template = None
if url_template:
# Decode URL-encoded characters in the template