replace puppeteer with playwright
Build and Push Docker Images / build-and-push (push) Has been cancelled
Build and Push Docker Images / build-and-push (push) Has been cancelled
This commit is contained in:
@@ -10,7 +10,7 @@ async def visit_url_service(decoded_url):
|
||||
# Define the operation to perform with the browser
|
||||
async def visit_operation(page):
|
||||
try:
|
||||
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
||||
if not response:
|
||||
print(f"Warning: No response object returned for {decoded_url}")
|
||||
|
||||
@@ -36,7 +36,7 @@ async def extract_seo_service(decoded_url):
|
||||
# Define the operation to perform with the browser
|
||||
async def seo_operation(page):
|
||||
try:
|
||||
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
||||
|
||||
# Extract SEO information
|
||||
seo_data = await page.evaluate('''() => {
|
||||
@@ -154,7 +154,7 @@ async def detect_pagination_service(decoded_url):
|
||||
try:
|
||||
# Navigate to the URL
|
||||
try:
|
||||
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
|
||||
await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
||||
except Exception as e:
|
||||
print(f"Error navigating to URL: {e}")
|
||||
# Try to get the current URL even if navigation failed
|
||||
@@ -433,7 +433,7 @@ async def detect_pagination_service(decoded_url):
|
||||
if clicked:
|
||||
print("Successfully clicked on pagination element")
|
||||
# Wait for navigation to complete
|
||||
await page.waitForNavigation(waitUntil='networkidle2', timeout=10000)
|
||||
await page.wait_for_load_state('networkidle', timeout=10000)
|
||||
next_page_url = page.url
|
||||
else:
|
||||
print("No clickable pagination element found")
|
||||
@@ -447,8 +447,8 @@ async def detect_pagination_service(decoded_url):
|
||||
print('Searching for pagination parameter')
|
||||
# Parse both URLs
|
||||
try:
|
||||
original_parsed = await page.evaluate(f'''(originalUrl) => {{
|
||||
try {{
|
||||
original_parsed = await page.evaluate('''([originalUrl]) => {
|
||||
try {
|
||||
const original = new URL(originalUrl);
|
||||
const current = new URL(window.location.href);
|
||||
|
||||
@@ -458,99 +458,99 @@ async def detect_pagination_service(decoded_url):
|
||||
// Common pagination parameters to check
|
||||
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
|
||||
|
||||
for (const param of paginationParams) {{
|
||||
for (const param of paginationParams) {
|
||||
const originalValue = original.searchParams.get(param);
|
||||
const currentValue = current.searchParams.get(param);
|
||||
|
||||
if (originalValue !== currentValue && currentValue !== null) {{
|
||||
paramDiff = {{
|
||||
if (originalValue !== currentValue && currentValue !== null) {
|
||||
paramDiff = {
|
||||
name: param,
|
||||
originalValue: originalValue,
|
||||
currentValue: currentValue
|
||||
}};
|
||||
};
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2)
|
||||
const originalPath = original.pathname;
|
||||
const currentPath = current.pathname;
|
||||
|
||||
let pathDiff = null;
|
||||
if (originalPath !== currentPath) {{
|
||||
if (originalPath !== currentPath) {
|
||||
const originalSegments = originalPath.split('/').filter(s => s);
|
||||
const currentSegments = currentPath.split('/').filter(s => s);
|
||||
|
||||
// Case 1: Same number of segments - find the one that changed
|
||||
if (originalSegments.length === currentSegments.length) {{
|
||||
for (let i = 0; i < originalSegments.length; i++) {{
|
||||
if (originalSegments[i] !== currentSegments[i]) {{
|
||||
if (originalSegments.length === currentSegments.length) {
|
||||
for (let i = 0; i < originalSegments.length; i++) {
|
||||
if (originalSegments[i] !== currentSegments[i]) {
|
||||
// Check if the difference is numeric
|
||||
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
|
||||
pathDiff = {{
|
||||
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {
|
||||
pathDiff = {
|
||||
type: 'replace',
|
||||
index: i,
|
||||
originalValue: originalSegments[i],
|
||||
currentValue: currentSegments[i]
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Case 2: Current path has more segments - check for added pagination segments
|
||||
else if (currentSegments.length > originalSegments.length) {{
|
||||
else if (currentSegments.length > originalSegments.length) {
|
||||
// Look for patterns like /page/NUMBER or /p/NUMBER at the end
|
||||
const pagePattern = /^(page|p)\/(\d+)$/i;
|
||||
|
||||
// Check the last two segments of the current path
|
||||
if (currentSegments.length >= 2) {{
|
||||
if (currentSegments.length >= 2) {
|
||||
const lastTwoSegments = currentSegments.slice(-2).join('/');
|
||||
const match = lastTwoSegments.match(pagePattern);
|
||||
|
||||
if (match) {{
|
||||
pathDiff = {{
|
||||
if (match) {
|
||||
pathDiff = {
|
||||
type: 'append',
|
||||
pageSegment: match[1], // 'page' or 'p'
|
||||
pageNumber: match[2], // the actual number
|
||||
originalSegments: originalSegments,
|
||||
currentSegments: currentSegments
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// If no pattern match, check if the last segment is numeric
|
||||
if (!pathDiff && currentSegments.length > 0) {{
|
||||
if (!pathDiff && currentSegments.length > 0) {
|
||||
const lastSegment = currentSegments[currentSegments.length - 1];
|
||||
if (!isNaN(lastSegment)) {{
|
||||
pathDiff = {{
|
||||
if (!isNaN(lastSegment)) {
|
||||
pathDiff = {
|
||||
type: 'append',
|
||||
pageSegment: null,
|
||||
pageNumber: lastSegment,
|
||||
originalSegments: originalSegments,
|
||||
currentSegments: currentSegments
|
||||
}};
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {{
|
||||
return {
|
||||
paramDiff,
|
||||
pathDiff,
|
||||
originalUrl: originalUrl,
|
||||
currentUrl: window.location.href
|
||||
}};
|
||||
}} catch (error) {{
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Error during URL analysis:", error);
|
||||
return {{
|
||||
return {
|
||||
paramDiff: null,
|
||||
pathDiff: null,
|
||||
originalUrl: originalUrl,
|
||||
currentUrl: window.location.href,
|
||||
error: error.message
|
||||
}};
|
||||
}}
|
||||
}}''', original_url)
|
||||
};
|
||||
}
|
||||
}''', [original_url])
|
||||
except Exception as e:
|
||||
print(f"Error during URL analysis: {e}")
|
||||
original_parsed = {
|
||||
@@ -585,16 +585,16 @@ async def detect_pagination_service(decoded_url):
|
||||
|
||||
# Create URL template for query parameter
|
||||
try:
|
||||
url_obj = await page.evaluate(f'''(url, paramName) => {{
|
||||
try {{
|
||||
url_obj = await page.evaluate('''([url, paramName]) => {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
} catch (error) {
|
||||
console.error("Error creating URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, param_name)
|
||||
}
|
||||
}''', [original_url, param_name])
|
||||
|
||||
url_template = url_obj
|
||||
except Exception as e:
|
||||
@@ -626,18 +626,18 @@ async def detect_pagination_service(decoded_url):
|
||||
|
||||
# Create URL template for path parameter replacement
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, pathIndex) => {{
|
||||
try {{
|
||||
url_template = await page.evaluate('''([url, pathIndex]) => {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
const pathSegments = urlObj.pathname.split('/').filter(s => s);
|
||||
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
|
||||
pathSegments[pathIndex] = "{PAGE_NUMBER}";
|
||||
urlObj.pathname = '/' + pathSegments.join('/');
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
} catch (error) {
|
||||
console.error("Error creating path URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, path_index)
|
||||
}
|
||||
}''', [original_url, path_index])
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for path parameter: {e}")
|
||||
url_template = None
|
||||
@@ -662,30 +662,30 @@ async def detect_pagination_service(decoded_url):
|
||||
|
||||
# Create URL template for appended path parameter
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, pageSegment) => {{
|
||||
try {{
|
||||
url_template = await page.evaluate('''([url, pageSegment]) => {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
let newPath = urlObj.pathname;
|
||||
|
||||
// Remove trailing slash if present
|
||||
if (newPath.endsWith('/')) {{
|
||||
if (newPath.endsWith('/')) {
|
||||
newPath = newPath.slice(0, -1);
|
||||
}}
|
||||
}
|
||||
|
||||
// Append the pagination segment
|
||||
if (pageSegment) {{
|
||||
newPath += '/' + pageSegment + '/{{PAGE_NUMBER}}';
|
||||
}} else {{
|
||||
newPath += '/{{PAGE_NUMBER}}';
|
||||
}}
|
||||
if (pageSegment) {
|
||||
newPath += '/' + pageSegment + '/{PAGE_NUMBER}';
|
||||
} else {
|
||||
newPath += '/{PAGE_NUMBER}';
|
||||
}
|
||||
|
||||
urlObj.pathname = newPath;
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
} catch (error) {
|
||||
console.error("Error creating appended path URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, path_diff.get('pageSegment'))
|
||||
}
|
||||
}''', [original_url, path_diff.get('pageSegment')])
|
||||
except Exception as e:
|
||||
print(f"Error creating URL template for appended path parameter: {e}")
|
||||
url_template = None
|
||||
@@ -699,16 +699,16 @@ async def detect_pagination_service(decoded_url):
|
||||
if not url_template and pagination_data['detectedParameter']:
|
||||
param_name = pagination_data['detectedParameter']['name']
|
||||
try:
|
||||
url_template = await page.evaluate(f'''(url, paramName) => {{
|
||||
try {{
|
||||
url_template = await page.evaluate('''([url, paramName]) => {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
|
||||
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
|
||||
return urlObj.toString();
|
||||
}} catch (error) {{
|
||||
} catch (error) {
|
||||
console.error("Error creating inferred URL template:", error);
|
||||
return null;
|
||||
}}
|
||||
}}''', original_url, param_name)
|
||||
}
|
||||
}''', [original_url, param_name])
|
||||
except Exception as e:
|
||||
print(f"Error creating inferred URL template: {e}")
|
||||
url_template = None
|
||||
@@ -716,7 +716,7 @@ async def detect_pagination_service(decoded_url):
|
||||
# If still no template and we have pagination elements, try to infer from the current URL structure
|
||||
if not url_template and pagination_data['hasPagination']:
|
||||
try:
|
||||
url_template = await page.evaluate('''(originalUrl) => {
|
||||
url_template = await page.evaluate('''([originalUrl]) => {
|
||||
try {
|
||||
const urlObj = new URL(originalUrl);
|
||||
let path = urlObj.pathname;
|
||||
@@ -730,10 +730,10 @@ async def detect_pagination_service(decoded_url):
|
||||
const pagePattern = /\/(page|p)\/\d+$/i;
|
||||
if (pagePattern.test(path)) {
|
||||
// Replace the existing page number with placeholder
|
||||
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{{PAGE_NUMBER}}');
|
||||
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{PAGE_NUMBER}');
|
||||
} else {
|
||||
// Add pagination pattern
|
||||
path += '/page/{{PAGE_NUMBER}}';
|
||||
path += '/page/{PAGE_NUMBER}';
|
||||
}
|
||||
|
||||
urlObj.pathname = path;
|
||||
@@ -742,7 +742,7 @@ async def detect_pagination_service(decoded_url):
|
||||
console.error("Error creating fallback URL template:", error);
|
||||
return null;
|
||||
}
|
||||
}''', original_url)
|
||||
}''', [original_url])
|
||||
except Exception as e:
|
||||
print(f"Error creating fallback URL template: {e}")
|
||||
url_template = None
|
||||
|
||||
Reference in New Issue
Block a user