Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

replace puppeteer with playwright
Build and Push Docker Images / build-and-push (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Bram
2025-07-04 13:15:59 +02:00
parent 8c4a27c87e
commit 129c6a74e8
10 changed files with 1030 additions and 463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+79 -79
View File
@@ -10,7 +10,7 @@ async def visit_url_service(decoded_url):
# Define the operation to perform with the browser
async def visit_operation(page):
try:
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
if not response:
print(f"Warning: No response object returned for {decoded_url}")
@@ -36,7 +36,7 @@ async def extract_seo_service(decoded_url):
# Define the operation to perform with the browser
async def seo_operation(page):
try:
response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
# Extract SEO information
seo_data = await page.evaluate('''() => {
@@ -154,7 +154,7 @@ async def detect_pagination_service(decoded_url):
try:
# Navigate to the URL
try:
await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
except Exception as e:
print(f"Error navigating to URL: {e}")
# Try to get the current URL even if navigation failed
@@ -433,7 +433,7 @@ async def detect_pagination_service(decoded_url):
if clicked:
print("Successfully clicked on pagination element")
# Wait for navigation to complete
await page.waitForNavigation(waitUntil='networkidle2', timeout=10000)
await page.wait_for_load_state('networkidle', timeout=10000)
next_page_url = page.url
else:
print("No clickable pagination element found")
@@ -447,8 +447,8 @@ async def detect_pagination_service(decoded_url):
print('Searching for pagination parameter')
# Parse both URLs
try:
original_parsed = await page.evaluate(f'''(originalUrl) => {{
try {{
original_parsed = await page.evaluate('''([originalUrl]) => {
try {
const original = new URL(originalUrl);
const current = new URL(window.location.href);
@@ -458,99 +458,99 @@ async def detect_pagination_service(decoded_url):
// Common pagination parameters to check
const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
for (const param of paginationParams) {{
for (const param of paginationParams) {
const originalValue = original.searchParams.get(param);
const currentValue = current.searchParams.get(param);
if (originalValue !== currentValue && currentValue !== null) {{
paramDiff = {{
if (originalValue !== currentValue && currentValue !== null) {
paramDiff = {
name: param,
originalValue: originalValue,
currentValue: currentValue
}};
};
break;
}}
}}
}
}
// Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2)
const originalPath = original.pathname;
const currentPath = current.pathname;
let pathDiff = null;
if (originalPath !== currentPath) {{
if (originalPath !== currentPath) {
const originalSegments = originalPath.split('/').filter(s => s);
const currentSegments = currentPath.split('/').filter(s => s);
// Case 1: Same number of segments - find the one that changed
if (originalSegments.length === currentSegments.length) {{
for (let i = 0; i < originalSegments.length; i++) {{
if (originalSegments[i] !== currentSegments[i]) {{
if (originalSegments.length === currentSegments.length) {
for (let i = 0; i < originalSegments.length; i++) {
if (originalSegments[i] !== currentSegments[i]) {
// Check if the difference is numeric
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
pathDiff = {{
if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {
pathDiff = {
type: 'replace',
index: i,
originalValue: originalSegments[i],
currentValue: currentSegments[i]
}};
}}
}}
}}
}}
};
}
}
}
}
// Case 2: Current path has more segments - check for added pagination segments
else if (currentSegments.length > originalSegments.length) {{
else if (currentSegments.length > originalSegments.length) {
// Look for patterns like /page/NUMBER or /p/NUMBER at the end
const pagePattern = /^(page|p)\/(\d+)$/i;
// Check the last two segments of the current path
if (currentSegments.length >= 2) {{
if (currentSegments.length >= 2) {
const lastTwoSegments = currentSegments.slice(-2).join('/');
const match = lastTwoSegments.match(pagePattern);
if (match) {{
pathDiff = {{
if (match) {
pathDiff = {
type: 'append',
pageSegment: match[1], // 'page' or 'p'
pageNumber: match[2], // the actual number
originalSegments: originalSegments,
currentSegments: currentSegments
}};
}}
}}
};
}
}
// If no pattern match, check if the last segment is numeric
if (!pathDiff && currentSegments.length > 0) {{
if (!pathDiff && currentSegments.length > 0) {
const lastSegment = currentSegments[currentSegments.length - 1];
if (!isNaN(lastSegment)) {{
pathDiff = {{
if (!isNaN(lastSegment)) {
pathDiff = {
type: 'append',
pageSegment: null,
pageNumber: lastSegment,
originalSegments: originalSegments,
currentSegments: currentSegments
}};
}}
}}
}}
}}
};
}
}
}
}
return {{
return {
paramDiff,
pathDiff,
originalUrl: originalUrl,
currentUrl: window.location.href
}};
}} catch (error) {{
};
} catch (error) {
console.error("Error during URL analysis:", error);
return {{
return {
paramDiff: null,
pathDiff: null,
originalUrl: originalUrl,
currentUrl: window.location.href,
error: error.message
}};
}}
}}''', original_url)
};
}
}''', [original_url])
except Exception as e:
print(f"Error during URL analysis: {e}")
original_parsed = {
@@ -585,16 +585,16 @@ async def detect_pagination_service(decoded_url):
# Create URL template for query parameter
try:
url_obj = await page.evaluate(f'''(url, paramName) => {{
try {{
url_obj = await page.evaluate('''([url, paramName]) => {
try {
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
return urlObj.toString();
}} catch (error) {{
} catch (error) {
console.error("Error creating URL template:", error);
return null;
}}
}}''', original_url, param_name)
}
}''', [original_url, param_name])
url_template = url_obj
except Exception as e:
@@ -626,18 +626,18 @@ async def detect_pagination_service(decoded_url):
# Create URL template for path parameter replacement
try:
url_template = await page.evaluate(f'''(url, pathIndex) => {{
try {{
url_template = await page.evaluate('''([url, pathIndex]) => {
try {
const urlObj = new URL(url);
const pathSegments = urlObj.pathname.split('/').filter(s => s);
pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
pathSegments[pathIndex] = "{PAGE_NUMBER}";
urlObj.pathname = '/' + pathSegments.join('/');
return urlObj.toString();
}} catch (error) {{
} catch (error) {
console.error("Error creating path URL template:", error);
return null;
}}
}}''', original_url, path_index)
}
}''', [original_url, path_index])
except Exception as e:
print(f"Error creating URL template for path parameter: {e}")
url_template = None
@@ -662,30 +662,30 @@ async def detect_pagination_service(decoded_url):
# Create URL template for appended path parameter
try:
url_template = await page.evaluate(f'''(url, pageSegment) => {{
try {{
url_template = await page.evaluate('''([url, pageSegment]) => {
try {
const urlObj = new URL(url);
let newPath = urlObj.pathname;
// Remove trailing slash if present
if (newPath.endsWith('/')) {{
if (newPath.endsWith('/')) {
newPath = newPath.slice(0, -1);
}}
}
// Append the pagination segment
if (pageSegment) {{
newPath += '/' + pageSegment + '/{{PAGE_NUMBER}}';
}} else {{
newPath += '/{{PAGE_NUMBER}}';
}}
if (pageSegment) {
newPath += '/' + pageSegment + '/{PAGE_NUMBER}';
} else {
newPath += '/{PAGE_NUMBER}';
}
urlObj.pathname = newPath;
return urlObj.toString();
}} catch (error) {{
} catch (error) {
console.error("Error creating appended path URL template:", error);
return null;
}}
}}''', original_url, path_diff.get('pageSegment'))
}
}''', [original_url, path_diff.get('pageSegment')])
except Exception as e:
print(f"Error creating URL template for appended path parameter: {e}")
url_template = None
@@ -699,16 +699,16 @@ async def detect_pagination_service(decoded_url):
if not url_template and pagination_data['detectedParameter']:
param_name = pagination_data['detectedParameter']['name']
try:
url_template = await page.evaluate(f'''(url, paramName) => {{
try {{
url_template = await page.evaluate('''([url, paramName]) => {
try {
const urlObj = new URL(url);
urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
return urlObj.toString();
}} catch (error) {{
} catch (error) {
console.error("Error creating inferred URL template:", error);
return null;
}}
}}''', original_url, param_name)
}
}''', [original_url, param_name])
except Exception as e:
print(f"Error creating inferred URL template: {e}")
url_template = None
@@ -716,7 +716,7 @@ async def detect_pagination_service(decoded_url):
# If still no template and we have pagination elements, try to infer from the current URL structure
if not url_template and pagination_data['hasPagination']:
try:
url_template = await page.evaluate('''(originalUrl) => {
url_template = await page.evaluate('''([originalUrl]) => {
try {
const urlObj = new URL(originalUrl);
let path = urlObj.pathname;
@@ -730,10 +730,10 @@ async def detect_pagination_service(decoded_url):
const pagePattern = /\/(page|p)\/\d+$/i;
if (pagePattern.test(path)) {
// Replace the existing page number with placeholder
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{{PAGE_NUMBER}}');
path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{PAGE_NUMBER}');
} else {
// Add pagination pattern
path += '/page/{{PAGE_NUMBER}}';
path += '/page/{PAGE_NUMBER}';
}
urlObj.pathname = path;
@@ -742,7 +742,7 @@ async def detect_pagination_service(decoded_url):
console.error("Error creating fallback URL template:", error);
return null;
}
}''', original_url)
}''', [original_url])
except Exception as e:
print(f"Error creating fallback URL template: {e}")
url_template = None
Dockers/puppeteer-api/app/utils/browser_utils.py
+22 -12
View File
@@ -1,31 +1,34 @@
from pyppeteer import launch
from playwright.async_api import async_playwright
from app.config import CUSTOM_USER_AGENT
import asyncio
async def wait_for_network_idle(page):
"""Wait until no network requests are in flight"""
await page.waitForNetworkIdle(idleTime=500, timeout=30000)
await page.wait_for_load_state('networkidle')
async def safe_browser_operation(url, operation_func):
"""Safely perform browser operations with proper cleanup"""
browser = None
context = None
page = None
try:
browser = await launch(
# Get or create playwright instance
playwright = await async_playwright().start()
browser = await playwright.chromium.launch(
headless=True,
executablePath='/usr/bin/google-chrome',
args=['--no-sandbox', '--disable-setuid-sandbox'],
handleSIGINT=False,
handleSIGTERM=False,
handleSIGHUP=False
)
# Create new page with timeout
page = await browser.newPage()
page.setDefaultNavigationTimeout(30000)
# Create context and page
context = await browser.new_context(
user_agent=CUSTOM_USER_AGENT,
viewport={'width': 1920, 'height': 1080},
ignore_https_errors=True,
)
# Set custom user agent
await page.setUserAgent(CUSTOM_USER_AGENT)
page = await context.new_page()
page.set_default_timeout(30000)
# Call the operation function that uses the page
result = await operation_func(page)
@@ -48,6 +51,13 @@ async def safe_browser_operation(url, operation_func):
except Exception as e:
print(f"Error closing page: {e}")
# Ensure context is closed properly
if context:
try:
await context.close()
except Exception as e:
print(f"Error closing context: {e}")
# Ensure browser is closed properly
if browser:
try: