Bram/projects
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

wait for page
Build and Push Docker Images / build-and-push (push) Successful in 21s
Details

Browse Source
This commit is contained in:
Bram
2025-07-07 15:03:05 +02:00
parent 8bad7b41ec
commit 34d22118ca
1 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockers/puppeteer-api/app/services/browser.py
+9 -3
View File
@@ -166,6 +166,7 @@ async def detect_pagination_service(decoded_url):
original_url = page.url original_url = page.url
# Check for common pagination indicators # Check for common pagination indicators
await asyncio.sleep(2)
try: try:
pagination_data = await page.evaluate('''() => { pagination_data = await page.evaluate('''() => {
const data = { const data = {
@@ -192,7 +193,6 @@ async def detect_pagination_service(decoded_url):
el.closest('button, [role="button"]')); el.closest('button, [role="button"]'));
}); });
console.log(numberedLinks);
if(numberedLinks.length <= 1) { if(numberedLinks.length <= 1) {
return data; return data;
} }
@@ -237,7 +237,7 @@ async def detect_pagination_service(decoded_url):
const pageNumbers = numberedLinks const pageNumbers = numberedLinks
.map(el => parseInt(el.innerText.replace(/[^\d]/g, ''))) .map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
.filter(num => !isNaN(num)); .filter(num => !isNaN(num));
console.log(pageNumbers);
if (pageNumbers.length > 0) { if (pageNumbers.length > 0) {
data.lastPageNumber = Math.max(...pageNumbers); data.lastPageNumber = Math.max(...pageNumbers);
} }
@@ -261,6 +261,7 @@ async def detect_pagination_service(decoded_url):
if (url.searchParams.has(param)) { if (url.searchParams.has(param)) {
const value = parseInt(url.searchParams.get(param)); const value = parseInt(url.searchParams.get(param));
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
console.log("Last page number: " + value);
data.lastPageNumber = value; data.lastPageNumber = value;
} }
} }
@@ -271,6 +272,7 @@ async def detect_pagination_service(decoded_url):
if (pathMatch && pathMatch[1]) { if (pathMatch && pathMatch[1]) {
const value = parseInt(pathMatch[1]); const value = parseInt(pathMatch[1]);
if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
console.log("Last page number: " + value);
data.lastPageNumber = value; data.lastPageNumber = value;
} }
} }
@@ -323,6 +325,7 @@ async def detect_pagination_service(decoded_url):
'lastPageNumber': None 'lastPageNumber': None
} }
print(f"Pagination data: {pagination_data}")
# If pagination is detected, try to navigate to the next page by clicking # If pagination is detected, try to navigate to the next page by clicking
next_page_url = None next_page_url = None
pagination_parameter = None pagination_parameter = None
@@ -440,6 +443,7 @@ async def detect_pagination_service(decoded_url):
print("Successfully clicked on pagination element") print("Successfully clicked on pagination element")
# Wait for navigation to complete # Wait for navigation to complete
await page.wait_for_load_state('networkidle', timeout=10000) await page.wait_for_load_state('networkidle', timeout=10000)
await asyncio.sleep(2)
next_page_url = page.url next_page_url = page.url
else: else:
print("No clickable pagination element found") print("No clickable pagination element found")
@@ -448,8 +452,10 @@ async def detect_pagination_service(decoded_url):
print(f"Error clicking on pagination element: {e}") print(f"Error clicking on pagination element: {e}")
# Continue with the process even if clicking fails # Continue with the process even if clicking fails
print(f"Next page URL: {next_page_url}")
# If we successfully navigated to the next page, analyze the URL difference # If we successfully navigated to the next page, analyze the URL difference
if next_page_url: if next_page_url and next_page_url != original_url_before_navigation:
print('Searching for pagination parameter') print('Searching for pagination parameter')
# Parse both URLs using Python instead of JavaScript to avoid execution context issues # Parse both URLs using Python instead of JavaScript to avoid execution context issues
try: try: