diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 81c20c7..16073d6 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -166,6 +166,7 @@ async def detect_pagination_service(decoded_url): original_url = page.url # Check for common pagination indicators + await asyncio.sleep(2) try: pagination_data = await page.evaluate('''() => { const data = { @@ -192,7 +193,6 @@ async def detect_pagination_service(decoded_url): el.closest('button, [role="button"]')); }); - console.log(numberedLinks); if(numberedLinks.length <= 1) { return data; } @@ -237,7 +237,7 @@ async def detect_pagination_service(decoded_url): const pageNumbers = numberedLinks .map(el => parseInt(el.innerText.replace(/[^\d]/g, ''))) .filter(num => !isNaN(num)); - + console.log(pageNumbers); if (pageNumbers.length > 0) { data.lastPageNumber = Math.max(...pageNumbers); } @@ -261,6 +261,7 @@ async def detect_pagination_service(decoded_url): if (url.searchParams.has(param)) { const value = parseInt(url.searchParams.get(param)); if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { + console.log("Last page number: " + value); data.lastPageNumber = value; } } @@ -271,6 +272,7 @@ async def detect_pagination_service(decoded_url): if (pathMatch && pathMatch[1]) { const value = parseInt(pathMatch[1]); if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) { + console.log("Last page number: " + value); data.lastPageNumber = value; } } @@ -323,6 +325,7 @@ async def detect_pagination_service(decoded_url): 'lastPageNumber': None } + print(f"Pagination data: {pagination_data}") # If pagination is detected, try to navigate to the next page by clicking next_page_url = None pagination_parameter = None @@ -440,6 +443,7 @@ async def detect_pagination_service(decoded_url): print("Successfully clicked on pagination element") # Wait for navigation to complete await page.wait_for_load_state('networkidle', timeout=10000) + await asyncio.sleep(2) next_page_url = page.url else: print("No clickable pagination element found") @@ -448,8 +452,10 @@ async def detect_pagination_service(decoded_url): print(f"Error clicking on pagination element: {e}") # Continue with the process even if clicking fails + print(f"Next page URL: {next_page_url}") + # If we successfully navigated to the next page, analyze the URL difference - if next_page_url: + if next_page_url and next_page_url != original_url_before_navigation: print('Searching for pagination parameter') # Parse both URLs using Python instead of JavaScript to avoid execution context issues try: