wait for page

Dockers/puppeteer-api/app/services/browser.py

@@ -166,6 +166,7 @@ async def detect_pagination_service(decoded_url):
                 original_url = page.url
 
             # Check for common pagination indicators
+            await asyncio.sleep(2)
             try:
                 pagination_data = await page.evaluate('''() => {
                     const data = {
@@ -192,7 +193,6 @@ async def detect_pagination_service(decoded_url):
                                     el.closest('button, [role="button"]'));
                         });
 
-                    console.log(numberedLinks);
                     if(numberedLinks.length <= 1) {
                         return data;
                     }
@@ -237,7 +237,7 @@ async def detect_pagination_service(decoded_url):
                         const pageNumbers = numberedLinks
                             .map(el => parseInt(el.innerText.replace(/[^\d]/g, '')))
                             .filter(num => !isNaN(num));
-
+                        console.log(pageNumbers);
                         if (pageNumbers.length > 0) {
                             data.lastPageNumber = Math.max(...pageNumbers);
                         }
@@ -261,6 +261,7 @@ async def detect_pagination_service(decoded_url):
                                     if (url.searchParams.has(param)) {
                                         const value = parseInt(url.searchParams.get(param));
                                         if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
+                                            console.log("Last page number: " + value);
                                             data.lastPageNumber = value;
                                         }
                                     }
@@ -271,6 +272,7 @@ async def detect_pagination_service(decoded_url):
                                 if (pathMatch && pathMatch[1]) {
                                     const value = parseInt(pathMatch[1]);
                                     if (!isNaN(value) && (data.lastPageNumber === null || value > data.lastPageNumber)) {
+                                        console.log("Last page number: " + value);
                                         data.lastPageNumber = value;
                                     }
                                 }
@@ -323,6 +325,7 @@ async def detect_pagination_service(decoded_url):
                     'lastPageNumber': None
                 }
 
+            print(f"Pagination data: {pagination_data}")
             # If pagination is detected, try to navigate to the next page by clicking
             next_page_url = None
             pagination_parameter = None
@@ -440,6 +443,7 @@ async def detect_pagination_service(decoded_url):
                         print("Successfully clicked on pagination element")
                         # Wait for navigation to complete
                         await page.wait_for_load_state('networkidle', timeout=10000)
+                        await asyncio.sleep(2)
                         next_page_url = page.url
                     else:
                         print("No clickable pagination element found")
@@ -448,8 +452,10 @@ async def detect_pagination_service(decoded_url):
                     print(f"Error clicking on pagination element: {e}")
                     # Continue with the process even if clicking fails
 
+            print(f"Next page URL: {next_page_url}")
+
             # If we successfully navigated to the next page, analyze the URL difference
-            if next_page_url:
+            if next_page_url and next_page_url != original_url_before_navigation:
                 print('Searching for pagination parameter')
                 # Parse both URLs using Python instead of JavaScript to avoid execution context issues
                 try: