implement resulting url endpoint

Dockers/puppeteer-api/README.md

@@ -92,6 +92,7 @@ RATE_LIMIT_MINUTE=60             # Requests per minute (default: 60)
 - `GET /seo` - Extract SEO information
 - `GET /meta` - Extract meta tags and Open Graph data
 - `GET /json` - Fetch JSON content from a website
+- `GET /resulting-url` - Get the final URL after navigation (handles redirects)
 
 The `/json` endpoint intelligently extracts JSON data from websites by:
 
@@ -124,6 +125,9 @@ curl -H "X-API-Key: your-api-key" "http://localhost:8000/seo?url=https://example
 # Fetch JSON content
 curl -H "X-API-Key: your-api-key" "http://localhost:8000/json?url=https://api.example.com/data"
 
+# Get resulting URL (handles redirects)
+curl -H "X-API-Key: your-api-key" "http://localhost:8000/resulting-url?url=https://example.com"
+
 # Get system status
 curl -H "X-API-Key: your-api-key" "http://localhost:8000/status"
 

Dockers/puppeteer-api/app/routes/browser.py

@@ -9,7 +9,8 @@ from app.services.browser import (
     extract_meta_tags_service,
     detect_pagination_service,
     capture_outgoing_calls_service,
-    fetch_json_service
+    fetch_json_service,
+    get_resulting_url_service
 )
 
 router = APIRouter()
@@ -182,4 +183,33 @@ async def fetch_json(url: str, skipCache: bool = False, x_api_key: Optional[str]
         return result
     except Exception as e:
         print(f"Error fetching JSON for URL {decoded_url}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/resulting-url")
+async def get_resulting_url(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
+    """Get the resulting URL after navigation (handles redirects)"""
+    # Validate API key
+    if not x_api_key or x_api_key != API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    # Decode URL if it's encoded
+    decoded_url = unquote(url)
+
+    # Check cache first (unless skipCache is True)
+    if not skipCache:
+        cached_result = get_cached_data(decoded_url, "resulting_url")
+        if cached_result:
+            return cached_result
+
+    try:
+        # Call the service function
+        result = await get_resulting_url_service(decoded_url)
+
+        # Save to cache
+        if(result["status"] == "success"):
+            save_to_cache(decoded_url, "resulting_url", result)
+
+        return result
+    except Exception as e:
+        print(f"Error getting resulting URL for {decoded_url}: {e}")
         raise HTTPException(status_code=500, detail=str(e))

Dockers/puppeteer-api/app/services/browser.py

@@ -2,6 +2,7 @@ from app.utils.browser_utils import safe_browser_operation
 from app.utils.http_utils import fetch_url
 import asyncio
 from bs4 import BeautifulSoup
+from urllib.parse import unquote
 
 async def visit_url_service(decoded_url):
     """Service function to visit a URL and get its content"""
@@ -902,99 +903,84 @@ async def fetch_json_service(decoded_url):
     # Define the operation to perform with the browser
     async def json_operation(page):
         try:
-            # Navigate to the URL
             response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
-
             if not response:
-                raise Exception("No response received from the URL")
+                print(f"Warning: No response object returned for {decoded_url}")
 
-            # Check if the response is JSON
+            # Get the final URL after any redirects and decode it
-            content_type = response.headers.get('content-type', '').lower()
+            final_url = page.url
-            if 'application/json' not in content_type and 'text/json' not in content_type:
+            decoded_final_url = unquote(final_url)
-                # If not JSON, try to find JSON content in the page
-                json_content = await page.evaluate('''() => {
-                    // Look for JSON in script tags
-                    const scripts = document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]');
-                    if (scripts.length > 0) {
-                        return Array.from(scripts).map(script => {
-                            try {
-                                return JSON.parse(script.textContent);
-                            } catch (e) {
-                                return null;
-                            }
-                        }).filter(json => json !== null);
-                    }
 
-                    // Look for JSON in data attributes
+            # Try to get JSON content
-                    const elementsWithData = document.querySelectorAll('[data-json]');
+            try:
-                    if (elementsWithData.length > 0) {
+                content = await page.content()
-                        return Array.from(elementsWithData).map(el => {
+                # Check if the page contains JSON
-                            try {
+                if content.strip().startswith('{') or content.strip().startswith('['):
-                                return JSON.parse(el.getAttribute('data-json'));
-                            } catch (e) {
-                                return null;
-                            }
-                        }).filter(json => json !== null);
-                    }
-
-                    // Look for JSON in the page content (try to find JSON-like structures)
-                    const bodyText = document.body.innerText;
-                    const jsonMatches = bodyText.match(/\\{[^{}]*\\}/g);
-                    if (jsonMatches) {
-                        const validJsons = [];
-                        for (const match of jsonMatches) {
-                            try {
-                                const parsed = JSON.parse(match);
-                                validJsons.push(parsed);
-                            } catch (e) {
-                                // Skip invalid JSON
-                            }
-                        }
-                        if (validJsons.length > 0) {
-                            return validJsons;
-                        }
-                    }
-
-                    return null;
-                }''')
-
-                if json_content:
                     return {
                         "status": "success",
                         "url": decoded_url,
-                        "content_type": "json_extracted",
+                        "final_url": decoded_final_url,
-                        "json_data": json_content
+                        "content": content,
+                        "content_type": "json"
                     }
                 else:
-                    # Try to get the page content and check if it's JSON
-                    content = await page.content()
-                    try:
-                        import json
-                        json_data = json.loads(content)
-                        return {
-                            "status": "success",
-                            "url": decoded_url,
-                            "content_type": "json_direct",
-                            "json_data": json_data
-                        }
-                    except json.JSONDecodeError:
-                        raise Exception("No JSON content found on the page")
-            else:
-                # Response is already JSON
-                try:
-                    json_data = await response.json()
                     return {
                         "status": "success",
                         "url": decoded_url,
-                        "content_type": "json_response",
+                        "final_url": decoded_final_url,
-                        "json_data": json_data
+                        "content": content,
+                        "content_type": "html"
                     }
-                except Exception as e:
+            except Exception as e:
-                    raise Exception(f"Failed to parse JSON response: {str(e)}")
+                return {
+                    "status": "partial",
+                    "url": decoded_url,
+                    "final_url": decoded_final_url,
+                    "error": f"Could not get content: {str(e)}"
+                }
 
         except Exception as e:
-            print(f"Error during JSON fetching: {e}")
+            print(f"Error during JSON fetch: {e}")
             return {"status": "error", "url": decoded_url, "error": str(e)}
 
     # Perform the operation
     return await safe_browser_operation(decoded_url, json_operation)
+
+async def get_resulting_url_service(decoded_url):
+    """Service function to get the resulting URL after navigation (handles redirects)"""
+    print(f"Getting resulting URL for: {decoded_url}")
+
+    # Define the operation to perform with the browser
+    async def resulting_url_operation(page):
+        try:
+            # Navigate to the URL and wait for network to be idle
+            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
+
+            # Get the final URL after any redirects and decode it
+            final_url = page.url
+            decoded_final_url = unquote(final_url)
+
+            # Get response status if available
+            status_code = response.status if response else None
+
+            result = {
+                "status": "success",
+                "original_url": decoded_url,
+                "resulting_url": decoded_final_url,
+                "status_code": status_code
+            }
+
+            # Check if there was a redirect
+            if decoded_final_url != decoded_url:
+                result["redirected"] = True
+            else:
+                result["redirected"] = False
+
+            return result
+
+        except Exception as e:
+            print(f"Error during URL navigation: {e}")
+            return {"status": "error", "original_url": decoded_url, "error": str(e)}
+
+    # Perform the operation
+    return await safe_browser_operation(decoded_url, resulting_url_operation)