outgoing calls route

Dockers/puppeteer-api/app/routes/browser.py

@@ -7,7 +7,8 @@ from app.services.browser import (
     visit_url_service,
     extract_seo_service,
     extract_meta_tags_service,
-    detect_pagination_service
+    detect_pagination_service,
+    capture_outgoing_calls_service
 )
 
 router = APIRouter()
@@ -123,3 +124,32 @@ async def detect_pagination(url: str, skipCache: bool = False, x_api_key: Option
         return result
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/outgoing-calls")
+async def capture_outgoing_calls(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
+    """Capture outgoing API calls from a website"""
+    # Validate API key
+    if not x_api_key or x_api_key != API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    # Decode URL if it's encoded
+    decoded_url = unquote(url)
+
+    # Check cache first (unless skipCache is True)
+    if not skipCache:
+        cached_result = get_cached_data(decoded_url, "outgoing_calls")
+        if cached_result:
+            return cached_result
+
+    try:
+        # Call the service function
+        result = await capture_outgoing_calls_service(decoded_url)
+
+        # Save to cache
+        if(result["status"] == "success"):
+            save_to_cache(decoded_url, "outgoing_calls", result)
+
+        return result
+    except Exception as e:
+        print(f"Error capturing outgoing calls for URL {decoded_url}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

Dockers/puppeteer-api/app/services/browser.py

@@ -776,3 +776,121 @@ async def detect_pagination_service(decoded_url):
         return result
 
     return result
+
+async def capture_outgoing_calls_service(decoded_url):
+    """Service function to capture outgoing API calls from a website"""
+    print(f"Capturing outgoing calls from: {decoded_url}")
+
+    # Define the operation to perform with the browser
+    async def outgoing_calls_operation(page):
+        try:
+            # List to store all network requests
+            network_requests = []
+
+            # Set up network request listener
+            async def handle_request(request):
+                # Only capture API-like requests (not static assets)
+                url = request.url
+                method = request.method
+                headers = request.headers
+
+                # Skip static assets and common non-API requests
+                static_extensions = ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.eot']
+                if any(url.endswith(ext) for ext in static_extensions):
+                    return
+
+                # Skip data URLs and blob URLs
+                if url.startswith(('data:', 'blob:')):
+                    return
+
+                # Skip same-origin requests that are likely static assets
+                if url.startswith(decoded_url) and any(static_ext in url.lower() for static_ext in static_extensions):
+                    return
+
+                # Capture the request details
+                request_data = {
+                    "url": url,
+                    "method": method,
+                    "headers": dict(headers),
+                    "timestamp": None  # Will be set when request is finished
+                }
+
+                # Store request for later processing
+                network_requests.append(request_data)
+
+            # Listen to all requests
+            page.on("request", handle_request)
+
+            # Navigate to the URL and wait for network to be idle
+            try:
+                await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
+            except Exception as e:
+                print(f"Error during page navigation: {e}")
+                # Continue anyway to capture any requests that were made
+
+            # Wait a bit more to catch any delayed requests
+            await page.wait_for_timeout(2000)
+
+            # Process and categorize the requests
+            api_calls = []
+            for req in network_requests:
+                # Determine if this looks like an API call
+                is_api_call = False
+                api_type = "unknown"
+
+                # Check for common API patterns
+                if any(pattern in req["url"].lower() for pattern in ['/api/', '/rest/', '/graphql', '/json', '/xml']):
+                    is_api_call = True
+                    api_type = "rest"
+                elif req["url"].endswith('.json'):
+                    is_api_call = True
+                    api_type = "json"
+                elif 'application/json' in req["headers"].get('content-type', '').lower():
+                    is_api_call = True
+                    api_type = "json"
+                elif 'application/xml' in req["headers"].get('content-type', '').lower():
+                    is_api_call = True
+                    api_type = "xml"
+                elif req["method"] in ['POST', 'PUT', 'PATCH', 'DELETE']:
+                    # These methods are typically API calls
+                    is_api_call = True
+                    api_type = "rest"
+                elif any(domain in req["url"] for domain in ['api.', 'rest.', 'graphql.']):
+                    is_api_call = True
+                    api_type = "rest"
+
+                # Include all requests but mark API calls specifically
+                call_info = {
+                    "url": req["url"],
+                    "method": req["method"],
+                    "is_api_call": is_api_call,
+                    "api_type": api_type if is_api_call else None,
+                    "headers": {k: v for k, v in req["headers"].items() if k.lower() not in ['user-agent', 'accept-encoding', 'accept-language', 'cache-control']}
+                }
+
+                api_calls.append(call_info)
+
+            # Sort by whether it's an API call (API calls first), then by URL
+            api_calls.sort(key=lambda x: (not x["is_api_call"], x["url"]))
+
+            result = {
+                "status": "success",
+                "url": decoded_url,
+                "total_requests": len(api_calls),
+                "api_calls": [call for call in api_calls if call["is_api_call"]],
+                "other_requests": [call for call in api_calls if not call["is_api_call"]],
+                "summary": {
+                    "api_calls_count": len([call for call in api_calls if call["is_api_call"]]),
+                    "other_requests_count": len([call for call in api_calls if not call["is_api_call"]]),
+                    "unique_domains": len(set([call["url"].split('/')[2] for call in api_calls if len(call["url"].split('/')) > 2]))
+                }
+            }
+
+            return result
+
+        except Exception as e:
+            print(f"Error during outgoing calls capture: {e}")
+            return {"status": "error", "url": decoded_url, "error": str(e)}
+
+    # Perform the operation
+    return await safe_browser_operation(decoded_url, outgoing_calls_operation)