diff --git a/Dockers/puppeteer-api/app/routes/browser.py b/Dockers/puppeteer-api/app/routes/browser.py index 5d0eef7..b895490 100644 --- a/Dockers/puppeteer-api/app/routes/browser.py +++ b/Dockers/puppeteer-api/app/routes/browser.py @@ -7,7 +7,8 @@ from app.services.browser import ( visit_url_service, extract_seo_service, extract_meta_tags_service, - detect_pagination_service + detect_pagination_service, + capture_outgoing_calls_service ) router = APIRouter() @@ -122,4 +123,33 @@ async def detect_pagination(url: str, skipCache: bool = False, x_api_key: Option return result except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +@router.get("/outgoing-calls") +async def capture_outgoing_calls(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)): + """Capture outgoing API calls from a website""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + # Decode URL if it's encoded + decoded_url = unquote(url) + + # Check cache first (unless skipCache is True) + if not skipCache: + cached_result = get_cached_data(decoded_url, "outgoing_calls") + if cached_result: + return cached_result + + try: + # Call the service function + result = await capture_outgoing_calls_service(decoded_url) + + # Save to cache + if(result["status"] == "success"): + save_to_cache(decoded_url, "outgoing_calls", result) + + return result + except Exception as e: + print(f"Error capturing outgoing calls for URL {decoded_url}: {e}") raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 3263241..2bc9c03 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -776,3 +776,121 @@ async def detect_pagination_service(decoded_url): return result return result + +async def capture_outgoing_calls_service(decoded_url): + """Service function to capture outgoing API calls from a website""" + print(f"Capturing outgoing calls from: {decoded_url}") + + # Define the operation to perform with the browser + async def outgoing_calls_operation(page): + try: + # List to store all network requests + network_requests = [] + + # Set up network request listener + async def handle_request(request): + # Only capture API-like requests (not static assets) + url = request.url + method = request.method + headers = request.headers + + # Skip static assets and common non-API requests + static_extensions = ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.eot'] + if any(url.endswith(ext) for ext in static_extensions): + return + + # Skip data URLs and blob URLs + if url.startswith(('data:', 'blob:')): + return + + # Skip same-origin requests that are likely static assets + if url.startswith(decoded_url) and any(static_ext in url.lower() for static_ext in static_extensions): + return + + # Capture the request details + request_data = { + "url": url, + "method": method, + "headers": dict(headers), + "timestamp": None # Will be set when request is finished + } + + # Store request for later processing + network_requests.append(request_data) + + # Listen to all requests + page.on("request", handle_request) + + # Navigate to the URL and wait for network to be idle + try: + await page.goto(decoded_url, wait_until='networkidle', timeout=30000) + except Exception as e: + print(f"Error during page navigation: {e}") + # Continue anyway to capture any requests that were made + + # Wait a bit more to catch any delayed requests + await page.wait_for_timeout(2000) + + # Process and categorize the requests + api_calls = [] + for req in network_requests: + # Determine if this looks like an API call + is_api_call = False + api_type = "unknown" + + # Check for common API patterns + if any(pattern in req["url"].lower() for pattern in ['/api/', '/rest/', '/graphql', '/json', '/xml']): + is_api_call = True + api_type = "rest" + elif req["url"].endswith('.json'): + is_api_call = True + api_type = "json" + elif 'application/json' in req["headers"].get('content-type', '').lower(): + is_api_call = True + api_type = "json" + elif 'application/xml' in req["headers"].get('content-type', '').lower(): + is_api_call = True + api_type = "xml" + elif req["method"] in ['POST', 'PUT', 'PATCH', 'DELETE']: + # These methods are typically API calls + is_api_call = True + api_type = "rest" + elif any(domain in req["url"] for domain in ['api.', 'rest.', 'graphql.']): + is_api_call = True + api_type = "rest" + + # Include all requests but mark API calls specifically + call_info = { + "url": req["url"], + "method": req["method"], + "is_api_call": is_api_call, + "api_type": api_type if is_api_call else None, + "headers": {k: v for k, v in req["headers"].items() if k.lower() not in ['user-agent', 'accept-encoding', 'accept-language', 'cache-control']} + } + + api_calls.append(call_info) + + # Sort by whether it's an API call (API calls first), then by URL + api_calls.sort(key=lambda x: (not x["is_api_call"], x["url"])) + + result = { + "status": "success", + "url": decoded_url, + "total_requests": len(api_calls), + "api_calls": [call for call in api_calls if call["is_api_call"]], + "other_requests": [call for call in api_calls if not call["is_api_call"]], + "summary": { + "api_calls_count": len([call for call in api_calls if call["is_api_call"]]), + "other_requests_count": len([call for call in api_calls if not call["is_api_call"]]), + "unique_domains": len(set([call["url"].split('/')[2] for call in api_calls if len(call["url"].split('/')) > 2])) + } + } + + return result + + except Exception as e: + print(f"Error during outgoing calls capture: {e}") + return {"status": "error", "url": decoded_url, "error": str(e)} + + # Perform the operation + return await safe_browser_operation(decoded_url, outgoing_calls_operation)