diff --git a/Dockers/puppeteer-api/README.md b/Dockers/puppeteer-api/README.md index c881bf0..6436267 100644 --- a/Dockers/puppeteer-api/README.md +++ b/Dockers/puppeteer-api/README.md @@ -92,6 +92,7 @@ RATE_LIMIT_MINUTE=60 # Requests per minute (default: 60) - `GET /seo` - Extract SEO information - `GET /meta` - Extract meta tags and Open Graph data - `GET /json` - Fetch JSON content from a website +- `GET /resulting-url` - Get the final URL after navigation (handles redirects) The `/json` endpoint intelligently extracts JSON data from websites by: @@ -124,6 +125,9 @@ curl -H "X-API-Key: your-api-key" "http://localhost:8000/seo?url=https://example # Fetch JSON content curl -H "X-API-Key: your-api-key" "http://localhost:8000/json?url=https://api.example.com/data" +# Get resulting URL (handles redirects) +curl -H "X-API-Key: your-api-key" "http://localhost:8000/resulting-url?url=https://example.com" + # Get system status curl -H "X-API-Key: your-api-key" "http://localhost:8000/status" diff --git a/Dockers/puppeteer-api/app/routes/browser.py b/Dockers/puppeteer-api/app/routes/browser.py index 8a22fda..f66c78d 100644 --- a/Dockers/puppeteer-api/app/routes/browser.py +++ b/Dockers/puppeteer-api/app/routes/browser.py @@ -9,7 +9,8 @@ from app.services.browser import ( extract_meta_tags_service, detect_pagination_service, capture_outgoing_calls_service, - fetch_json_service + fetch_json_service, + get_resulting_url_service ) router = APIRouter() @@ -182,4 +183,33 @@ async def fetch_json(url: str, skipCache: bool = False, x_api_key: Optional[str] return result except Exception as e: print(f"Error fetching JSON for URL {decoded_url}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +@router.get("/resulting-url") +async def get_resulting_url(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)): + """Get the resulting URL after navigation (handles redirects)""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + # Decode URL if it's encoded + decoded_url = unquote(url) + + # Check cache first (unless skipCache is True) + if not skipCache: + cached_result = get_cached_data(decoded_url, "resulting_url") + if cached_result: + return cached_result + + try: + # Call the service function + result = await get_resulting_url_service(decoded_url) + + # Save to cache + if(result["status"] == "success"): + save_to_cache(decoded_url, "resulting_url", result) + + return result + except Exception as e: + print(f"Error getting resulting URL for {decoded_url}: {e}") raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 8343b0d..979da2f 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -2,6 +2,7 @@ from app.utils.browser_utils import safe_browser_operation from app.utils.http_utils import fetch_url import asyncio from bs4 import BeautifulSoup +from urllib.parse import unquote async def visit_url_service(decoded_url): """Service function to visit a URL and get its content""" @@ -902,99 +903,84 @@ async def fetch_json_service(decoded_url): # Define the operation to perform with the browser async def json_operation(page): try: - # Navigate to the URL response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000) - if not response: - raise Exception("No response received from the URL") + print(f"Warning: No response object returned for {decoded_url}") - # Check if the response is JSON - content_type = response.headers.get('content-type', '').lower() - if 'application/json' not in content_type and 'text/json' not in content_type: - # If not JSON, try to find JSON content in the page - json_content = await page.evaluate('''() => { - // Look for JSON in script tags - const scripts = document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]'); - if (scripts.length > 0) { - return Array.from(scripts).map(script => { - try { - return JSON.parse(script.textContent); - } catch (e) { - return null; - } - }).filter(json => json !== null); - } + # Get the final URL after any redirects and decode it + final_url = page.url + decoded_final_url = unquote(final_url) - // Look for JSON in data attributes - const elementsWithData = document.querySelectorAll('[data-json]'); - if (elementsWithData.length > 0) { - return Array.from(elementsWithData).map(el => { - try { - return JSON.parse(el.getAttribute('data-json')); - } catch (e) { - return null; - } - }).filter(json => json !== null); - } - - // Look for JSON in the page content (try to find JSON-like structures) - const bodyText = document.body.innerText; - const jsonMatches = bodyText.match(/\\{[^{}]*\\}/g); - if (jsonMatches) { - const validJsons = []; - for (const match of jsonMatches) { - try { - const parsed = JSON.parse(match); - validJsons.push(parsed); - } catch (e) { - // Skip invalid JSON - } - } - if (validJsons.length > 0) { - return validJsons; - } - } - - return null; - }''') - - if json_content: + # Try to get JSON content + try: + content = await page.content() + # Check if the page contains JSON + if content.strip().startswith('{') or content.strip().startswith('['): return { "status": "success", "url": decoded_url, - "content_type": "json_extracted", - "json_data": json_content + "final_url": decoded_final_url, + "content": content, + "content_type": "json" } else: - # Try to get the page content and check if it's JSON - content = await page.content() - try: - import json - json_data = json.loads(content) - return { - "status": "success", - "url": decoded_url, - "content_type": "json_direct", - "json_data": json_data - } - except json.JSONDecodeError: - raise Exception("No JSON content found on the page") - else: - # Response is already JSON - try: - json_data = await response.json() return { "status": "success", "url": decoded_url, - "content_type": "json_response", - "json_data": json_data + "final_url": decoded_final_url, + "content": content, + "content_type": "html" } - except Exception as e: - raise Exception(f"Failed to parse JSON response: {str(e)}") + except Exception as e: + return { + "status": "partial", + "url": decoded_url, + "final_url": decoded_final_url, + "error": f"Could not get content: {str(e)}" + } except Exception as e: - print(f"Error during JSON fetching: {e}") + print(f"Error during JSON fetch: {e}") return {"status": "error", "url": decoded_url, "error": str(e)} # Perform the operation return await safe_browser_operation(decoded_url, json_operation) + +async def get_resulting_url_service(decoded_url): + """Service function to get the resulting URL after navigation (handles redirects)""" + print(f"Getting resulting URL for: {decoded_url}") + + # Define the operation to perform with the browser + async def resulting_url_operation(page): + try: + # Navigate to the URL and wait for network to be idle + response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000) + + # Get the final URL after any redirects and decode it + final_url = page.url + decoded_final_url = unquote(final_url) + + # Get response status if available + status_code = response.status if response else None + + result = { + "status": "success", + "original_url": decoded_url, + "resulting_url": decoded_final_url, + "status_code": status_code + } + + # Check if there was a redirect + if decoded_final_url != decoded_url: + result["redirected"] = True + else: + result["redirected"] = False + + return result + + except Exception as e: + print(f"Error during URL navigation: {e}") + return {"status": "error", "original_url": decoded_url, "error": str(e)} + + # Perform the operation + return await safe_browser_operation(decoded_url, resulting_url_operation)