implement resulting url endpoint
Build and Push Docker Images / build-and-push (push) Has been cancelled
Build and Push Docker Images / build-and-push (push) Has been cancelled
This commit is contained in:
@@ -92,6 +92,7 @@ RATE_LIMIT_MINUTE=60 # Requests per minute (default: 60)
|
||||
- `GET /seo` - Extract SEO information
|
||||
- `GET /meta` - Extract meta tags and Open Graph data
|
||||
- `GET /json` - Fetch JSON content from a website
|
||||
- `GET /resulting-url` - Get the final URL after navigation (handles redirects)
|
||||
|
||||
The `/json` endpoint intelligently extracts JSON data from websites by:
|
||||
|
||||
@@ -124,6 +125,9 @@ curl -H "X-API-Key: your-api-key" "http://localhost:8000/seo?url=https://example
|
||||
# Fetch JSON content
|
||||
curl -H "X-API-Key: your-api-key" "http://localhost:8000/json?url=https://api.example.com/data"
|
||||
|
||||
# Get resulting URL (handles redirects)
|
||||
curl -H "X-API-Key: your-api-key" "http://localhost:8000/resulting-url?url=https://example.com"
|
||||
|
||||
# Get system status
|
||||
curl -H "X-API-Key: your-api-key" "http://localhost:8000/status"
|
||||
|
||||
|
||||
@@ -9,7 +9,8 @@ from app.services.browser import (
|
||||
extract_meta_tags_service,
|
||||
detect_pagination_service,
|
||||
capture_outgoing_calls_service,
|
||||
fetch_json_service
|
||||
fetch_json_service,
|
||||
get_resulting_url_service
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
@@ -183,3 +184,32 @@ async def fetch_json(url: str, skipCache: bool = False, x_api_key: Optional[str]
|
||||
except Exception as e:
|
||||
print(f"Error fetching JSON for URL {decoded_url}: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@router.get("/resulting-url")
|
||||
async def get_resulting_url(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
|
||||
"""Get the resulting URL after navigation (handles redirects)"""
|
||||
# Validate API key
|
||||
if not x_api_key or x_api_key != API_KEY:
|
||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||
|
||||
# Decode URL if it's encoded
|
||||
decoded_url = unquote(url)
|
||||
|
||||
# Check cache first (unless skipCache is True)
|
||||
if not skipCache:
|
||||
cached_result = get_cached_data(decoded_url, "resulting_url")
|
||||
if cached_result:
|
||||
return cached_result
|
||||
|
||||
try:
|
||||
# Call the service function
|
||||
result = await get_resulting_url_service(decoded_url)
|
||||
|
||||
# Save to cache
|
||||
if(result["status"] == "success"):
|
||||
save_to_cache(decoded_url, "resulting_url", result)
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"Error getting resulting URL for {decoded_url}: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
@@ -2,6 +2,7 @@ from app.utils.browser_utils import safe_browser_operation
|
||||
from app.utils.http_utils import fetch_url
|
||||
import asyncio
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import unquote
|
||||
|
||||
async def visit_url_service(decoded_url):
|
||||
"""Service function to visit a URL and get its content"""
|
||||
@@ -902,99 +903,84 @@ async def fetch_json_service(decoded_url):
|
||||
# Define the operation to perform with the browser
|
||||
async def json_operation(page):
|
||||
try:
|
||||
# Navigate to the URL
|
||||
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
||||
|
||||
if not response:
|
||||
raise Exception("No response received from the URL")
|
||||
print(f"Warning: No response object returned for {decoded_url}")
|
||||
|
||||
# Check if the response is JSON
|
||||
content_type = response.headers.get('content-type', '').lower()
|
||||
if 'application/json' not in content_type and 'text/json' not in content_type:
|
||||
# If not JSON, try to find JSON content in the page
|
||||
json_content = await page.evaluate('''() => {
|
||||
// Look for JSON in script tags
|
||||
const scripts = document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]');
|
||||
if (scripts.length > 0) {
|
||||
return Array.from(scripts).map(script => {
|
||||
try {
|
||||
return JSON.parse(script.textContent);
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}).filter(json => json !== null);
|
||||
}
|
||||
# Get the final URL after any redirects and decode it
|
||||
final_url = page.url
|
||||
decoded_final_url = unquote(final_url)
|
||||
|
||||
// Look for JSON in data attributes
|
||||
const elementsWithData = document.querySelectorAll('[data-json]');
|
||||
if (elementsWithData.length > 0) {
|
||||
return Array.from(elementsWithData).map(el => {
|
||||
try {
|
||||
return JSON.parse(el.getAttribute('data-json'));
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}).filter(json => json !== null);
|
||||
}
|
||||
|
||||
// Look for JSON in the page content (try to find JSON-like structures)
|
||||
const bodyText = document.body.innerText;
|
||||
const jsonMatches = bodyText.match(/\\{[^{}]*\\}/g);
|
||||
if (jsonMatches) {
|
||||
const validJsons = [];
|
||||
for (const match of jsonMatches) {
|
||||
try {
|
||||
const parsed = JSON.parse(match);
|
||||
validJsons.push(parsed);
|
||||
} catch (e) {
|
||||
// Skip invalid JSON
|
||||
}
|
||||
}
|
||||
if (validJsons.length > 0) {
|
||||
return validJsons;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}''')
|
||||
|
||||
if json_content:
|
||||
return {
|
||||
"status": "success",
|
||||
"url": decoded_url,
|
||||
"content_type": "json_extracted",
|
||||
"json_data": json_content
|
||||
}
|
||||
else:
|
||||
# Try to get the page content and check if it's JSON
|
||||
# Try to get JSON content
|
||||
try:
|
||||
content = await page.content()
|
||||
try:
|
||||
import json
|
||||
json_data = json.loads(content)
|
||||
# Check if the page contains JSON
|
||||
if content.strip().startswith('{') or content.strip().startswith('['):
|
||||
return {
|
||||
"status": "success",
|
||||
"url": decoded_url,
|
||||
"content_type": "json_direct",
|
||||
"json_data": json_data
|
||||
"final_url": decoded_final_url,
|
||||
"content": content,
|
||||
"content_type": "json"
|
||||
}
|
||||
except json.JSONDecodeError:
|
||||
raise Exception("No JSON content found on the page")
|
||||
else:
|
||||
# Response is already JSON
|
||||
try:
|
||||
json_data = await response.json()
|
||||
return {
|
||||
"status": "success",
|
||||
"url": decoded_url,
|
||||
"content_type": "json_response",
|
||||
"json_data": json_data
|
||||
"final_url": decoded_final_url,
|
||||
"content": content,
|
||||
"content_type": "html"
|
||||
}
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to parse JSON response: {str(e)}")
|
||||
return {
|
||||
"status": "partial",
|
||||
"url": decoded_url,
|
||||
"final_url": decoded_final_url,
|
||||
"error": f"Could not get content: {str(e)}"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during JSON fetching: {e}")
|
||||
print(f"Error during JSON fetch: {e}")
|
||||
return {"status": "error", "url": decoded_url, "error": str(e)}
|
||||
|
||||
# Perform the operation
|
||||
return await safe_browser_operation(decoded_url, json_operation)
|
||||
|
||||
async def get_resulting_url_service(decoded_url):
|
||||
"""Service function to get the resulting URL after navigation (handles redirects)"""
|
||||
print(f"Getting resulting URL for: {decoded_url}")
|
||||
|
||||
# Define the operation to perform with the browser
|
||||
async def resulting_url_operation(page):
|
||||
try:
|
||||
# Navigate to the URL and wait for network to be idle
|
||||
response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
|
||||
|
||||
# Get the final URL after any redirects and decode it
|
||||
final_url = page.url
|
||||
decoded_final_url = unquote(final_url)
|
||||
|
||||
# Get response status if available
|
||||
status_code = response.status if response else None
|
||||
|
||||
result = {
|
||||
"status": "success",
|
||||
"original_url": decoded_url,
|
||||
"resulting_url": decoded_final_url,
|
||||
"status_code": status_code
|
||||
}
|
||||
|
||||
# Check if there was a redirect
|
||||
if decoded_final_url != decoded_url:
|
||||
result["redirected"] = True
|
||||
else:
|
||||
result["redirected"] = False
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during URL navigation: {e}")
|
||||
return {"status": "error", "original_url": decoded_url, "error": str(e)}
|
||||
|
||||
# Perform the operation
|
||||
return await safe_browser_operation(decoded_url, resulting_url_operation)
|
||||
|
||||
Reference in New Issue
Block a user