From 4e3ef5d207f7aba8671a986d57299a21653e7012 Mon Sep 17 00:00:00 2001
From: Bram <info@bramkelchtermans.be>
Date: Sun, 13 Jul 2025 14:46:10 +0200
Subject: [PATCH] remove /json route

---
 Dockers/puppeteer-api/README.md               | 12 -----
 Dockers/puppeteer-api/app/routes/browser.py   | 29 -----------
 Dockers/puppeteer-api/app/services/browser.py | 51 -------------------
 3 files changed, 92 deletions(-)

diff --git a/Dockers/puppeteer-api/README.md b/Dockers/puppeteer-api/README.md
index 6436267..051dda7 100644
--- a/Dockers/puppeteer-api/README.md
+++ b/Dockers/puppeteer-api/README.md
@@ -91,17 +91,8 @@ RATE_LIMIT_MINUTE=60             # Requests per minute (default: 60)
 - `GET /` - Visit URL and get HTML content
 - `GET /seo` - Extract SEO information
 - `GET /meta` - Extract meta tags and Open Graph data
-- `GET /json` - Fetch JSON content from a website
 - `GET /resulting-url` - Get the final URL after navigation (handles redirects)
 
-The `/json` endpoint intelligently extracts JSON data from websites by:
-
-- Directly parsing JSON responses (when Content-Type is application/json)
-- Extracting JSON from `<script type="application/json">` tags
-- Finding JSON in `data-json` attributes
-- Searching for JSON-like structures in page content
-- Parsing the entire page content as JSON if possible
-
 ### Management Endpoints
 
 - `HEAD /` - Health check
@@ -122,9 +113,6 @@ curl -H "X-API-Key: your-api-key" "http://localhost:8000/?url=https://example.co
 # Extract SEO data
 curl -H "X-API-Key: your-api-key" "http://localhost:8000/seo?url=https://example.com"
 
-# Fetch JSON content
-curl -H "X-API-Key: your-api-key" "http://localhost:8000/json?url=https://api.example.com/data"
-
 # Get resulting URL (handles redirects)
 curl -H "X-API-Key: your-api-key" "http://localhost:8000/resulting-url?url=https://example.com"
 
diff --git a/Dockers/puppeteer-api/app/routes/browser.py b/Dockers/puppeteer-api/app/routes/browser.py
index f66c78d..aefe81d 100644
--- a/Dockers/puppeteer-api/app/routes/browser.py
+++ b/Dockers/puppeteer-api/app/routes/browser.py
@@ -9,7 +9,6 @@ from app.services.browser import (
     extract_meta_tags_service,
     detect_pagination_service,
     capture_outgoing_calls_service,
-    fetch_json_service,
     get_resulting_url_service
 )
 
@@ -156,34 +155,6 @@ async def capture_outgoing_calls(url: str, skipCache: bool = False, x_api_key: O
         print(f"Error capturing outgoing calls for URL {decoded_url}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
-@router.get("/json")
-async def fetch_json(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
-    """Fetch JSON content from a website"""
-    # Validate API key
-    if not x_api_key or x_api_key != API_KEY:
-        raise HTTPException(status_code=401, detail="Invalid API key")
-
-    # Decode URL if it's encoded
-    decoded_url = unquote(url)
-
-    # Check cache first (unless skipCache is True)
-    if not skipCache:
-        cached_result = get_cached_data(decoded_url, "json")
-        if cached_result:
-            return cached_result
-
-    try:
-        # Call the service function
-        result = await fetch_json_service(decoded_url)
-
-        # Save to cache
-        if(result["status"] == "success"):
-            save_to_cache(decoded_url, "json", result)
-
-        return result
-    except Exception as e:
-        print(f"Error fetching JSON for URL {decoded_url}: {e}")
-        raise HTTPException(status_code=500, detail=str(e))
 
 @router.get("/resulting-url")
 async def get_resulting_url(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py
index 979da2f..7a30f32 100644
--- a/Dockers/puppeteer-api/app/services/browser.py
+++ b/Dockers/puppeteer-api/app/services/browser.py
@@ -895,57 +895,6 @@ async def capture_outgoing_calls_service(decoded_url):
 
     # Perform the operation
     return await safe_browser_operation(decoded_url, outgoing_calls_operation)
-
-async def fetch_json_service(decoded_url):
-    """Service function to fetch JSON content from a website"""
-    print(f"Fetching JSON from: {decoded_url}")
-
-    # Define the operation to perform with the browser
-    async def json_operation(page):
-        try:
-            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
-            if not response:
-                print(f"Warning: No response object returned for {decoded_url}")
-
-            # Get the final URL after any redirects and decode it
-            final_url = page.url
-            decoded_final_url = unquote(final_url)
-
-            # Try to get JSON content
-            try:
-                content = await page.content()
-                # Check if the page contains JSON
-                if content.strip().startswith('{') or content.strip().startswith('['):
-                    return {
-                        "status": "success",
-                        "url": decoded_url,
-                        "final_url": decoded_final_url,
-                        "content": content,
-                        "content_type": "json"
-                    }
-                else:
-                    return {
-                        "status": "success",
-                        "url": decoded_url,
-                        "final_url": decoded_final_url,
-                        "content": content,
-                        "content_type": "html"
-                    }
-            except Exception as e:
-                return {
-                    "status": "partial",
-                    "url": decoded_url,
-                    "final_url": decoded_final_url,
-                    "error": f"Could not get content: {str(e)}"
-                }
-
-        except Exception as e:
-            print(f"Error during JSON fetch: {e}")
-            return {"status": "error", "url": decoded_url, "error": str(e)}
-
-    # Perform the operation
-    return await safe_browser_operation(decoded_url, json_operation)
-
 async def get_resulting_url_service(decoded_url):
     """Service function to get the resulting URL after navigation (handles redirects)"""
     print(f"Getting resulting URL for: {decoded_url}")