cleanup

Dockers/puppeteer-api/app/config.py

@@ -11,6 +11,9 @@ CACHE_EXPIRY_HOURS = int(os.getenv('CACHE_EXPIRY_HOURS', '36'))
 # Get cleanup cron schedule from environment variable (default: every day at 3 AM)
 CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *')
 
+# Get browser instance timeout from environment variable (default: 10 minutes)
+BROWSER_INSTANCE_TIMEOUT_MINUTES = int(os.getenv('BROWSER_INSTANCE_TIMEOUT_MINUTES', '10'))
+
 # Define custom user agent
 CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
 

Dockers/puppeteer-api/app/routes/browser.py

@@ -8,7 +8,8 @@ from app.services.browser import (
     extract_seo_service,
     extract_meta_tags_service,
     detect_pagination_service,
-    capture_outgoing_calls_service
+    capture_outgoing_calls_service,
+    fetch_json_service
 )
 
 router = APIRouter()
@@ -152,4 +153,33 @@ async def capture_outgoing_calls(url: str, skipCache: bool = False, x_api_key: O
         return result
     except Exception as e:
         print(f"Error capturing outgoing calls for URL {decoded_url}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/json")
+async def fetch_json(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)):
+    """Fetch JSON content from a website"""
+    # Validate API key
+    if not x_api_key or x_api_key != API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    # Decode URL if it's encoded
+    decoded_url = unquote(url)
+
+    # Check cache first (unless skipCache is True)
+    if not skipCache:
+        cached_result = get_cached_data(decoded_url, "json")
+        if cached_result:
+            return cached_result
+
+    try:
+        # Call the service function
+        result = await fetch_json_service(decoded_url)
+
+        # Save to cache
+        if(result["status"] == "success"):
+            save_to_cache(decoded_url, "json", result)
+
+        return result
+    except Exception as e:
+        print(f"Error fetching JSON for URL {decoded_url}: {e}")
         raise HTTPException(status_code=500, detail=str(e))

Dockers/puppeteer-api/app/services/browser.py

@@ -894,3 +894,107 @@ async def capture_outgoing_calls_service(decoded_url):
 
     # Perform the operation
     return await safe_browser_operation(decoded_url, outgoing_calls_operation)
+
+async def fetch_json_service(decoded_url):
+    """Service function to fetch JSON content from a website"""
+    print(f"Fetching JSON from: {decoded_url}")
+
+    # Define the operation to perform with the browser
+    async def json_operation(page):
+        try:
+            # Navigate to the URL
+            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
+
+            if not response:
+                raise Exception("No response received from the URL")
+
+            # Check if the response is JSON
+            content_type = response.headers.get('content-type', '').lower()
+            if 'application/json' not in content_type and 'text/json' not in content_type:
+                # If not JSON, try to find JSON content in the page
+                json_content = await page.evaluate('''() => {
+                    // Look for JSON in script tags
+                    const scripts = document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]');
+                    if (scripts.length > 0) {
+                        return Array.from(scripts).map(script => {
+                            try {
+                                return JSON.parse(script.textContent);
+                            } catch (e) {
+                                return null;
+                            }
+                        }).filter(json => json !== null);
+                    }
+
+                    // Look for JSON in data attributes
+                    const elementsWithData = document.querySelectorAll('[data-json]');
+                    if (elementsWithData.length > 0) {
+                        return Array.from(elementsWithData).map(el => {
+                            try {
+                                return JSON.parse(el.getAttribute('data-json'));
+                            } catch (e) {
+                                return null;
+                            }
+                        }).filter(json => json !== null);
+                    }
+
+                    // Look for JSON in the page content (try to find JSON-like structures)
+                    const bodyText = document.body.innerText;
+                    const jsonMatches = bodyText.match(/\\{[^{}]*\\}/g);
+                    if (jsonMatches) {
+                        const validJsons = [];
+                        for (const match of jsonMatches) {
+                            try {
+                                const parsed = JSON.parse(match);
+                                validJsons.push(parsed);
+                            } catch (e) {
+                                // Skip invalid JSON
+                            }
+                        }
+                        if (validJsons.length > 0) {
+                            return validJsons;
+                        }
+                    }
+
+                    return null;
+                }''')
+
+                if json_content:
+                    return {
+                        "status": "success",
+                        "url": decoded_url,
+                        "content_type": "json_extracted",
+                        "json_data": json_content
+                    }
+                else:
+                    # Try to get the page content and check if it's JSON
+                    content = await page.content()
+                    try:
+                        import json
+                        json_data = json.loads(content)
+                        return {
+                            "status": "success",
+                            "url": decoded_url,
+                            "content_type": "json_direct",
+                            "json_data": json_data
+                        }
+                    except json.JSONDecodeError:
+                        raise Exception("No JSON content found on the page")
+            else:
+                # Response is already JSON
+                try:
+                    json_data = await response.json()
+                    return {
+                        "status": "success",
+                        "url": decoded_url,
+                        "content_type": "json_response",
+                        "json_data": json_data
+                    }
+                except Exception as e:
+                    raise Exception(f"Failed to parse JSON response: {str(e)}")
+
+        except Exception as e:
+            print(f"Error during JSON fetching: {e}")
+            return {"status": "error", "url": decoded_url, "error": str(e)}
+
+    # Perform the operation
+    return await safe_browser_operation(decoded_url, json_operation)

Dockers/puppeteer-api/app/utils/browser_utils.py

@@ -1,6 +1,11 @@
 from playwright.async_api import async_playwright
-from app.config import CUSTOM_USER_AGENT
+from app.config import CUSTOM_USER_AGENT, BROWSER_INSTANCE_TIMEOUT_MINUTES
 import asyncio
+import time
+
+# Global tracking for browser instances created by this module
+page_creation_times = {}
+active_pages = set()
 
 async def wait_for_network_idle(page):
     """Wait until no network requests are in flight"""
@@ -32,6 +37,10 @@ async def safe_browser_operation(url, operation_func):
         page = await context.new_page()
         page.set_default_timeout(30000)
 
+        # Track page creation time for force cleanup
+        page_creation_times[page] = time.time()
+        active_pages.add(page)
+
         # Call the operation function that uses the page
         result = await operation_func(page)
 
@@ -46,9 +55,14 @@ async def safe_browser_operation(url, operation_func):
             "url": url
         }
     finally:
-        # Ensure page is closed properly
+        # Cleanup page tracking
         if page:
             try:
+                # Remove from tracking
+                if page in active_pages:
+                    active_pages.remove(page)
+                if page in page_creation_times:
+                    del page_creation_times[page]
                 await page.close()
             except Exception as e:
                 print(f"Error closing page: {e}")
@@ -72,4 +86,33 @@ async def safe_browser_operation(url, operation_func):
             try:
                 await playwright.stop()
             except Exception as e:
-                print(f"Error closing playwright: {e}")
+                print(f"Error closing playwright: {e}")
+
+async def force_cleanup_old_pages():
+    """Force cleanup old page instances created by this module"""
+    print(f"Checking for old page instances in browser_utils (timeout: {BROWSER_INSTANCE_TIMEOUT_MINUTES} minutes)...")
+
+    current_time = time.time()
+    timeout_seconds = BROWSER_INSTANCE_TIMEOUT_MINUTES * 60
+    cleaned_pages = 0
+
+    # Clean up old pages
+    pages_to_cleanup = []
+    for page in list(active_pages):
+        creation_time = page_creation_times.get(page, 0)
+        if current_time - creation_time > timeout_seconds:
+            pages_to_cleanup.append(page)
+            print(f"Marking page for cleanup (age: {(current_time - creation_time)/60:.1f} minutes)")
+
+    for page in pages_to_cleanup:
+        try:
+            if page in active_pages:
+                active_pages.remove(page)
+            if page in page_creation_times:
+                del page_creation_times[page]
+            await page.close()
+            cleaned_pages += 1
+        except Exception as e:
+            print(f"Error cleaning up old page: {e}")
+
+    print(f"Force cleanup completed: {cleaned_pages} pages cleaned")