cleanup

Dockers/puppeteer-api/app/services/browser.py

@@ -894,3 +894,107 @@ async def capture_outgoing_calls_service(decoded_url):
 
     # Perform the operation
     return await safe_browser_operation(decoded_url, outgoing_calls_operation)
+
+async def fetch_json_service(decoded_url):
+    """Service function to fetch JSON content from a website"""
+    print(f"Fetching JSON from: {decoded_url}")
+
+    # Define the operation to perform with the browser
+    async def json_operation(page):
+        try:
+            # Navigate to the URL
+            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
+
+            if not response:
+                raise Exception("No response received from the URL")
+
+            # Check if the response is JSON
+            content_type = response.headers.get('content-type', '').lower()
+            if 'application/json' not in content_type and 'text/json' not in content_type:
+                # If not JSON, try to find JSON content in the page
+                json_content = await page.evaluate('''() => {
+                    // Look for JSON in script tags
+                    const scripts = document.querySelectorAll('script[type="application/json"], script[type="application/ld+json"]');
+                    if (scripts.length > 0) {
+                        return Array.from(scripts).map(script => {
+                            try {
+                                return JSON.parse(script.textContent);
+                            } catch (e) {
+                                return null;
+                            }
+                        }).filter(json => json !== null);
+                    }
+
+                    // Look for JSON in data attributes
+                    const elementsWithData = document.querySelectorAll('[data-json]');
+                    if (elementsWithData.length > 0) {
+                        return Array.from(elementsWithData).map(el => {
+                            try {
+                                return JSON.parse(el.getAttribute('data-json'));
+                            } catch (e) {
+                                return null;
+                            }
+                        }).filter(json => json !== null);
+                    }
+
+                    // Look for JSON in the page content (try to find JSON-like structures)
+                    const bodyText = document.body.innerText;
+                    const jsonMatches = bodyText.match(/\\{[^{}]*\\}/g);
+                    if (jsonMatches) {
+                        const validJsons = [];
+                        for (const match of jsonMatches) {
+                            try {
+                                const parsed = JSON.parse(match);
+                                validJsons.push(parsed);
+                            } catch (e) {
+                                // Skip invalid JSON
+                            }
+                        }
+                        if (validJsons.length > 0) {
+                            return validJsons;
+                        }
+                    }
+
+                    return null;
+                }''')
+
+                if json_content:
+                    return {
+                        "status": "success",
+                        "url": decoded_url,
+                        "content_type": "json_extracted",
+                        "json_data": json_content
+                    }
+                else:
+                    # Try to get the page content and check if it's JSON
+                    content = await page.content()
+                    try:
+                        import json
+                        json_data = json.loads(content)
+                        return {
+                            "status": "success",
+                            "url": decoded_url,
+                            "content_type": "json_direct",
+                            "json_data": json_data
+                        }
+                    except json.JSONDecodeError:
+                        raise Exception("No JSON content found on the page")
+            else:
+                # Response is already JSON
+                try:
+                    json_data = await response.json()
+                    return {
+                        "status": "success",
+                        "url": decoded_url,
+                        "content_type": "json_response",
+                        "json_data": json_data
+                    }
+                except Exception as e:
+                    raise Exception(f"Failed to parse JSON response: {str(e)}")
+
+        except Exception as e:
+            print(f"Error during JSON fetching: {e}")
+            return {"status": "error", "url": decoded_url, "error": str(e)}
+
+    # Perform the operation
+    return await safe_browser_operation(decoded_url, json_operation)