better meta tags

Dockers/puppeteer-api/app/services/browser.py

@@ -1,7 +1,5 @@
 from app.utils.browser_utils import safe_browser_operation
-from app.utils.http_utils import fetch_url
 import asyncio
-from bs4 import BeautifulSoup
 from urllib.parse import unquote
 
 async def visit_url_service(decoded_url):
@@ -103,41 +101,49 @@ async def extract_meta_tags_service(decoded_url):
     """Service function to extract meta tags from a website"""
     print(f"Extracting meta tags from: {decoded_url}")
 
+    # Define the operation to perform with the browser
+    async def meta_tags_operation(page):
         try:
-        # Fetch the page content using plain HTTP request
+            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
-        content, status = await fetch_url(decoded_url)
 
-        # Parse the HTML
+            # Extract meta tags using JavaScript
-        soup = BeautifulSoup(content, 'html.parser')
+            meta_data = await page.evaluate('''() => {
+                const data = {
+                    meta_tags: [],
+                    open_graph: {},
+                    twitter_card: {},
+                    title: document.title || ''
+                };
 
-        # Extract all meta tags
+                // Extract all meta tags
-        meta_tags = []
+                document.querySelectorAll('meta').forEach(meta => {
-        for meta in soup.find_all('meta'):
+                    const attributes = {};
-            attributes = {}
+                    for (let attr of meta.attributes) {
-            for attr in meta.attrs:
+                        attributes[attr.name] = attr.value;
-                attributes[attr] = meta[attr]
+                    }
-            meta_tags.append(attributes)
+                    data.meta_tags.push(attributes);
+                });
 
-        # Extract Open Graph tags
+                // Extract Open Graph tags
-        og_tags = {}
+                document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
-        for meta in soup.find_all('meta', property=lambda x: x and x.startswith('og:')):
+                    data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
-            og_tags[meta.get('property')] = meta.get('content')
+                });
 
-        # Extract Twitter card tags
+                // Extract Twitter card tags
-        twitter_tags = {}
+                document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
-        for meta in soup.find_all('meta', attrs={'name': lambda x: x and x.startswith('twitter:')}):
+                    data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
-            twitter_tags[meta.get('name')] = meta.get('content')
+                });
 
-        # Get page title
+                return data;
-        title = soup.title.string if soup.title else ''
+            }''')
 
             result = {
                 "status": "success",
                 "url": decoded_url,
-            "meta_tags": meta_tags,
+                "meta_tags": meta_data["meta_tags"],
-            "open_graph": og_tags,
+                "open_graph": meta_data["open_graph"],
-            "twitter_card": twitter_tags,
+                "twitter_card": meta_data["twitter_card"],
-            "title": title
+                "title": meta_data["title"]
             }
 
             return result
@@ -146,6 +152,9 @@ async def extract_meta_tags_service(decoded_url):
             print(f"Error during meta tag extraction: {e}")
             return {"status": "error", "url": decoded_url, "error": str(e)}
 
+    # Perform the operation
+    return await safe_browser_operation(decoded_url, meta_tags_operation)
+
 async def capture_outgoing_calls_service(decoded_url):
     """Service function to capture outgoing API calls from a website"""
     print(f"Capturing outgoing calls from: {decoded_url}")