lightweight meta tags

Dockers/puppeteer-api/app/services/browser.py

@@ -1,5 +1,8 @@
 from app.utils.browser_utils import safe_browser_operation
+from app.utils.http_utils import fetch_url
 import asyncio
+from bs4 import BeautifulSoup
+
 async def visit_url_service(decoded_url):
     """Service function to visit a URL and get its content"""
     print(f"Visiting URL: {decoded_url}")
@@ -99,60 +102,48 @@ async def extract_meta_tags_service(decoded_url):
     """Service function to extract meta tags from a website"""
     print(f"Extracting meta tags from: {decoded_url}")
 
-    # Define the operation to perform with the browser
+    try:
-    async def meta_operation(page):
+        # Fetch the page content using plain HTTP request
-        try:
+        content, status = await fetch_url(decoded_url)
-            response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
 
-            # Extract all meta tags
+        # Parse the HTML
-            meta_tags = await page.evaluate('''() => {
+        soup = BeautifulSoup(content, 'html.parser')
-                const metas = Array.from(document.querySelectorAll('meta'));
-                return metas.map(meta => {
-                    const attributes = {};
-                    Array.from(meta.attributes).forEach(attr => {
-                        attributes[attr.name] = attr.value;
-                    });
-                    return attributes;
-                });
-            }''')
 
-            # Extract Open Graph tags
+        # Extract all meta tags
-            og_tags = await page.evaluate('''() => {
+        meta_tags = []
-                const ogTags = {};
+        for meta in soup.find_all('meta'):
-                document.querySelectorAll('meta[property^="og:"]').forEach(tag => {
+            attributes = {}
-                    const property = tag.getAttribute('property');
+            for attr in meta.attrs:
-                    ogTags[property] = tag.getAttribute('content');
+                attributes[attr] = meta[attr]
-                });
+            meta_tags.append(attributes)
-                return ogTags;
-            }''')
 
-            # Extract Twitter card tags
+        # Extract Open Graph tags
-            twitter_tags = await page.evaluate('''() => {
+        og_tags = {}
-                const twitterTags = {};
+        for meta in soup.find_all('meta', property=lambda x: x and x.startswith('og:')):
-                document.querySelectorAll('meta[name^="twitter:"]').forEach(tag => {
+            og_tags[meta.get('property')] = meta.get('content')
-                    const name = tag.getAttribute('name');
-                    twitterTags[name] = tag.getAttribute('content');
-                });
-                return twitterTags;
-            }''')
 
-            result = {
+        # Extract Twitter card tags
-                "status": "success",
+        twitter_tags = {}
-                "url": decoded_url,
+        for meta in soup.find_all('meta', attrs={'name': lambda x: x and x.startswith('twitter:')}):
-                "meta_tags": meta_tags,
+            twitter_tags[meta.get('name')] = meta.get('content')
-                "open_graph": og_tags,
-                "twitter_card": twitter_tags,
-                "title": await page.title()
-            }
 
-            return result
+        # Get page title
+        title = soup.title.string if soup.title else ''
 
-        except Exception as e:
+        result = {
-            print(f"Error during meta tag extraction: {e}")
+            "status": "success",
-            return {"status": "error", "url": decoded_url, "error": str(e)}
+            "url": decoded_url,
+            "meta_tags": meta_tags,
+            "open_graph": og_tags,
+            "twitter_card": twitter_tags,
+            "title": title
+        }
 
-    # Perform the operation
+        return result
-    return await safe_browser_operation(decoded_url, meta_operation)
+
+    except Exception as e:
+        print(f"Error during meta tag extraction: {e}")
+        return {"status": "error", "url": decoded_url, "error": str(e)}
 
 async def detect_pagination_service(decoded_url):
     """Service function to detect pagination on a website"""

Dockers/puppeteer-api/app/utils/http_utils.py

@@ -0,0 +1,15 @@
+from aiohttp import ClientSession, ClientTimeout
+from app.config import CUSTOM_USER_AGENT
+
+async def fetch_url(url: str) -> tuple[str, int]:
+    """
+    Fetch a URL using aiohttp and return the content and status code
+    """
+    timeout = ClientTimeout(total=30)  # 30 second timeout
+    async with ClientSession(timeout=timeout) as session:
+        headers = {
+            'User-Agent': CUSTOM_USER_AGENT
+        }
+        async with session.get(url, headers=headers) as response:
+            content = await response.text()
+            return content, response.status

Dockers/puppeteer-api/requirements.txt

@@ -2,4 +2,6 @@ fastapi==0.68.1
 uvicorn==0.15.0
 pyppeteer==1.0.2
 psutil==6.0.0
-apscheduler
+apscheduler
+aiohttp==3.9.1
+beautifulsoup4==4.12.2