lightweight meta tags

Dockers/puppeteer-api/app/services/browser.py

@@ -1,5 +1,8 @@
 from app.utils.browser_utils import safe_browser_operation
+from app.utils.http_utils import fetch_url
 import asyncio
+from bs4 import BeautifulSoup
+
 async def visit_url_service(decoded_url):
     """Service function to visit a URL and get its content"""
     print(f"Visiting URL: {decoded_url}")
@@ -99,42 +102,33 @@ async def extract_meta_tags_service(decoded_url):
     """Service function to extract meta tags from a website"""
     print(f"Extracting meta tags from: {decoded_url}")
 
-    # Define the operation to perform with the browser
-    async def meta_operation(page):
     try:
-            response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+        # Fetch the page content using plain HTTP request
+        content, status = await fetch_url(decoded_url)
+
+        # Parse the HTML
+        soup = BeautifulSoup(content, 'html.parser')
 
         # Extract all meta tags
-            meta_tags = await page.evaluate('''() => {
+        meta_tags = []
-                const metas = Array.from(document.querySelectorAll('meta'));
+        for meta in soup.find_all('meta'):
-                return metas.map(meta => {
+            attributes = {}
-                    const attributes = {};
+            for attr in meta.attrs:
-                    Array.from(meta.attributes).forEach(attr => {
+                attributes[attr] = meta[attr]
-                        attributes[attr.name] = attr.value;
+            meta_tags.append(attributes)
-                    });
-                    return attributes;
-                });
-            }''')
 
         # Extract Open Graph tags
-            og_tags = await page.evaluate('''() => {
+        og_tags = {}
-                const ogTags = {};
+        for meta in soup.find_all('meta', property=lambda x: x and x.startswith('og:')):
-                document.querySelectorAll('meta[property^="og:"]').forEach(tag => {
+            og_tags[meta.get('property')] = meta.get('content')
-                    const property = tag.getAttribute('property');
-                    ogTags[property] = tag.getAttribute('content');
-                });
-                return ogTags;
-            }''')
 
         # Extract Twitter card tags
-            twitter_tags = await page.evaluate('''() => {
+        twitter_tags = {}
-                const twitterTags = {};
+        for meta in soup.find_all('meta', attrs={'name': lambda x: x and x.startswith('twitter:')}):
-                document.querySelectorAll('meta[name^="twitter:"]').forEach(tag => {
+            twitter_tags[meta.get('name')] = meta.get('content')
-                    const name = tag.getAttribute('name');
+
-                    twitterTags[name] = tag.getAttribute('content');
+        # Get page title
-                });
+        title = soup.title.string if soup.title else ''
-                return twitterTags;
-            }''')
 
         result = {
             "status": "success",
@@ -142,7 +136,7 @@ async def extract_meta_tags_service(decoded_url):
             "meta_tags": meta_tags,
             "open_graph": og_tags,
             "twitter_card": twitter_tags,
-                "title": await page.title()
+            "title": title
         }
 
         return result
@@ -151,9 +145,6 @@ async def extract_meta_tags_service(decoded_url):
         print(f"Error during meta tag extraction: {e}")
         return {"status": "error", "url": decoded_url, "error": str(e)}
 
-    # Perform the operation
-    return await safe_browser_operation(decoded_url, meta_operation)
-
 async def detect_pagination_service(decoded_url):
     """Service function to detect pagination on a website"""
     print(f"Detecting pagination on: {decoded_url}")

Dockers/puppeteer-api/app/utils/http_utils.py

@@ -0,0 +1,15 @@
+from aiohttp import ClientSession, ClientTimeout
+from app.config import CUSTOM_USER_AGENT
+
+async def fetch_url(url: str) -> tuple[str, int]:
+    """
+    Fetch a URL using aiohttp and return the content and status code
+    """
+    timeout = ClientTimeout(total=30)  # 30 second timeout
+    async with ClientSession(timeout=timeout) as session:
+        headers = {
+            'User-Agent': CUSTOM_USER_AGENT
+        }
+        async with session.get(url, headers=headers) as response:
+            content = await response.text()
+            return content, response.status

Dockers/puppeteer-api/requirements.txt

@@ -3,3 +3,5 @@ uvicorn==0.15.0
 pyppeteer==1.0.2
 psutil==6.0.0
 apscheduler
+aiohttp==3.9.1
+beautifulsoup4==4.12.2