From e2cfcfad34261300c9487eff60bc60843174e909 Mon Sep 17 00:00:00 2001
From: Bram <info@bramkelchtermans.be>
Date: Thu, 5 Jun 2025 14:05:38 +0200
Subject: [PATCH] lightweight meta tags

---
 Dockers/puppeteer-api/app/services/browser.py | 85 +++++++++----------
 Dockers/puppeteer-api/app/utils/http_utils.py | 15 ++++
 Dockers/puppeteer-api/requirements.txt        |  4 +-
 3 files changed, 56 insertions(+), 48 deletions(-)
 create mode 100644 Dockers/puppeteer-api/app/utils/http_utils.py

diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py
index a805e70..ff0cd96 100644
--- a/Dockers/puppeteer-api/app/services/browser.py
+++ b/Dockers/puppeteer-api/app/services/browser.py
@@ -1,5 +1,8 @@
 from app.utils.browser_utils import safe_browser_operation
+from app.utils.http_utils import fetch_url
 import asyncio
+from bs4 import BeautifulSoup
+
 async def visit_url_service(decoded_url):
     """Service function to visit a URL and get its content"""
     print(f"Visiting URL: {decoded_url}")
@@ -99,60 +102,48 @@ async def extract_meta_tags_service(decoded_url):
     """Service function to extract meta tags from a website"""
     print(f"Extracting meta tags from: {decoded_url}")
 
-    # Define the operation to perform with the browser
-    async def meta_operation(page):
-        try:
-            response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+    try:
+        # Fetch the page content using plain HTTP request
+        content, status = await fetch_url(decoded_url)
 
-            # Extract all meta tags
-            meta_tags = await page.evaluate('''() => {
-                const metas = Array.from(document.querySelectorAll('meta'));
-                return metas.map(meta => {
-                    const attributes = {};
-                    Array.from(meta.attributes).forEach(attr => {
-                        attributes[attr.name] = attr.value;
-                    });
-                    return attributes;
-                });
-            }''')
+        # Parse the HTML
+        soup = BeautifulSoup(content, 'html.parser')
 
-            # Extract Open Graph tags
-            og_tags = await page.evaluate('''() => {
-                const ogTags = {};
-                document.querySelectorAll('meta[property^="og:"]').forEach(tag => {
-                    const property = tag.getAttribute('property');
-                    ogTags[property] = tag.getAttribute('content');
-                });
-                return ogTags;
-            }''')
+        # Extract all meta tags
+        meta_tags = []
+        for meta in soup.find_all('meta'):
+            attributes = {}
+            for attr in meta.attrs:
+                attributes[attr] = meta[attr]
+            meta_tags.append(attributes)
 
-            # Extract Twitter card tags
-            twitter_tags = await page.evaluate('''() => {
-                const twitterTags = {};
-                document.querySelectorAll('meta[name^="twitter:"]').forEach(tag => {
-                    const name = tag.getAttribute('name');
-                    twitterTags[name] = tag.getAttribute('content');
-                });
-                return twitterTags;
-            }''')
+        # Extract Open Graph tags
+        og_tags = {}
+        for meta in soup.find_all('meta', property=lambda x: x and x.startswith('og:')):
+            og_tags[meta.get('property')] = meta.get('content')
 
-            result = {
-                "status": "success",
-                "url": decoded_url,
-                "meta_tags": meta_tags,
-                "open_graph": og_tags,
-                "twitter_card": twitter_tags,
-                "title": await page.title()
-            }
+        # Extract Twitter card tags
+        twitter_tags = {}
+        for meta in soup.find_all('meta', attrs={'name': lambda x: x and x.startswith('twitter:')}):
+            twitter_tags[meta.get('name')] = meta.get('content')
 
-            return result
+        # Get page title
+        title = soup.title.string if soup.title else ''
 
-        except Exception as e:
-            print(f"Error during meta tag extraction: {e}")
-            return {"status": "error", "url": decoded_url, "error": str(e)}
+        result = {
+            "status": "success",
+            "url": decoded_url,
+            "meta_tags": meta_tags,
+            "open_graph": og_tags,
+            "twitter_card": twitter_tags,
+            "title": title
+        }
 
-    # Perform the operation
-    return await safe_browser_operation(decoded_url, meta_operation)
+        return result
+
+    except Exception as e:
+        print(f"Error during meta tag extraction: {e}")
+        return {"status": "error", "url": decoded_url, "error": str(e)}
 
 async def detect_pagination_service(decoded_url):
     """Service function to detect pagination on a website"""
diff --git a/Dockers/puppeteer-api/app/utils/http_utils.py b/Dockers/puppeteer-api/app/utils/http_utils.py
new file mode 100644
index 0000000..ae281a6
--- /dev/null
+++ b/Dockers/puppeteer-api/app/utils/http_utils.py
@@ -0,0 +1,15 @@
+from aiohttp import ClientSession, ClientTimeout
+from app.config import CUSTOM_USER_AGENT
+
+async def fetch_url(url: str) -> tuple[str, int]:
+    """
+    Fetch a URL using aiohttp and return the content and status code
+    """
+    timeout = ClientTimeout(total=30)  # 30 second timeout
+    async with ClientSession(timeout=timeout) as session:
+        headers = {
+            'User-Agent': CUSTOM_USER_AGENT
+        }
+        async with session.get(url, headers=headers) as response:
+            content = await response.text()
+            return content, response.status
\ No newline at end of file
diff --git a/Dockers/puppeteer-api/requirements.txt b/Dockers/puppeteer-api/requirements.txt
index c876748..df34eee 100644
--- a/Dockers/puppeteer-api/requirements.txt
+++ b/Dockers/puppeteer-api/requirements.txt
@@ -2,4 +2,6 @@ fastapi==0.68.1
 uvicorn==0.15.0
 pyppeteer==1.0.2
 psutil==6.0.0
-apscheduler
\ No newline at end of file
+apscheduler
+aiohttp==3.9.1
+beautifulsoup4==4.12.2
\ No newline at end of file