diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index e9285b0..979529b 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -1,7 +1,5 @@ from app.utils.browser_utils import safe_browser_operation -from app.utils.http_utils import fetch_url import asyncio -from bs4 import BeautifulSoup from urllib.parse import unquote async def visit_url_service(decoded_url): @@ -103,48 +101,59 @@ async def extract_meta_tags_service(decoded_url): """Service function to extract meta tags from a website""" print(f"Extracting meta tags from: {decoded_url}") - try: - # Fetch the page content using plain HTTP request - content, status = await fetch_url(decoded_url) + # Define the operation to perform with the browser + async def meta_tags_operation(page): + try: + response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000) - # Parse the HTML - soup = BeautifulSoup(content, 'html.parser') + # Extract meta tags using JavaScript + meta_data = await page.evaluate('''() => { + const data = { + meta_tags: [], + open_graph: {}, + twitter_card: {}, + title: document.title || '' + }; - # Extract all meta tags - meta_tags = [] - for meta in soup.find_all('meta'): - attributes = {} - for attr in meta.attrs: - attributes[attr] = meta[attr] - meta_tags.append(attributes) + // Extract all meta tags + document.querySelectorAll('meta').forEach(meta => { + const attributes = {}; + for (let attr of meta.attributes) { + attributes[attr.name] = attr.value; + } + data.meta_tags.push(attributes); + }); - # Extract Open Graph tags - og_tags = {} - for meta in soup.find_all('meta', property=lambda x: x and x.startswith('og:')): - og_tags[meta.get('property')] = meta.get('content') + // Extract Open Graph tags + document.querySelectorAll('meta[property^="og:"]').forEach(meta => { + data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || ''; + }); - # Extract Twitter card tags - twitter_tags = {} - for meta in soup.find_all('meta', attrs={'name': lambda x: x and x.startswith('twitter:')}): - twitter_tags[meta.get('name')] = meta.get('content') + // Extract Twitter card tags + document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => { + data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || ''; + }); - # Get page title - title = soup.title.string if soup.title else '' + return data; + }''') - result = { - "status": "success", - "url": decoded_url, - "meta_tags": meta_tags, - "open_graph": og_tags, - "twitter_card": twitter_tags, - "title": title - } + result = { + "status": "success", + "url": decoded_url, + "meta_tags": meta_data["meta_tags"], + "open_graph": meta_data["open_graph"], + "twitter_card": meta_data["twitter_card"], + "title": meta_data["title"] + } - return result + return result - except Exception as e: - print(f"Error during meta tag extraction: {e}") - return {"status": "error", "url": decoded_url, "error": str(e)} + except Exception as e: + print(f"Error during meta tag extraction: {e}") + return {"status": "error", "url": decoded_url, "error": str(e)} + + # Perform the operation + return await safe_browser_operation(decoded_url, meta_tags_operation) async def capture_outgoing_calls_service(decoded_url): """Service function to capture outgoing API calls from a website"""