diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index db70ec1..8f39baf 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -2,7 +2,9 @@ from fastapi import FastAPI, HTTPException, Header from pyppeteer import launch import os import asyncio -from typing import Optional +import json +import re +from typing import Optional, Dict, List, Any from urllib.parse import unquote app = FastAPI() @@ -67,6 +69,179 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): if browser: await browser.close() +@app.get("/seo") +async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)): + """Extract SEO information from a website""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + browser = None + try: + # Decode URL if it's encoded + decoded_url = unquote(url) + print(f"Extracting SEO from: {decoded_url}") + + # Launch browser using installed Chrome + browser = await launch( + headless=True, + executablePath='/usr/bin/google-chrome', + args=['--no-sandbox', '--disable-setuid-sandbox'] + ) + + # Create new page + page = await browser.newPage() + + # Set custom user agent + await page.setUserAgent(CUSTOM_USER_AGENT) + + # Navigate to URL and wait for network idle + await page.goto(decoded_url, waitUntil='networkidle2') + + # Extract JSON-LD data + json_ld_data = await page.evaluate('''() => { + const jsonLdScripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); + return jsonLdScripts.map(script => { + try { + return JSON.parse(script.textContent); + } catch (e) { + return { error: "Failed to parse JSON-LD", content: script.textContent }; + } + }); + }''') + + # Extract meta tags + meta_tags = await page.evaluate('''() => { + const metaTags = Array.from(document.querySelectorAll('meta')); + return metaTags.map(tag => { + const attributes = Array.from(tag.attributes); + const result = {}; + attributes.forEach(attr => { + result[attr.name] = attr.value; + }); + return result; + }); + }''') + + # Extract title + title = await page.evaluate('document.title') + + # Extract canonical URL + canonical_url = await page.evaluate('''() => { + const link = document.querySelector('link[rel="canonical"]'); + return link ? link.href : null; + }''') + + # Extract Open Graph data + og_data = await page.evaluate('''() => { + const ogTags = Array.from(document.querySelectorAll('meta[property^="og:"]')); + const result = {}; + ogTags.forEach(tag => { + const property = tag.getAttribute('property'); + const content = tag.getAttribute('content'); + result[property] = content; + }); + return result; + }''') + + # Extract Twitter Card data + twitter_data = await page.evaluate('''() => { + const twitterTags = Array.from(document.querySelectorAll('meta[name^="twitter:"]')); + const result = {}; + twitterTags.forEach(tag => { + const name = tag.getAttribute('name'); + const content = tag.getAttribute('content'); + result[name] = content; + }); + return result; + }''') + + # Close page + await page.close() + + # Compile all SEO data + seo_data = { + "title": title, + "canonical_url": canonical_url, + "json_ld": json_ld_data, + "meta_tags": meta_tags, + "open_graph": og_data, + "twitter_card": twitter_data + } + + return {"status": "success", "seo_data": seo_data} + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + finally: + # Ensure browser is closed even if an error occurs + if browser: + await browser.close() + +@app.get("/meta") +async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)): + """Extract meta tags from a website""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + browser = None + try: + # Decode URL if it's encoded + decoded_url = unquote(url) + print(f"Extracting meta tags from: {decoded_url}") + + # Launch browser using installed Chrome + browser = await launch( + headless=True, + executablePath='/usr/bin/google-chrome', + args=['--no-sandbox', '--disable-setuid-sandbox'] + ) + + # Create new page + page = await browser.newPage() + + # Set custom user agent + await page.setUserAgent(CUSTOM_USER_AGENT) + + # Navigate to URL and wait for network idle + await page.goto(decoded_url, waitUntil='networkidle2') + + # Extract meta tags + meta_tags = await page.evaluate('''() => { + const metaTags = Array.from(document.querySelectorAll('meta')); + return metaTags.map(tag => { + const attributes = Array.from(tag.attributes); + const result = {}; + attributes.forEach(attr => { + result[attr.name] = attr.value; + }); + return result; + }); + }''') + + # Extract title + title = await page.evaluate('document.title') + + # Close page + await page.close() + + return { + "status": "success", + "url": decoded_url, + "title": title, + "meta_tags": meta_tags + } + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + finally: + # Ensure browser is closed even if an error occurs + if browser: + await browser.close() + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000)