diff --git a/Dockers/puppeteer-api/__pycache__/install_playwright.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/install_playwright.cpython-313.pyc new file mode 100644 index 0000000..5ab8dfb Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/install_playwright.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/__pycache__/main.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000..6f291ab Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/main.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/__pycache__/monitor.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/monitor.cpython-313.pyc new file mode 100644 index 0000000..a1d5ac8 Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/monitor.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/__pycache__/run.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/run.cpython-313.pyc new file mode 100644 index 0000000..1f3ab83 Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/run.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/__pycache__/test_cloudflare_bypass.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/test_cloudflare_bypass.cpython-313.pyc new file mode 100644 index 0000000..60a71b1 Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/test_cloudflare_bypass.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/__pycache__/test_database.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/test_database.cpython-313.pyc new file mode 100644 index 0000000..4898e21 Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/test_database.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/__pycache__/test_postgres_setup.cpython-313.pyc b/Dockers/puppeteer-api/__pycache__/test_postgres_setup.cpython-313.pyc new file mode 100644 index 0000000..ea29050 Binary files /dev/null and b/Dockers/puppeteer-api/__pycache__/test_postgres_setup.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/__pycache__/__init__.cpython-313.pyc b/Dockers/puppeteer-api/app/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..e0e2ba5 Binary files /dev/null and b/Dockers/puppeteer-api/app/__pycache__/__init__.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/__pycache__/config.cpython-313.pyc b/Dockers/puppeteer-api/app/__pycache__/config.cpython-313.pyc new file mode 100644 index 0000000..57ddc22 Binary files /dev/null and b/Dockers/puppeteer-api/app/__pycache__/config.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/__pycache__/database.cpython-313.pyc b/Dockers/puppeteer-api/app/__pycache__/database.cpython-313.pyc new file mode 100644 index 0000000..d53bbbb Binary files /dev/null and b/Dockers/puppeteer-api/app/__pycache__/database.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/__pycache__/main.cpython-313.pyc b/Dockers/puppeteer-api/app/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000..e23b098 Binary files /dev/null and b/Dockers/puppeteer-api/app/__pycache__/main.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/routes/__pycache__/__init__.cpython-313.pyc b/Dockers/puppeteer-api/app/routes/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..0988cd6 Binary files /dev/null and b/Dockers/puppeteer-api/app/routes/__pycache__/__init__.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/routes/__pycache__/browser.cpython-313.pyc b/Dockers/puppeteer-api/app/routes/__pycache__/browser.cpython-313.pyc index 22600c9..973ec9e 100644 Binary files a/Dockers/puppeteer-api/app/routes/__pycache__/browser.cpython-313.pyc and b/Dockers/puppeteer-api/app/routes/__pycache__/browser.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/routes/__pycache__/cache.cpython-313.pyc b/Dockers/puppeteer-api/app/routes/__pycache__/cache.cpython-313.pyc new file mode 100644 index 0000000..f33fef7 Binary files /dev/null and b/Dockers/puppeteer-api/app/routes/__pycache__/cache.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/routes/__pycache__/health.cpython-313.pyc b/Dockers/puppeteer-api/app/routes/__pycache__/health.cpython-313.pyc new file mode 100644 index 0000000..746b158 Binary files /dev/null and b/Dockers/puppeteer-api/app/routes/__pycache__/health.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/routes/browser.py b/Dockers/puppeteer-api/app/routes/browser.py index 987b1f6..98611a9 100644 --- a/Dockers/puppeteer-api/app/routes/browser.py +++ b/Dockers/puppeteer-api/app/routes/browser.py @@ -3,9 +3,11 @@ from typing import Optional from urllib.parse import unquote from app.config import API_KEY from app.database import get_cached_data, save_to_cache +from pydantic import BaseModel from app.services.browser import ( visit_url_service, screenshot_url_service, + click_selector_service, extract_seo_service, extract_meta_tags_service, capture_outgoing_calls_service, @@ -14,6 +16,12 @@ from app.services.browser import ( router = APIRouter() +class ClickRequest(BaseModel): + url: str + selector: str + # Default behavior: return screenshot after clicking. + returnType: Optional[str] = "screenshot" + @router.get("/") async def visit_url(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)): # Validate API key @@ -66,6 +74,36 @@ async def screenshot_url(url: str, fullPage: bool = True, x_api_key: Optional[st print(f"Error taking screenshot for URL {decoded_url}: {e}") raise HTTPException(status_code=500, detail=str(e)) +@router.post("/click") +async def click_selector(payload: ClickRequest, x_api_key: Optional[str] = Header(None)): + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + decoded_url = unquote(payload.url) + return_type = (payload.returnType or "screenshot").lower() + if return_type not in ("screenshot", "html"): + raise HTTPException(status_code=400, detail="Invalid returnType. Use 'screenshot' or 'html'.") + + try: + result = await click_selector_service(decoded_url, payload.selector, return_type) + + if isinstance(result, dict) and result.get("status") == "error": + raise HTTPException(status_code=500, detail=result.get("error", "Click failed")) + + if return_type == "html": + if not isinstance(result, str): + raise HTTPException(status_code=500, detail="Unexpected HTML response") + return Response(content=result, media_type="text/html; charset=utf-8") + + if not isinstance(result, (bytes, bytearray)): + raise HTTPException(status_code=500, detail="Unexpected screenshot response") + return Response(content=result, media_type="image/png") + except HTTPException: + raise + except Exception as e: + print(f"Error clicking selector on URL {decoded_url}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + @router.get("/seo") async def extract_seo(url: str, skipCache: bool = False, x_api_key: Optional[str] = Header(None)): """Extract SEO information from a website""" diff --git a/Dockers/puppeteer-api/app/services/__pycache__/__init__.cpython-313.pyc b/Dockers/puppeteer-api/app/services/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..b9172e4 Binary files /dev/null and b/Dockers/puppeteer-api/app/services/__pycache__/__init__.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/services/__pycache__/browser.cpython-313.pyc b/Dockers/puppeteer-api/app/services/__pycache__/browser.cpython-313.pyc index 165868a..769555b 100644 Binary files a/Dockers/puppeteer-api/app/services/__pycache__/browser.cpython-313.pyc and b/Dockers/puppeteer-api/app/services/__pycache__/browser.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/services/__pycache__/cache.cpython-313.pyc b/Dockers/puppeteer-api/app/services/__pycache__/cache.cpython-313.pyc new file mode 100644 index 0000000..5429473 Binary files /dev/null and b/Dockers/puppeteer-api/app/services/__pycache__/cache.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 9d8b247..306db86 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -119,6 +119,62 @@ async def screenshot_url_service(decoded_url, full_page=True): return await safe_browser_operation(decoded_url, screenshot_operation) +async def click_selector_service(decoded_url: str, selector: str, return_type: str): + """Navigate to a URL, click a CSS selector, and return PNG bytes or resulting HTML.""" + print(f"Clicking selector on {decoded_url}: {selector}") + + async def click_operation(page): + try: + # Navigate and wait for initial DOM. + await page.goto(decoded_url, wait_until='domcontentloaded', timeout=30000) + await page.wait_for_load_state('load', timeout=30000) + + # Ensure the page is fully loaded (helps with JS-driven UIs). + await page.wait_for_function("document.readyState === 'complete'", timeout=30000) + + # Best-effort cookie banner handling (helps avoid click interception). + cookie_buttons = [ + 'Accept all', + 'Accepteer', + 'Accepteren', + 'Accept' + ] + for button in cookie_buttons: + try: + element = await page.query_selector(f'text="{button}"') + if element: + await element.click() + await page.wait_for_timeout(1000) + break + except Exception: + # Ignore cookie/banner interaction errors; click may still work. + pass + + # Wait for the target selector, scroll it into view, then click. + await page.wait_for_selector(selector, timeout=30000) + locator = page.locator(selector) + await locator.scroll_into_view_if_needed(timeout=30000) + await locator.click(force=True, timeout=30000) + + # Give the page a moment to react to the click. + await page.wait_for_timeout(1000) + try: + await page.wait_for_load_state('networkidle', timeout=10000) + except Exception: + pass + + if return_type == "html": + return await page.content() + + # Return a full-page screenshot of the resulting view. + return await page.screenshot(full_page=True, type='png') + + except Exception as e: + print(f"Error during click capture: {e}") + return {"status": "error", "url": decoded_url, "error": str(e)} + + return await safe_browser_operation(decoded_url, click_operation) + async def extract_seo_service(decoded_url): """Service function to extract SEO information from a website""" print(f"Extracting SEO from: {decoded_url}") diff --git a/Dockers/puppeteer-api/app/utils/__pycache__/__init__.cpython-313.pyc b/Dockers/puppeteer-api/app/utils/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..f7f4d5f Binary files /dev/null and b/Dockers/puppeteer-api/app/utils/__pycache__/__init__.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/utils/__pycache__/browser_utils.cpython-313.pyc b/Dockers/puppeteer-api/app/utils/__pycache__/browser_utils.cpython-313.pyc new file mode 100644 index 0000000..e26f6e0 Binary files /dev/null and b/Dockers/puppeteer-api/app/utils/__pycache__/browser_utils.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/utils/__pycache__/cloudflare_bypass.cpython-313.pyc b/Dockers/puppeteer-api/app/utils/__pycache__/cloudflare_bypass.cpython-313.pyc new file mode 100644 index 0000000..6424e5e Binary files /dev/null and b/Dockers/puppeteer-api/app/utils/__pycache__/cloudflare_bypass.cpython-313.pyc differ diff --git a/Dockers/puppeteer-api/app/utils/__pycache__/http_utils.cpython-313.pyc b/Dockers/puppeteer-api/app/utils/__pycache__/http_utils.cpython-313.pyc new file mode 100644 index 0000000..c9c24f8 Binary files /dev/null and b/Dockers/puppeteer-api/app/utils/__pycache__/http_utils.cpython-313.pyc differ