diff --git a/Dockers/puppeteer-api/app/services/browser.py b/Dockers/puppeteer-api/app/services/browser.py index 2fc4533..670f61b 100644 --- a/Dockers/puppeteer-api/app/services/browser.py +++ b/Dockers/puppeteer-api/app/services/browser.py @@ -188,6 +188,60 @@ async def interactions_service( async def interactions_operation(page): try: + async def resolve_viewport_point(px: int, py: int) -> tuple[float, float]: + """ + Playwright's `document.elementFromPoint(x, y)` uses viewport coordinates. + Heuristic: if the provided coords clearly exceed the viewport, assume they are + document coordinates and convert by subtracting current scroll offsets. + """ + vp = await page.evaluate( + "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY })" + ) + inner_w = vp.get("innerWidth", 0) + inner_h = vp.get("innerHeight", 0) + cur_scroll_x = vp.get("scrollX", 0) + cur_scroll_y = vp.get("scrollY", 0) + + x = float(px) + y = float(py) + + if inner_h and y > inner_h + 50: + y = y - float(cur_scroll_y) + if inner_w and x > inner_w + 50: + x = x - float(cur_scroll_x) + + return x, y + + async def element_from_point_retry(vx: float, vy: float, attempts: int = 10): + """Retry `elementFromPoint` to avoid timing issues around scroll/render.""" + last_error = None + for _ in range(attempts): + handle = None + try: + handle = await page.evaluate_handle( + "(x, y) => document.elementFromPoint(x, y)", + vx, + vy, + ) + element = handle.as_element() + if element is not None: + saved_handle = handle + # Prevent `finally` from disposing the backing JSHandle; + # the caller will dispose after clicking/focusing. + handle = None + return element, saved_handle + last_error = f"No element found at viewport point ({vx}, {vy})" + except Exception as e: + last_error = str(e) + finally: + try: + if handle is not None: + await handle.dispose() + except Exception: + pass + await page.wait_for_timeout(100) + raise ValueError(last_error or f"No element found at viewport point ({vx}, {vy})") + # Configure viewport if provided (before navigation). if viewport is not None: try: @@ -214,10 +268,29 @@ async def interactions_service( if scroll_x is not None or scroll_y is not None: try: await page.evaluate( - "(x, y) => { window.scrollTo(x ?? window.scrollX, y ?? window.scrollY); }", - scroll_x, - scroll_y, + """(x, y) => { + const left = (x === null ? window.scrollX : x); + const top = (y === null ? window.scrollY : y); + window.scrollTo({ left, top, behavior: 'auto' }); + }""", + scroll_x if scroll_x is not None else None, + scroll_y if scroll_y is not None else None, ) + + # Wait for scroll offsets to actually settle before doing coordinate clicks. + if scroll_x is not None: + await page.wait_for_function( + "(x) => Math.abs(window.scrollX - x) <= 2", + scroll_x, + timeout=10000, + ) + if scroll_y is not None: + await page.wait_for_function( + "(y) => Math.abs(window.scrollY - y) <= 2", + scroll_y, + timeout=10000, + ) + await page.wait_for_timeout(300) except Exception as e: print(f"Error applying initial scroll position: {e}") @@ -283,37 +356,38 @@ async def interactions_service( if action == "click": # Use DOM elementFromPoint so React/SPA components receive real click events. try: - handle = await page.evaluate_handle( - f"document.elementFromPoint({x}, {y})" - ) - element = handle.as_element() - if element is None: - raise ValueError(f"No element found at coordinates ({x}, {y})") - await element.click() - finally: + vx, vy = await resolve_viewport_point(x, y) + element, handle = await element_from_point_retry(vx, vy) try: - await handle.dispose() - except Exception: - pass + await element.click(force=True, timeout=30000) + finally: + try: + await handle.dispose() + except Exception: + pass + except Exception as e: + raise ValueError(f"Click via elementFromPoint failed at ({x}, {y}): {e}") elif action == "type": if text is None: raise ValueError("Type interaction must contain text") # Focus the element at the given coordinates, then type. + vx, vy = await resolve_viewport_point(x, y) + element, handle = await element_from_point_retry(vx, vy) try: - handle = await page.evaluate_handle( - f"document.elementFromPoint({x}, {y})" - ) - element = handle.as_element() - if element is None: - raise ValueError(f"No element found at coordinates ({x}, {y})") - await element.click() + await element.click(force=True, timeout=30000) + # Ensure the click actually put focus on the intended control. + try: + await element.focus(timeout=30000) + except Exception: + pass + await page.wait_for_timeout(100) + await page.keyboard.type(text) finally: try: await handle.dispose() except Exception: pass - await page.keyboard.type(text) else: raise ValueError(f"Unknown interaction action: {action}")