slopslopslop

Dockers/puppeteer-api/app/services/browser.py

@@ -188,6 +188,60 @@ async def interactions_service(
 
     async def interactions_operation(page):
         try:
+            async def resolve_viewport_point(px: int, py: int) -> tuple[float, float]:
+                """
+                Playwright's `document.elementFromPoint(x, y)` uses viewport coordinates.
+                Heuristic: if the provided coords clearly exceed the viewport, assume they are
+                document coordinates and convert by subtracting current scroll offsets.
+                """
+                vp = await page.evaluate(
+                    "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY })"
+                )
+                inner_w = vp.get("innerWidth", 0)
+                inner_h = vp.get("innerHeight", 0)
+                cur_scroll_x = vp.get("scrollX", 0)
+                cur_scroll_y = vp.get("scrollY", 0)
+
+                x = float(px)
+                y = float(py)
+
+                if inner_h and y > inner_h + 50:
+                    y = y - float(cur_scroll_y)
+                if inner_w and x > inner_w + 50:
+                    x = x - float(cur_scroll_x)
+
+                return x, y
+
+            async def element_from_point_retry(vx: float, vy: float, attempts: int = 10):
+                """Retry `elementFromPoint` to avoid timing issues around scroll/render."""
+                last_error = None
+                for _ in range(attempts):
+                    handle = None
+                    try:
+                        handle = await page.evaluate_handle(
+                            "(x, y) => document.elementFromPoint(x, y)",
+                            vx,
+                            vy,
+                        )
+                        element = handle.as_element()
+                        if element is not None:
+                            saved_handle = handle
+                            # Prevent `finally` from disposing the backing JSHandle;
+                            # the caller will dispose after clicking/focusing.
+                            handle = None
+                            return element, saved_handle
+                        last_error = f"No element found at viewport point ({vx}, {vy})"
+                    except Exception as e:
+                        last_error = str(e)
+                    finally:
+                        try:
+                            if handle is not None:
+                                await handle.dispose()
+                        except Exception:
+                            pass
+                    await page.wait_for_timeout(100)
+                raise ValueError(last_error or f"No element found at viewport point ({vx}, {vy})")
+
             # Configure viewport if provided (before navigation).
             if viewport is not None:
                 try:
@@ -214,10 +268,29 @@ async def interactions_service(
             if scroll_x is not None or scroll_y is not None:
                 try:
                     await page.evaluate(
-                        "(x, y) => { window.scrollTo(x ?? window.scrollX, y ?? window.scrollY); }",
+                        """(x, y) => {
-                        scroll_x,
+                            const left = (x === null ? window.scrollX : x);
-                        scroll_y,
+                            const top = (y === null ? window.scrollY : y);
+                            window.scrollTo({ left, top, behavior: 'auto' });
+                        }""",
+                        scroll_x if scroll_x is not None else None,
+                        scroll_y if scroll_y is not None else None,
                     )
+
+                    # Wait for scroll offsets to actually settle before doing coordinate clicks.
+                    if scroll_x is not None:
+                        await page.wait_for_function(
+                            "(x) => Math.abs(window.scrollX - x) <= 2",
+                            scroll_x,
+                            timeout=10000,
+                        )
+                    if scroll_y is not None:
+                        await page.wait_for_function(
+                            "(y) => Math.abs(window.scrollY - y) <= 2",
+                            scroll_y,
+                            timeout=10000,
+                        )
+                    await page.wait_for_timeout(300)
                 except Exception as e:
                     print(f"Error applying initial scroll position: {e}")
 
@@ -283,37 +356,38 @@ async def interactions_service(
                     if action == "click":
                         # Use DOM elementFromPoint so React/SPA components receive real click events.
                         try:
-                            handle = await page.evaluate_handle(
+                            vx, vy = await resolve_viewport_point(x, y)
-                                f"document.elementFromPoint({x}, {y})"
+                            element, handle = await element_from_point_retry(vx, vy)
-                            )
+                            try:
-                            element = handle.as_element()
+                                await element.click(force=True, timeout=30000)
-                            if element is None:
-                                raise ValueError(f"No element found at coordinates ({x}, {y})")
-                            await element.click()
                             finally:
                                 try:
                                     await handle.dispose()
                                 except Exception:
                                     pass
+                        except Exception as e:
+                            raise ValueError(f"Click via elementFromPoint failed at ({x}, {y}): {e}")
 
                     elif action == "type":
                         if text is None:
                             raise ValueError("Type interaction must contain text")
                         # Focus the element at the given coordinates, then type.
+                        vx, vy = await resolve_viewport_point(x, y)
+                        element, handle = await element_from_point_retry(vx, vy)
                         try:
-                            handle = await page.evaluate_handle(
+                            await element.click(force=True, timeout=30000)
-                                f"document.elementFromPoint({x}, {y})"
+                            # Ensure the click actually put focus on the intended control.
-                            )
+                            try:
-                            element = handle.as_element()
+                                await element.focus(timeout=30000)
-                            if element is None:
+                            except Exception:
-                                raise ValueError(f"No element found at coordinates ({x}, {y})")
+                                pass
-                            await element.click()
+                            await page.wait_for_timeout(100)
+                            await page.keyboard.type(text)
                         finally:
                             try:
                                 await handle.dispose()
                             except Exception:
                                 pass
-                        await page.keyboard.type(text)
 
                     else:
                         raise ValueError(f"Unknown interaction action: {action}")