enhance viewport coordinate handling in interactions_service to support document coordinates and improve scrolling stability

Dockers/puppeteer-api/app/services/browser.py

@@ -196,28 +196,64 @@ async def interactions_service(
 
             async def resolve_viewport_point(px: int, py: int) -> tuple[float, float]:
                 """
-                Playwright's `document.elementFromPoint(x, y)` uses viewport coordinates.
+                `document.elementFromPoint(x, y)` uses *viewport* (CSS pixel) coordinates.
-                Heuristic: if the provided coords clearly exceed the viewport, assume they are
-                document coordinates and convert by subtracting current scroll offsets.
-                """
-                vp = await page.evaluate(
-                    "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY })"
-                )
-                inner_w = vp.get("innerWidth", 0)
-                inner_h = vp.get("innerHeight", 0)
-                cur_scroll_x = vp.get("scrollX", 0)
-                cur_scroll_y = vp.get("scrollY", 0)
 
+                This API also supports *document* coordinates (e.g. from a full-page
+                screenshot). In that case we must scroll so the point is inside the viewport,
+                then convert to viewport coordinates by subtracting the current scroll offsets.
+                """
                 x = float(px)
                 y = float(py)
 
-                if inner_h and y > inner_h + 50:
+                vp = await page.evaluate(
-                    y = y - float(cur_scroll_y)
+                    "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY })"
-                if inner_w and x > inner_w + 50:
+                )
-                    x = x - float(cur_scroll_x)
+                inner_w = float(vp.get("innerWidth") or 0)
+                inner_h = float(vp.get("innerHeight") or 0)
+                cur_scroll_x = float(vp.get("scrollX") or 0)
+                cur_scroll_y = float(vp.get("scrollY") or 0)
 
+                # If the point is plausibly already a viewport coordinate, use it directly.
+                # (We allow a small slack to account for fractional/rounded coordinates.)
+                if inner_w and inner_h and (-5.0 <= x <= inner_w + 5.0) and (-5.0 <= y <= inner_h + 5.0):
                     return x, y
 
+                # Otherwise treat as document coordinates and auto-scroll to bring it into view.
+                # Scroll so the point lands near the center of the viewport for stability.
+                if inner_w and inner_h:
+                    target_scroll_x = max(0.0, x - (inner_w / 2.0))
+                    target_scroll_y = max(0.0, y - (inner_h / 2.0))
+
+                    await page.evaluate(
+                        """(sx, sy) => {
+                            window.scrollTo({ left: sx, top: sy, behavior: 'auto' });
+                        }""",
+                        target_scroll_x,
+                        target_scroll_y,
+                    )
+
+                    # Wait for scroll offsets to settle before resolving the viewport point.
+                    try:
+                        await page.wait_for_function(
+                            """(sx, sy) => Math.abs(window.scrollX - sx) <= 2 && Math.abs(window.scrollY - sy) <= 2""",
+                            target_scroll_x,
+                            target_scroll_y,
+                            timeout=10000,
+                        )
+                    except Exception:
+                        pass
+                    await page.wait_for_timeout(150)
+
+                    vp2 = await page.evaluate(
+                        "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY })"
+                    )
+                    cur_scroll_x = float(vp2.get("scrollX") or 0)
+                    cur_scroll_y = float(vp2.get("scrollY") or 0)
+
+                vx = x - cur_scroll_x
+                vy = y - cur_scroll_y
+                return vx, vy
+
             async def element_from_point_retry(vx: float, vy: float, attempts: int = 10):
                 """Retry `elementFromPoint` to avoid timing issues around scroll/render."""
                 last_error = None