improve scrolling logic in interactions_service to handle viewport metrics more reliably and expand viewport for full-page interactions

Dockers/puppeteer-api/app/services/browser.py

@@ -219,10 +219,38 @@ async def interactions_service(
                     return x, y
 
                 # Otherwise treat as document coordinates and auto-scroll to bring it into view.
-                # Scroll so the point lands near the center of the viewport for stability.
+                # We may need to iterate because some pages clamp/adjust scroll requests.
                 if inner_w and inner_h:
-                    target_scroll_x = max(0.0, x - (inner_w / 2.0))
+                    for _ in range(6):
-                    target_scroll_y = max(0.0, y - (inner_h / 2.0))
+                        metrics = await page.evaluate(
+                            "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY, scrollW: Math.max(document.body?.scrollWidth||0, document.documentElement?.scrollWidth||0), scrollH: Math.max(document.body?.scrollHeight||0, document.documentElement?.scrollHeight||0) })"
+                        )
+                        inner_w = float(metrics.get("innerWidth") or inner_w or 0)
+                        inner_h = float(metrics.get("innerHeight") or inner_h or 0)
+                        cur_scroll_x = float(metrics.get("scrollX") or 0)
+                        cur_scroll_y = float(metrics.get("scrollY") or 0)
+                        scroll_w = float(metrics.get("scrollW") or 0)
+                        scroll_h = float(metrics.get("scrollH") or 0)
+
+                        max_scroll_x = max(0.0, scroll_w - inner_w) if inner_w else 0.0
+                        max_scroll_y = max(0.0, scroll_h - inner_h) if inner_h else 0.0
+
+                        vx = x - cur_scroll_x
+                        vy = y - cur_scroll_y
+
+                        # If the mapped viewport point is inside the viewport, we're done.
+                        if (0.0 <= vx <= max(inner_w - 1.0, 0.0)) and (0.0 <= vy <= max(inner_h - 1.0, 0.0)):
+                            # Avoid exact-edge coordinates which sometimes hit nothing (scrollbars).
+                            vx = min(max(vx, 0.5), max(inner_w - 1.5, 0.5))
+                            vy = min(max(vy, 0.5), max(inner_h - 1.5, 0.5))
+                            return vx, vy
+
+                        # Scroll so the point lands near the center of the viewport.
+                        target_scroll_x = x - (inner_w / 2.0)
+                        target_scroll_y = y - (inner_h / 2.0)
+
+                        target_scroll_x = min(max(target_scroll_x, 0.0), max_scroll_x)
+                        target_scroll_y = min(max(target_scroll_y, 0.0), max_scroll_y)
 
                         await page.evaluate(
                             """({ sx, sy }) => {
@@ -231,23 +259,10 @@ async def interactions_service(
                             {"sx": target_scroll_x, "sy": target_scroll_y},
                         )
 
-                    # Wait for scroll offsets to settle before resolving the viewport point.
+                        # Give the browser a moment to reflow after scroll.
-                    try:
-                        await page.wait_for_function(
-                            """({ sx, sy }) => Math.abs(window.scrollX - sx) <= 2 && Math.abs(window.scrollY - sy) <= 2""",
-                            {"sx": target_scroll_x, "sy": target_scroll_y},
-                            timeout=10000,
-                        )
-                    except Exception:
-                        pass
                         await page.wait_for_timeout(150)
 
-                    vp2 = await page.evaluate(
+                # If we couldn't get reliable viewport metrics, fall back to best-effort conversion.
-                        "() => ({ innerWidth: window.innerWidth, innerHeight: window.innerHeight, scrollX: window.scrollX, scrollY: window.scrollY })"
-                    )
-                    cur_scroll_x = float(vp2.get("scrollX") or 0)
-                    cur_scroll_y = float(vp2.get("scrollY") or 0)
-
                 vx = x - cur_scroll_x
                 vy = y - cur_scroll_y
                 return vx, vy
@@ -358,6 +373,30 @@ async def interactions_service(
                 except Exception as e:
                     print(f"Error applying initial scroll position: {e}")
 
+            # If the caller requested full-page via viewport (viewport.height = -1), do the
+            # viewport expansion *now* so coordinate-based interactions can use viewport
+            # coordinates without requiring scrolling.
+            if full_page_via_viewport:
+                try:
+                    max_vp_h = 16384
+
+                    # Expand viewport height to the document height (capped), and reset scroll.
+                    await page.evaluate("() => window.scrollTo({ left: 0, top: 0, behavior: 'auto' })")
+                    await page.wait_for_timeout(250)
+
+                    full_h = await page.evaluate(
+                        "() => Math.max(document.body?.scrollHeight || 0, document.documentElement?.scrollHeight || 0)"
+                    )
+                    target_h = int(min(max(int(full_h or 0), 1), max_vp_h))
+
+                    cur_vp = page.viewport_size or {"width": 1280, "height": 720}
+                    target_w = int(requested_viewport_width or cur_vp.get("width", 1280))
+
+                    await page.set_viewport_size({"width": target_w, "height": target_h})
+                    await page.wait_for_timeout(500)
+                except Exception as e:
+                    print(f"Error resizing viewport for full-page interactions: {e}")
+
             for interaction in interactions:
                 # Support both Pydantic model instances and raw dicts.
                 if isinstance(interaction, dict):