This commit is contained in:
@@ -12,6 +12,9 @@ API_KEY = os.getenv('API_KEY')
|
|||||||
if not API_KEY:
|
if not API_KEY:
|
||||||
raise ValueError("API_KEY environment variable must be set")
|
raise ValueError("API_KEY environment variable must be set")
|
||||||
|
|
||||||
|
# Define custom user agent
|
||||||
|
CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
|
||||||
|
|
||||||
async def wait_for_network_idle(page):
|
async def wait_for_network_idle(page):
|
||||||
"""Wait until no network requests are in flight"""
|
"""Wait until no network requests are in flight"""
|
||||||
await page.waitForNetworkIdle(idleTime=500, timeout=30000)
|
await page.waitForNetworkIdle(idleTime=500, timeout=30000)
|
||||||
@@ -42,6 +45,9 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
|
|||||||
# Create new page
|
# Create new page
|
||||||
page = await browser.newPage()
|
page = await browser.newPage()
|
||||||
|
|
||||||
|
# Set custom user agent
|
||||||
|
await page.setUserAgent(CUSTOM_USER_AGENT)
|
||||||
|
|
||||||
# Navigate to URL and wait for network idle
|
# Navigate to URL and wait for network idle
|
||||||
await page.goto(decoded_url, waitUntil='networkidle2')
|
await page.goto(decoded_url, waitUntil='networkidle2')
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user