replace puppeteer with playwright

Dockers/puppeteer-api/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.11-slim
 
 # Install required system dependencies
 RUN apt-get update && apt-get install -y \
@@ -6,11 +6,6 @@ RUN apt-get update && apt-get install -y \
     gnupg2 \
     procps \
     sqlite3 \
-    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/google-chrome-keyring.gpg \
-    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \
-    && apt-get update \
-    && apt-get install -y \
-    google-chrome-stable \
     fonts-ipafont-gothic \
     fonts-wqy-zenhei \
     fonts-thai-tlwg \
@@ -19,9 +14,7 @@ RUN apt-get update && apt-get install -y \
     libxss1 \
     xvfb \
     --no-install-recommends \
-    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /var/lib/apt/lists/*
-    # Verify Chrome installation
-    && google-chrome --version
 
 # Set working directory
 WORKDIR /app
@@ -30,6 +23,23 @@ WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
+# Install Playwright system dependencies as root
+RUN playwright install-deps chromium
+
+# Create a non-root user
+RUN useradd -m playwright
+
+# Set HOME for playwright user and ensure cache directory exists
+ENV HOME=/home/playwright
+RUN mkdir -p /home/playwright/.cache/ms-playwright && chown -R playwright:playwright /home/playwright
+
+# Install Playwright browsers as the playwright user
+USER playwright
+RUN playwright install chromium
+
+# Switch back to root for file operations
+USER root
+
 # Copy the rest of the application
 COPY . .
 
@@ -39,17 +49,19 @@ ENV API_KEY=""
 ENV CACHE_EXPIRY_HOURS=36
 ENV CLEANUP_CRON="0 3 * * *"
 
-# Create a non-root user and switch to it
+# Set ownership of the application to playwright user
-RUN useradd -m puppeteer && chown -R puppeteer:puppeteer /app
+RUN chown -R playwright:playwright /app
+
 # Create the database directory and set permissions
-RUN mkdir -p /db && chmod 777 /db && chown -R puppeteer:puppeteer /app
+RUN mkdir -p /db && chmod 777 /db && chown -R playwright:playwright /app
 
 # Add system limits configuration
 RUN echo "* soft nofile 65535" >> /etc/security/limits.conf && \
     echo "* hard nofile 65535" >> /etc/security/limits.conf && \
     echo "session required pam_limits.so" >> /etc/pam.d/common-session
 
-USER puppeteer
+# Switch to playwright user for running the application
+USER playwright
 
 # Expose port
 EXPOSE 8000

Dockers/puppeteer-api/MIGRATION_GUIDE.md

@@ -0,0 +1,344 @@
+# Migration Guide: Puppeteer to Playwright
+
+This guide will help you migrate from Puppeteer (pyppeteer) to Playwright in your project.
+
+## Overview
+
+The migration from Puppeteer to Playwright provides several benefits:
+
+- Better performance and stability
+- Enhanced features for modern web applications
+- Active development and community support
+- Multi-browser support (Chromium, Firefox, WebKit)
+
+## Pre-Migration Checklist
+
+Before starting the migration, ensure you have:
+
+- [ ] Backed up your current codebase
+- [ ] Documented any custom Puppeteer configurations
+- [ ] Identified all Puppeteer-specific code in your project
+- [ ] Tested your current application thoroughly
+
+## Step-by-Step Migration
+
+### 1. Update Dependencies
+
+Replace Puppeteer dependencies with Playwright:
+
+**Before (requirements.txt):**
+
+```
+pyppeteer==1.0.2
+```
+
+**After (requirements.txt):**
+
+```
+playwright==1.40.0
+```
+
+### 2. Install Playwright
+
+```bash
+# Install Playwright
+pip install playwright
+
+# Install browsers (Chromium is recommended for compatibility)
+playwright install chromium
+
+# Install system dependencies (Linux)
+playwright install-deps chromium
+```
+
+### 3. Update Imports
+
+**Before:**
+
+```python
+from pyppeteer import launch
+```
+
+**After:**
+
+```python
+from playwright.async_api import async_playwright
+```
+
+### 4. Update Browser Launch
+
+**Before (Puppeteer):**
+
+```python
+browser = await launch(
+    headless=True,
+    executablePath='/usr/bin/google-chrome',
+    args=['--no-sandbox', '--disable-setuid-sandbox'],
+    handleSIGINT=False,
+    handleSIGTERM=False,
+    handleSIGHUP=False,
+    ignoreHTTPSErrors=True,
+    autoClose=True,
+)
+```
+
+**After (Playwright):**
+
+```python
+playwright = await async_playwright().start()
+browser = await playwright.chromium.launch(
+    headless=True,
+    args=['--no-sandbox', '--disable-setuid-sandbox'],
+    ignore_default_args=['--enable-automation'],
+)
+```
+
+### 5. Update Page Creation
+
+**Before (Puppeteer):**
+
+```python
+page = await browser.newPage()
+await page.setUserAgent(CUSTOM_USER_AGENT)
+await page.setViewport({'width': 1280, 'height': 800})
+page.setDefaultNavigationTimeout(30000)
+```
+
+**After (Playwright):**
+
+```python
+context = await browser.new_context(
+    user_agent=CUSTOM_USER_AGENT,
+    viewport={'width': 1920, 'height': 1080},
+    ignore_https_errors=True,
+)
+page = await context.new_page()
+page.set_default_timeout(30000)
+```
+
+### 6. Update Navigation
+
+**Before (Puppeteer):**
+
+```python
+response = await page.goto(url, waitUntil='networkidle2', timeout=30000)
+```
+
+**After (Playwright):**
+
+```python
+response = await page.goto(url, wait_until='networkidle', timeout=30000)
+```
+
+### 7. Update Wait Methods
+
+**Before (Puppeteer):**
+
+```python
+await page.waitForNetworkIdle(idleTime=500, timeout=30000)
+await page.waitForNavigation(waitUntil='networkidle2', timeout=10000)
+```
+
+**After (Playwright):**
+
+```python
+await page.wait_for_load_state('networkidle', timeout=30000)
+await page.wait_for_load_state('networkidle', timeout=10000)
+```
+
+### 8. Update Request Interception
+
+**Before (Puppeteer):**
+
+```python
+await page.setRequestInterception(True)
+
+async def intercept(request):
+    if request.resourceType in ['image', 'media', 'font', 'stylesheet']:
+        await request.abort()
+    else:
+        await request.continue_()
+
+page.on('request', lambda req: asyncio.ensure_future(intercept(req)))
+```
+
+**After (Playwright):**
+
+```python
+await page.route("**/*", lambda route: route.abort()
+               if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
+               else route.continue_())
+```
+
+### 9. Update Browser Cleanup
+
+**Before (Puppeteer):**
+
+```python
+pages = await browser.pages()
+for page in pages:
+    await page.close()
+await browser.close()
+```
+
+**After (Playwright):**
+
+```python
+pages = browser.contexts[0].pages if browser.contexts else []
+for page in pages:
+    await page.close()
+await browser.close()
+```
+
+## Docker Migration
+
+### Update Dockerfile
+
+**Before:**
+
+```dockerfile
+# Install Chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/google-chrome-keyring.gpg \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \
+    && apt-get update \
+    && apt-get install -y google-chrome-stable
+```
+
+**After:**
+
+```dockerfile
+# Install Playwright browsers
+RUN playwright install chromium
+RUN playwright install-deps chromium
+```
+
+## Testing the Migration
+
+### 1. Run the Test Script
+
+```bash
+python test_playwright_migration.py
+```
+
+This script tests:
+
+- Playwright installation
+- Browser utilities
+- Browser service functionality
+
+### 2. Manual Testing
+
+Test your key functionality:
+
+- Basic page navigation
+- JavaScript execution
+- Screenshot capture
+- PDF generation (if used)
+
+### 3. Performance Testing
+
+Compare performance metrics:
+
+- Page load times
+- Memory usage
+- CPU usage
+- Browser startup time
+
+## Common Issues and Solutions
+
+### Issue: Browser Not Starting
+
+**Solution:** Ensure Playwright browsers are installed:
+
+```bash
+playwright install chromium
+```
+
+### Issue: Permission Errors
+
+**Solution:** Install system dependencies:
+
+```bash
+playwright install-deps chromium
+```
+
+### Issue: Page Navigation Fails
+
+**Solution:** Check timeout settings and network conditions:
+
+```python
+# Increase timeout if needed
+await page.goto(url, wait_until='networkidle', timeout=60000)
+```
+
+### Issue: Memory Leaks
+
+**Solution:** Ensure proper cleanup:
+
+```python
+# Always close context and browser
+await context.close()
+await browser.close()
+```
+
+## Performance Optimizations
+
+### 1. Browser Pool Management
+
+```python
+# Use browser pooling for better performance
+browser_pool = Queue(maxsize=MAX_BROWSERS)
+```
+
+### 2. Context Reuse
+
+```python
+# Reuse contexts when possible
+context = await browser.new_context()
+# Use context for multiple pages
+```
+
+### 3. Resource Blocking
+
+```python
+# Block unnecessary resources
+await page.route("**/*", lambda route: route.abort()
+               if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
+               else route.continue_())
+```
+
+## Rollback Plan
+
+If you need to rollback to Puppeteer:
+
+1. Restore your backup
+2. Revert requirements.txt changes
+3. Revert Dockerfile changes
+4. Test thoroughly
+
+## Post-Migration Checklist
+
+After completing the migration:
+
+- [ ] All tests pass
+- [ ] Performance is acceptable
+- [ ] No memory leaks detected
+- [ ] Error handling works correctly
+- [ ] Documentation is updated
+- [ ] Team is trained on new API
+
+## Support
+
+If you encounter issues during migration:
+
+1. Check the [Playwright documentation](https://playwright.dev/python/)
+2. Review the [migration guide](https://playwright.dev/python/docs/migrate-from-puppeteer)
+3. Search existing issues on GitHub
+4. Create a new issue with detailed information
+
+## Additional Resources
+
+- [Playwright Python Documentation](https://playwright.dev/python/)
+- [Migration Guide from Puppeteer](https://playwright.dev/python/docs/migrate-from-puppeteer)
+- [Playwright GitHub Repository](https://github.com/microsoft/playwright)
+- [Community Discord](https://discord.gg/playwright)

Dockers/puppeteer-api/README.md

@@ -1,6 +1,27 @@
-# Puppeteer API Server
+# Playwright API Server
 
-A FastAPI-based server that uses Puppeteer (via pyppeteer) to scrape websites and extract various types of data.
+A FastAPI-based server that uses Playwright to scrape websites and extract various types of data.
+
+## Recent Migration (v3.0)
+
+### Migration from Puppeteer to Playwright
+
+This project has been successfully migrated from Puppeteer (pyppeteer) to Playwright for better performance, reliability, and maintainability.
+
+#### Key Benefits of Playwright
+
+- **Better performance**: More efficient browser management and faster page loads
+- **Improved stability**: Better handling of modern web applications
+- **Enhanced features**: Better support for modern web standards
+- **Active development**: More frequent updates and better community support
+- **Multi-browser support**: Can easily switch between Chromium, Firefox, and WebKit
+
+#### Migration Changes
+
+- **Dependencies**: Updated from `pyppeteer` to `playwright`
+- **Browser management**: Improved browser pool with better resource management
+- **API compatibility**: All existing endpoints remain the same
+- **Docker image**: Updated to use Playwright browsers instead of Chrome
 
 ## Recent Fixes (v2.0)
 
@@ -102,19 +123,34 @@ python monitor.py your-api-key 30 http://localhost:8000 cleanup
 
 ```bash
 # Build the image
-docker build -t puppeteer-api .
+docker build -t playwright-api .
 
 # Run with environment variables
 docker run -d \
-  --name puppeteer-api \
+  --name playwright-api \
   -p 8000:8000 \
   -e API_KEY=your-api-key \
   -e MAX_BROWSERS=3 \
   -e BROWSER_TTL=1800 \
   -v /path/to/cache:/db \
-  puppeteer-api
+  playwright-api
 ```
 
+## Testing the Migration
+
+To verify that the Playwright migration works correctly:
+
+```bash
+# Run the migration test script
+python test_playwright_migration.py
+```
+
+This will test:
+
+- Playwright installation and basic functionality
+- Browser utilities module
+- Browser service functionality
+
 ## Troubleshooting
 
 ### High CPU Usage
@@ -135,6 +171,12 @@ docker run -d \
 2. Trigger emergency cleanup
 3. Restart the container if needed
 
+### Playwright Installation Issues
+
+1. Ensure Playwright browsers are installed: `playwright install chromium`
+2. Check Docker build logs for browser installation
+3. Verify system dependencies are installed
+
 ## Performance Tips
 
 1. **Use caching**: The API caches results for 36 hours by default

Dockers/puppeteer-api/app/services/browser.py

@@ -10,7 +10,7 @@ async def visit_url_service(decoded_url):
     # Define the operation to perform with the browser
     async def visit_operation(page):
         try:
-            response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
             if not response:
                 print(f"Warning: No response object returned for {decoded_url}")
 
@@ -36,7 +36,7 @@ async def extract_seo_service(decoded_url):
     # Define the operation to perform with the browser
     async def seo_operation(page):
         try:
-            response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+            response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
 
             # Extract SEO information
             seo_data = await page.evaluate('''() => {
@@ -154,7 +154,7 @@ async def detect_pagination_service(decoded_url):
         try:
             # Navigate to the URL
             try:
-                await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
             except Exception as e:
                 print(f"Error navigating to URL: {e}")
                 # Try to get the current URL even if navigation failed
@@ -433,7 +433,7 @@ async def detect_pagination_service(decoded_url):
                     if clicked:
                         print("Successfully clicked on pagination element")
                         # Wait for navigation to complete
-                        await page.waitForNavigation(waitUntil='networkidle2', timeout=10000)
+                        await page.wait_for_load_state('networkidle', timeout=10000)
                         next_page_url = page.url
                     else:
                         print("No clickable pagination element found")
@@ -447,8 +447,8 @@ async def detect_pagination_service(decoded_url):
                 print('Searching for pagination parameter')
                 # Parse both URLs
                 try:
-                    original_parsed = await page.evaluate(f'''(originalUrl) => {{
+                    original_parsed = await page.evaluate('''([originalUrl]) => {
-                        try {{
+                        try {
                             const original = new URL(originalUrl);
                             const current = new URL(window.location.href);
 
@@ -458,99 +458,99 @@ async def detect_pagination_service(decoded_url):
                             // Common pagination parameters to check
                             const paginationParams = ['page', 'p', 'pg', 'offset', 'o', 'from', 'start', 'limit', 'currentPage', 'current_page', 'currentpage', 'pagenum', 'pageNumber', 'paged'];
 
-                            for (const param of paginationParams) {{
+                            for (const param of paginationParams) {
                                 const originalValue = original.searchParams.get(param);
                                 const currentValue = current.searchParams.get(param);
 
-                                if (originalValue !== currentValue && currentValue !== null) {{
+                                if (originalValue !== currentValue && currentValue !== null) {
-                                    paramDiff = {{
+                                    paramDiff = {
                                         name: param,
                                         originalValue: originalValue,
                                         currentValue: currentValue
-                                    }};
+                                    };
                                     break;
-                                }}
+                                }
-                            }}
+                            }
 
                             // Check for path differences (like /page/1 vs /page/2 or /vacatures vs /vacatures/page/2)
                             const originalPath = original.pathname;
                             const currentPath = current.pathname;
 
                             let pathDiff = null;
-                            if (originalPath !== currentPath) {{
+                            if (originalPath !== currentPath) {
                                 const originalSegments = originalPath.split('/').filter(s => s);
                                 const currentSegments = currentPath.split('/').filter(s => s);
 
                                 // Case 1: Same number of segments - find the one that changed
-                                if (originalSegments.length === currentSegments.length) {{
+                                if (originalSegments.length === currentSegments.length) {
-                                    for (let i = 0; i < originalSegments.length; i++) {{
+                                    for (let i = 0; i < originalSegments.length; i++) {
-                                        if (originalSegments[i] !== currentSegments[i]) {{
+                                        if (originalSegments[i] !== currentSegments[i]) {
                                             // Check if the difference is numeric
-                                            if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {{
+                                            if (!isNaN(originalSegments[i]) && !isNaN(currentSegments[i])) {
-                                                pathDiff = {{
+                                                pathDiff = {
                                                     type: 'replace',
                                                     index: i,
                                                     originalValue: originalSegments[i],
                                                     currentValue: currentSegments[i]
-                                                }};
+                                                };
-                                            }}
+                                            }
-                                        }}
+                                        }
-                                    }}
+                                    }
-                                }}
+                                }
                                 // Case 2: Current path has more segments - check for added pagination segments
-                                else if (currentSegments.length > originalSegments.length) {{
+                                else if (currentSegments.length > originalSegments.length) {
                                     // Look for patterns like /page/NUMBER or /p/NUMBER at the end
                                     const pagePattern = /^(page|p)\/(\d+)$/i;
 
                                     // Check the last two segments of the current path
-                                    if (currentSegments.length >= 2) {{
+                                    if (currentSegments.length >= 2) {
                                         const lastTwoSegments = currentSegments.slice(-2).join('/');
                                         const match = lastTwoSegments.match(pagePattern);
 
-                                        if (match) {{
+                                        if (match) {
-                                            pathDiff = {{
+                                            pathDiff = {
                                                 type: 'append',
                                                 pageSegment: match[1], // 'page' or 'p'
                                                 pageNumber: match[2],  // the actual number
                                                 originalSegments: originalSegments,
                                                 currentSegments: currentSegments
-                                            }};
+                                            };
-                                        }}
+                                        }
-                                    }}
+                                    }
 
                                     // If no pattern match, check if the last segment is numeric
-                                    if (!pathDiff && currentSegments.length > 0) {{
+                                    if (!pathDiff && currentSegments.length > 0) {
                                         const lastSegment = currentSegments[currentSegments.length - 1];
-                                        if (!isNaN(lastSegment)) {{
+                                        if (!isNaN(lastSegment)) {
-                                            pathDiff = {{
+                                            pathDiff = {
                                                 type: 'append',
                                                 pageSegment: null,
                                                 pageNumber: lastSegment,
                                                 originalSegments: originalSegments,
                                                 currentSegments: currentSegments
-                                            }};
+                                            };
-                                        }}
+                                        }
-                                    }}
+                                    }
-                                }}
+                                }
-                            }}
+                            }
 
-                            return {{
+                            return {
                                 paramDiff,
                                 pathDiff,
                                 originalUrl: originalUrl,
                                 currentUrl: window.location.href
-                            }};
+                            };
-                        }} catch (error) {{
+                        } catch (error) {
                             console.error("Error during URL analysis:", error);
-                            return {{
+                            return {
                                 paramDiff: null,
                                 pathDiff: null,
                                 originalUrl: originalUrl,
                                 currentUrl: window.location.href,
                                 error: error.message
-                            }};
+                            };
-                        }}
+                        }
-                    }}''', original_url)
+                    }''', [original_url])
                 except Exception as e:
                     print(f"Error during URL analysis: {e}")
                     original_parsed = {
@@ -585,16 +585,16 @@ async def detect_pagination_service(decoded_url):
 
                     # Create URL template for query parameter
                     try:
-                        url_obj = await page.evaluate(f'''(url, paramName) => {{
+                        url_obj = await page.evaluate('''([url, paramName]) => {
-                            try {{
+                            try {
                                 const urlObj = new URL(url);
-                                urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
+                                urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
                                 return urlObj.toString();
-                            }} catch (error) {{
+                            } catch (error) {
                                 console.error("Error creating URL template:", error);
                                 return null;
-                            }}
+                            }
-                        }}''', original_url, param_name)
+                        }''', [original_url, param_name])
 
                         url_template = url_obj
                     except Exception as e:
@@ -626,18 +626,18 @@ async def detect_pagination_service(decoded_url):
 
                         # Create URL template for path parameter replacement
                         try:
-                            url_template = await page.evaluate(f'''(url, pathIndex) => {{
+                            url_template = await page.evaluate('''([url, pathIndex]) => {
-                                try {{
+                                try {
                                     const urlObj = new URL(url);
                                     const pathSegments = urlObj.pathname.split('/').filter(s => s);
-                                    pathSegments[pathIndex] = "{{PAGE_NUMBER}}";
+                                    pathSegments[pathIndex] = "{PAGE_NUMBER}";
                                     urlObj.pathname = '/' + pathSegments.join('/');
                                     return urlObj.toString();
-                                }} catch (error) {{
+                                } catch (error) {
                                     console.error("Error creating path URL template:", error);
                                     return null;
-                                }}
+                                }
-                            }}''', original_url, path_index)
+                            }''', [original_url, path_index])
                         except Exception as e:
                             print(f"Error creating URL template for path parameter: {e}")
                             url_template = None
@@ -662,30 +662,30 @@ async def detect_pagination_service(decoded_url):
 
                         # Create URL template for appended path parameter
                         try:
-                            url_template = await page.evaluate(f'''(url, pageSegment) => {{
+                            url_template = await page.evaluate('''([url, pageSegment]) => {
-                                try {{
+                                try {
                                     const urlObj = new URL(url);
                                     let newPath = urlObj.pathname;
 
                                     // Remove trailing slash if present
-                                    if (newPath.endsWith('/')) {{
+                                    if (newPath.endsWith('/')) {
                                         newPath = newPath.slice(0, -1);
-                                    }}
+                                    }
 
                                     // Append the pagination segment
-                                    if (pageSegment) {{
+                                    if (pageSegment) {
-                                        newPath += '/' + pageSegment + '/{{PAGE_NUMBER}}';
+                                        newPath += '/' + pageSegment + '/{PAGE_NUMBER}';
-                                    }} else {{
+                                    } else {
-                                        newPath += '/{{PAGE_NUMBER}}';
+                                        newPath += '/{PAGE_NUMBER}';
-                                    }}
+                                    }
 
                                     urlObj.pathname = newPath;
                                     return urlObj.toString();
-                                }} catch (error) {{
+                                } catch (error) {
                                     console.error("Error creating appended path URL template:", error);
                                     return null;
-                                }}
+                                }
-                            }}''', original_url, path_diff.get('pageSegment'))
+                            }''', [original_url, path_diff.get('pageSegment')])
                         except Exception as e:
                             print(f"Error creating URL template for appended path parameter: {e}")
                             url_template = None
@@ -699,16 +699,16 @@ async def detect_pagination_service(decoded_url):
             if not url_template and pagination_data['detectedParameter']:
                 param_name = pagination_data['detectedParameter']['name']
                 try:
-                    url_template = await page.evaluate(f'''(url, paramName) => {{
+                    url_template = await page.evaluate('''([url, paramName]) => {
-                        try {{
+                        try {
                             const urlObj = new URL(url);
-                            urlObj.searchParams.set(paramName, "{{PAGE_NUMBER}}");
+                            urlObj.searchParams.set(paramName, "{PAGE_NUMBER}");
                             return urlObj.toString();
-                        }} catch (error) {{
+                        } catch (error) {
                             console.error("Error creating inferred URL template:", error);
                             return null;
-                        }}
+                        }
-                    }}''', original_url, param_name)
+                    }''', [original_url, param_name])
                 except Exception as e:
                     print(f"Error creating inferred URL template: {e}")
                     url_template = None
@@ -716,7 +716,7 @@ async def detect_pagination_service(decoded_url):
             # If still no template and we have pagination elements, try to infer from the current URL structure
             if not url_template and pagination_data['hasPagination']:
                 try:
-                    url_template = await page.evaluate('''(originalUrl) => {
+                    url_template = await page.evaluate('''([originalUrl]) => {
                         try {
                             const urlObj = new URL(originalUrl);
                             let path = urlObj.pathname;
@@ -730,10 +730,10 @@ async def detect_pagination_service(decoded_url):
                             const pagePattern = /\/(page|p)\/\d+$/i;
                             if (pagePattern.test(path)) {
                                 // Replace the existing page number with placeholder
-                                path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{{PAGE_NUMBER}}');
+                                path = path.replace(/\/(page|p)\/\d+$/i, '/$1/{PAGE_NUMBER}');
                             } else {
                                 // Add pagination pattern
-                                path += '/page/{{PAGE_NUMBER}}';
+                                path += '/page/{PAGE_NUMBER}';
                             }
 
                             urlObj.pathname = path;
@@ -742,7 +742,7 @@ async def detect_pagination_service(decoded_url):
                             console.error("Error creating fallback URL template:", error);
                             return null;
                         }
-                    }''', original_url)
+                    }''', [original_url])
                 except Exception as e:
                     print(f"Error creating fallback URL template: {e}")
                     url_template = None

Dockers/puppeteer-api/app/utils/browser_utils.py

@@ -1,31 +1,34 @@
-from pyppeteer import launch
+from playwright.async_api import async_playwright
 from app.config import CUSTOM_USER_AGENT
 import asyncio
 
 async def wait_for_network_idle(page):
     """Wait until no network requests are in flight"""
-    await page.waitForNetworkIdle(idleTime=500, timeout=30000)
+    await page.wait_for_load_state('networkidle')
 
 async def safe_browser_operation(url, operation_func):
     """Safely perform browser operations with proper cleanup"""
     browser = None
+    context = None
     page = None
     try:
-        browser = await launch(
+        # Get or create playwright instance
+        playwright = await async_playwright().start()
+
+        browser = await playwright.chromium.launch(
             headless=True,
-            executablePath='/usr/bin/google-chrome',
             args=['--no-sandbox', '--disable-setuid-sandbox'],
-            handleSIGINT=False,
-            handleSIGTERM=False,
-            handleSIGHUP=False
         )
 
-        # Create new page with timeout
+        # Create context and page
-        page = await browser.newPage()
+        context = await browser.new_context(
-        page.setDefaultNavigationTimeout(30000)
+            user_agent=CUSTOM_USER_AGENT,
+            viewport={'width': 1920, 'height': 1080},
+            ignore_https_errors=True,
+        )
 
-        # Set custom user agent
+        page = await context.new_page()
-        await page.setUserAgent(CUSTOM_USER_AGENT)
+        page.set_default_timeout(30000)
 
         # Call the operation function that uses the page
         result = await operation_func(page)
@@ -48,6 +51,13 @@ async def safe_browser_operation(url, operation_func):
             except Exception as e:
                 print(f"Error closing page: {e}")
 
+        # Ensure context is closed properly
+        if context:
+            try:
+                await context.close()
+            except Exception as e:
+                print(f"Error closing context: {e}")
+
         # Ensure browser is closed properly
         if browser:
             try:

Dockers/puppeteer-api/install_playwright.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+Installation script for Playwright migration
+"""
+
+import subprocess
+import sys
+import os
+
+def run_command(command, description):
+    """Run a command and handle errors"""
+    print(f"🔄 {description}...")
+    try:
+        result = subprocess.run(command, shell=True, check=True, capture_output=True, text=True)
+        print(f"✅ {description} completed successfully")
+        return True
+    except subprocess.CalledProcessError as e:
+        print(f"❌ {description} failed:")
+        print(f"   Error: {e.stderr}")
+        return False
+
+def main():
+    """Main installation process"""
+    print("🚀 Playwright Migration Installation Script")
+    print("=" * 50)
+
+    # Check if we're in a virtual environment
+    if not hasattr(sys, 'real_prefix') and not (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
+        print("⚠️  Warning: It's recommended to run this in a virtual environment")
+        response = input("Continue anyway? (y/N): ")
+        if response.lower() != 'y':
+            print("Installation cancelled.")
+            return 1
+
+    # Install Playwright
+    if not run_command("pip install playwright", "Installing Playwright"):
+        return 1
+
+    # Install Playwright browsers
+    if not run_command("playwright install chromium", "Installing Chromium browser"):
+        return 1
+
+    # Install system dependencies (for Linux)
+    if os.name == 'posix' and os.uname().sysname == 'Linux':
+        if not run_command("playwright install-deps chromium", "Installing system dependencies"):
+            print("⚠️  System dependencies installation failed. You may need to install them manually.")
+            print("   On Ubuntu/Debian: sudo apt-get install -y libnss3 libatk-bridge2.0-0 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxrandr2 libgbm1 libasound2")
+
+    # Test the installation
+    print("\n🧪 Testing Playwright installation...")
+    test_script = """
+import asyncio
+from playwright.async_api import async_playwright
+
+async def test():
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        context = await browser.new_context()
+        page = await context.new_page()
+        await page.goto('https://example.com')
+        title = await page.title()
+        await context.close()
+        await browser.close()
+        return title
+
+result = asyncio.run(test())
+print(f"✅ Playwright test successful! Page title: {result}")
+"""
+
+    try:
+        result = subprocess.run([sys.executable, '-c', test_script],
+                              capture_output=True, text=True, check=True)
+        print(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        print(f"❌ Playwright test failed: {e.stderr}")
+        return 1
+
+    print("\n🎉 Playwright installation completed successfully!")
+    print("\nNext steps:")
+    print("1. Update your requirements.txt to include 'playwright==1.40.0'")
+    print("2. Update your Dockerfile to install Playwright browsers")
+    print("3. Test your application with: python test_playwright_migration.py")
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

Dockers/puppeteer-api/main.py

@@ -1,5 +1,5 @@
 from fastapi import FastAPI, HTTPException, Header, Request
-from pyppeteer import launch
+from playwright.async_api import async_playwright
 import os
 import asyncio
 import json
@@ -46,6 +46,7 @@ browser_lock = Lock()
 browser_creation_times = {}
 active_browsers = set()  # Track active browsers
 operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS)  # Limit concurrent operations
+playwright_instance = None  # Global playwright instance
 
 # Rate limiting configuration
 RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60'))  # requests per minute
@@ -95,10 +96,14 @@ app.add_middleware(RateLimitMiddleware)
 
 async def create_browser():
     """Create a new browser instance with improved resource management"""
+    global playwright_instance
+
     try:
-        browser = await launch(
+        if playwright_instance is None:
+            playwright_instance = await async_playwright().start()
+
+        browser = await playwright_instance.chromium.launch(
             headless=True,
-            executablePath='/usr/bin/google-chrome',
             args=[
                 '--no-sandbox',
                 '--disable-setuid-sandbox',
@@ -127,11 +132,7 @@ async def create_browser():
                 '--disable-web-security',
                 '--disable-features=VizDisplayCompositor',
             ],
-            handleSIGINT=False,
+            ignore_default_args=['--enable-automation'],
-            handleSIGTERM=False,
-            handleSIGHUP=False,
-            ignoreHTTPSErrors=True,
-            autoClose=True,  # Ensure browser closes automatically
         )
 
         browser_creation_times[browser] = time.time()
@@ -152,7 +153,7 @@ async def cleanup_browser(browser):
             del browser_creation_times[browser]
 
         # Close all pages first
-        pages = await browser.pages()
+        pages = browser.contexts[0].pages if browser.contexts else []
         for page in pages:
             try:
                 await page.close()
@@ -191,7 +192,9 @@ async def check_browser_health():
                             browser = await create_browser()
                         else:
                             # Quick health check
-                            await browser.pages()
+                            contexts = browser.contexts
+                            if contexts:
+                                pages = contexts[0].pages
 
                         # Put back in pool if healthy
                         if not browser_pool.full():
@@ -200,208 +203,190 @@ async def check_browser_health():
                             # Pool is full, cleanup this browser
                             await cleanup_browser(browser)
                     except Exception as e:
-                        print(f"Browser health check failed: {e}")
+                        print(f"Error checking browser health: {e}")
-                        # If unhealthy, close and create new
+                        # Cleanup the problematic browser
-                        await cleanup_browser(browser)
+                        try:
-                        if not browser_pool.full():
+                            await cleanup_browser(browser)
-                            new_browser = await create_browser()
+                        except:
-                            await browser_pool.put(new_browser)
+                            pass
+
+                # Create new browsers if pool is empty
+                while browser_pool.qsize() < MAX_BROWSERS:
+                    try:
+                        browser = await create_browser()
+                        await browser_pool.put(browser)
+                    except Exception as e:
+                        print(f"Error creating browser for pool: {e}")
+                        break
+
         except Exception as e:
-            print(f"Error in browser health check: {str(e)}")
+            print(f"Error in browser health check: {e}")
+            await asyncio.sleep(60)  # Wait before retrying
 
 async def force_cleanup_all_browsers():
-    """Force cleanup all browsers in emergency situations"""
+    """Force cleanup all browsers - useful for emergency situations"""
     print("Force cleaning up all browsers...")
 
-    # Clean up pool
+    async with browser_lock:
-    while not browser_pool.empty():
+        # Clean up browsers in pool
-        try:
+        while not browser_pool.empty():
-            browser = await browser_pool.get_nowait()
+            try:
-            await cleanup_browser(browser)
+                browser = await browser_pool.get_nowait()
-        except asyncio.QueueEmpty:
+                await cleanup_browser(browser)
-            break
+            except asyncio.QueueEmpty:
+                break
 
-    # Clean up active browsers
+        # Clean up any remaining active browsers
-    for browser in list(active_browsers):
+        for browser in list(active_browsers):
-        await cleanup_browser(browser)
+            try:
+                await cleanup_browser(browser)
+            except Exception as e:
+                print(f"Error force cleaning browser: {e}")
+
+    print("Force cleanup completed")
 
-# Initialize browser pool
 @app.on_event("startup")
 async def init_browser_pool():
-    """Initialize the browser pool with some browsers"""
+    """Initialize the browser pool on startup"""
-    try:
+    print("Initializing browser pool...")
-        for _ in range(min(2, MAX_BROWSERS)):  # Start with 2 browsers instead of 3
+
+    # Start browser health check task
+    asyncio.create_task(check_browser_health())
+
+    # Pre-populate pool with initial browsers
+    for _ in range(min(2, MAX_BROWSERS)):
+        try:
             browser = await create_browser()
             await browser_pool.put(browser)
-        print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
+        except Exception as e:
+            print(f"Error creating initial browser: {e}")
 
-        # Start browser health check task
+    print(f"Browser pool initialized with {browser_pool.qsize()} browsers")
-        asyncio.create_task(check_browser_health())
-    except Exception as e:
-        print(f"Error initializing browser pool: {e}")
 
 @app.on_event("shutdown")
 async def cleanup_browser_pool():
-    """Clean up all browsers in the pool"""
+    """Cleanup browser pool on shutdown"""
+    print("Cleaning up browser pool...")
     await force_cleanup_all_browsers()
 
-# Signal handlers for graceful shutdown
+    # Stop playwright instance
+    global playwright_instance
+    if playwright_instance:
+        await playwright_instance.stop()
+        playwright_instance = None
+
+    print("Browser pool cleanup completed")
+
 def signal_handler(signum, frame):
+    """Handle shutdown signals"""
     print(f"Received signal {signum}, shutting down gracefully...")
-    asyncio.create_task(force_cleanup_all_browsers())
+    asyncio.create_task(cleanup_browser_pool())
+    exit(0)
 
-signal.signal(signal.SIGTERM, signal_handler)
+# Register signal handlers
 signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
 
-# Initialize SQLite database
 def init_db():
-    global DB_PATH
+    """Initialize the SQLite database"""
+    conn = sqlite3.connect('/db/cache.db')
+    cursor = conn.cursor()
 
-    # Try to use the mounted volume first
+    # Create cache table
-    db_path = '/db/cache.db'
+    cursor.execute('''
-    db_dir = os.path.dirname(db_path)
-
-    # Check if directory exists and is writable
-    dir_writable = False
-    if os.path.exists(db_dir):
-        try:
-            test_file = os.path.join(db_dir, '.write_test')
-            with open(test_file, 'w') as f:
-                f.write('test')
-            os.remove(test_file)
-            dir_writable = True
-        except (IOError, PermissionError):
-            print(f"Directory {db_dir} exists but is not writable")
-            dir_writable = False
-
-    # If directory doesn't exist or isn't writable, try to create it
-    if not os.path.exists(db_dir) or not dir_writable:
-        try:
-            os.makedirs(db_dir, exist_ok=True)
-            # Test if we can write to the directory
-            test_file = os.path.join(db_dir, '.write_test')
-            with open(test_file, 'w') as f:
-                f.write('test')
-            os.remove(test_file)
-            print(f"Created directory: {db_dir}")
-            dir_writable = True
-        except Exception as e:
-            print(f"Warning: Could not create or write to directory {db_dir}: {e}")
-            # Fallback to using a local database file
-            db_path = 'cache.db'
-            print(f"Using local database file: {db_path}")
-
-    try:
-        conn = sqlite3.connect(db_path)
-        cursor = conn.cursor()
-        cursor.execute('''
         CREATE TABLE IF NOT EXISTS cache (
-            url TEXT,
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            route TEXT,
+            url TEXT NOT NULL,
-            data TEXT,
+            route TEXT NOT NULL,
-            timestamp INTEGER,
+            data TEXT NOT NULL,
-            PRIMARY KEY (url, route)
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            UNIQUE(url, route)
         )
-        ''')
+    ''')
-        conn.commit()
-        conn.close()
-        print(f"Database initialized at {db_path}")
-        # Update the global DB_PATH
-        DB_PATH = db_path
-    except sqlite3.OperationalError as e:
-        print(f"Error initializing database at {db_path}: {e}")
-        # Fallback to using a local database file if the mounted volume has permission issues
-        db_path = 'cache.db'
-        print(f"Falling back to local database file: {db_path}")
-        try:
-            conn = sqlite3.connect(db_path)
-            cursor = conn.cursor()
-            cursor.execute('''
-            CREATE TABLE IF NOT EXISTS cache (
-                url TEXT,
-                route TEXT,
-                data TEXT,
-                timestamp INTEGER,
-                PRIMARY KEY (url, route)
-            )
-            ''')
-            conn.commit()
-            conn.close()
-            print(f"Local database initialized at {db_path}")
-            # Update the global DB_PATH
-            DB_PATH = db_path
-        except sqlite3.OperationalError as e2:
-            print(f"Error initializing local database: {e2}")
-            raise
 
-# Define the database path
+    # Create index for faster lookups
-DB_PATH = '/db/cache.db'
+    cursor.execute('''
+        CREATE INDEX IF NOT EXISTS idx_cache_url_route
+        ON cache(url, route)
+    ''')
 
-# Get cached data if it exists and is not older than the expiry time
+    # Create index for cleanup operations
-def get_cached_data(url, route):
+    cursor.execute('''
-    conn = sqlite3.connect(DB_PATH)
+        CREATE INDEX IF NOT EXISTS idx_cache_created_at
-    cursor = conn.cursor()
+        ON cache(created_at)
-    cache_expiry = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)  # Convert hours to seconds
+    ''')
-    cursor.execute(
-        "SELECT data FROM cache WHERE url = ? AND route = ? AND timestamp > ?",
-        (url, route, cache_expiry)
-    )
-    result = cursor.fetchone()
-    conn.close()
 
-    if result:
-        print(f"Cache hit for {url} on route {route}")
-        return json.loads(result[0])
-    return None
-
-# Save data to cache
-def save_to_cache(url, route, data):
-    conn = sqlite3.connect(DB_PATH)
-    cursor = conn.cursor()
-    timestamp = int(time.time())
-
-    # Convert data to JSON string
-    data_json = json.dumps(data)
-
-    cursor.execute(
-        "INSERT OR REPLACE INTO cache (url, route, data, timestamp) VALUES (?, ?, ?, ?)",
-        (url, route, data_json, timestamp)
-    )
     conn.commit()
     conn.close()
-    print(f"Saved to cache: {url} on route {route}")
+    print("Database initialized")
 
-# Function to clean up old cache entries
+def get_cached_data(url, route):
-def cleanup_old_cache_entries():
+    """Get cached data for a URL and route"""
     try:
-        print(f"Running scheduled cache cleanup (entries older than {CACHE_EXPIRY_HOURS} hours, pagination: {CACHE_EXPIRY_HOURS * 31} hours)")
+        conn = sqlite3.connect('/db/cache.db')
-        conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
 
-        # Calculate the timestamp for entries older than the expiry time
+        cursor.execute('''
-        expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 60 * 60)
+            SELECT data, created_at FROM cache
-        pagination_expiry_timestamp = int(time.time()) - (CACHE_EXPIRY_HOURS * 31 * 60 * 60)
+            WHERE url = ? AND route = ?
+        ''', (url, route))
 
-        # Get count of entries to be deleted (non-pagination)
+        result = cursor.fetchone()
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+        conn.close()
-        count_non_pagination = cursor.fetchone()[0]
 
-        # Get count of pagination entries to be deleted
+        if result:
-        cursor.execute("SELECT COUNT(*) FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
+            data, created_at = result
-        count_pagination = cursor.fetchone()[0]
+            created_time = datetime.fromisoformat(created_at)
 
-        # Delete old non-pagination entries
+            # Check if cache is still valid
-        cursor.execute("DELETE FROM cache WHERE route != 'pagination' AND timestamp < ?", (expiry_timestamp,))
+            if datetime.now() - created_time < timedelta(hours=CACHE_EXPIRY_HOURS):
+                return json.loads(data)
 
-        # Delete old pagination entries
+        return None
-        cursor.execute("DELETE FROM cache WHERE route = 'pagination' AND timestamp < ?", (pagination_expiry_timestamp,))
+    except Exception as e:
+        print(f"Error getting cached data: {e}")
+        return None
+
+def save_to_cache(url, route, data):
+    """Save data to cache"""
+    try:
+        conn = sqlite3.connect('/db/cache.db')
+        cursor = conn.cursor()
+
+        cursor.execute('''
+            INSERT OR REPLACE INTO cache (url, route, data, created_at)
+            VALUES (?, ?, ?, CURRENT_TIMESTAMP)
+        ''', (url, route, json.dumps(data)))
 
         conn.commit()
         conn.close()
-
-        print(f"Cache cleanup completed: {count_non_pagination} non-pagination entries and {count_pagination} pagination entries removed")
     except Exception as e:
-        print(f"Error during cache cleanup: {e}")
+        print(f"Error saving to cache: {e}")
+
+def cleanup_old_cache_entries():
+    """Clean up old cache entries"""
+    try:
+        conn = sqlite3.connect('/db/cache.db')
+        cursor = conn.cursor()
+
+        # Delete entries older than CACHE_EXPIRY_HOURS
+        cutoff_time = datetime.now() - timedelta(hours=CACHE_EXPIRY_HOURS)
+
+        cursor.execute('''
+            DELETE FROM cache
+            WHERE created_at < ?
+        ''', (cutoff_time.isoformat(),))
+
+        deleted_count = cursor.rowcount
+        conn.commit()
+        conn.close()
+
+        print(f"Cleaned up {deleted_count} old cache entries")
+    except Exception as e:
+        print(f"Error cleaning up cache: {e}")
+
+# Initialize database
+init_db()
 
 # Initialize scheduler for periodic cache cleanup
 scheduler = BackgroundScheduler()
@@ -412,74 +397,81 @@ scheduler.add_job(
     replace_existing=True
 )
 
-# Initialize database on startup
-init_db()
-
-# Start the scheduler when the application starts
 @app.on_event("startup")
 def start_scheduler():
     scheduler.start()
     print(f"Cache cleanup scheduler started with cron: {CLEANUP_CRON}")
-    print(f"Cache entries will expire after {CACHE_EXPIRY_HOURS} hours")
 
-# Shutdown the scheduler when the application stops
 @app.on_event("shutdown")
 def shutdown_scheduler():
     scheduler.shutdown(wait=False)
     print("Cache cleanup scheduler stopped")
 
 async def wait_for_network_idle(page):
-    """Wait until no network requests are in flight"""
+    """Wait for network to be idle"""
-    await page.waitForNetworkIdle(idleTime=500, timeout=30000)
+    await page.wait_for_load_state('networkidle')
 
 @app.head("/")
 async def health_check():
-    return {"status": "ok"}
+    return {"status": "healthy"}
 
 async def safe_browser_operation(url, operation_func):
-    """Safely perform a browser operation with proper cleanup and resource limits"""
+    """Safely perform a browser operation with proper resource management"""
-    async with operation_semaphore:  # Limit concurrent operations
+    async with operation_semaphore:
-        async with get_browser() as browser:
+        browser = None
-            page = None
+        context = None
+        page = None
+
+        try:
+            # Get browser from pool or create new one
             try:
-                # Create a new page
+                browser = await asyncio.wait_for(browser_pool.get(), timeout=10.0)
-                page = await browser.newPage()
+            except asyncio.TimeoutError:
+                print("Timeout getting browser from pool, creating new one")
+                browser = await create_browser()
 
-                # Set reasonable viewport
+            # Create context and page
-                await page.setViewport({'width': 1280, 'height': 800})
+            context = await browser.new_context(
+                user_agent=CUSTOM_USER_AGENT,
+                viewport={'width': 1920, 'height': 1080},
+                ignore_https_errors=True,
+            )
 
-                # Set user agent
+            page = await context.new_page()
-                await page.setUserAgent(CUSTOM_USER_AGENT)
 
-                # Set reasonable timeout
+            # Set up request interception for better performance
-                page.setDefaultNavigationTimeout(30000)
+            await page.route("**/*", lambda route: route.abort()
+                           if route.request.resource_type in ['image', 'stylesheet', 'font', 'media']
+                           else route.continue_())
 
-                # Enable request interception to block unnecessary resources
+            # Perform the operation
-                await page.setRequestInterception(True)
+            result = await operation_func(page)
+            return result
 
-                async def intercept(request):
+        except Exception as e:
-                    # Block unnecessary resource types
+            print(f"Error in browser operation: {e}")
-                    if request.resourceType in ['image', 'media', 'font', 'stylesheet']:
+            raise
-                        await request.abort()
+        finally:
-                    else:
+            # Cleanup
-                        await request.continue_()
+            if page:
-
-                page.on('request', lambda req: asyncio.ensure_future(intercept(req)))
-
-                # Perform the operation
-                result = await operation_func(page)
-
-                return result
-            except Exception as e:
-                print(f"Error during browser operation: {str(e)}")
-                raise
-            finally:
                 try:
-                    # Ensure page is properly closed
+                    await page.close()
-                    if page:
+                except:
-                        await page.close()
+                    pass
-                except Exception as e:
+            if context:
-                    print(f"Error closing page: {str(e)}")
+                try:
+                    await context.close()
+                except:
+                    pass
+            if browser:
+                try:
+                    # Return browser to pool if it's still healthy
+                    if not browser_pool.full():
+                        await browser_pool.put(browser)
+                    else:
+                        await cleanup_browser(browser)
+                except:
+                    pass
 
 @app.get("/")
 async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
@@ -496,12 +488,9 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
         return cached_result
 
     try:
-        print(f"Visiting URL: {decoded_url}")
-
-        # Define the operation to perform with the browser
         async def visit_operation(page):
             try:
-                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
                 if not response:
                     print(f"Warning: No response object returned for {decoded_url}")
 
@@ -515,14 +504,10 @@ async def visit_url(url: str, x_api_key: Optional[str] = Header(None)):
                     content = await page.content()
                     return {"status": "partial", "content": content, "error": str(e)}
                 except:
-                    raise HTTPException(status_code=500, detail=f"Failed to get page content: {str(e)}")
+                    raise Exception(f"Failed to get page content: {str(e)}")
 
-        # Perform the operation
         result = await safe_browser_operation(decoded_url, visit_operation)
-
-        # Save to cache
         save_to_cache(decoded_url, "visit", result)
-
         return result
 
     except Exception as e:
@@ -545,12 +530,9 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
         return cached_result
 
     try:
-        print(f"Extracting SEO from: {decoded_url}")
-
-        # Define the operation to perform with the browser
         async def seo_operation(page):
             try:
-                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
 
                 # Extract SEO information
                 seo_data = await page.evaluate('''() => {
@@ -609,12 +591,8 @@ async def extract_seo(url: str, x_api_key: Optional[str] = Header(None)):
                 print(f"Error during SEO extraction: {e}")
                 return {"status": "error", "url": decoded_url, "error": str(e)}
 
-        # Perform the operation
         result = await safe_browser_operation(decoded_url, seo_operation)
-
-        # Save to cache
         save_to_cache(decoded_url, "seo", result)
-
         return result
 
     except Exception as e:
@@ -636,52 +614,48 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
         return cached_result
 
     try:
-        print(f"Extracting meta tags from: {decoded_url}")
-
-        # Define the operation to perform with the browser
         async def meta_operation(page):
             try:
-                response = await page.goto(decoded_url, waitUntil='networkidle2', timeout=30000)
+                response = await page.goto(decoded_url, wait_until='networkidle', timeout=30000)
 
-                # Extract all meta tags
+                # Extract meta tags using Playwright
-                meta_tags = await page.evaluate('''() => {
+                meta_data = await page.evaluate('''() => {
-                    const metas = Array.from(document.querySelectorAll('meta'));
+                    const data = {
-                    return metas.map(meta => {
+                        meta_tags: [],
+                        open_graph: {},
+                        twitter_card: {},
+                        title: document.title || ''
+                    };
+
+                    // Extract all meta tags
+                    document.querySelectorAll('meta').forEach(meta => {
                         const attributes = {};
-                        Array.from(meta.attributes).forEach(attr => {
+                        for (let attr of meta.attributes) {
                             attributes[attr.name] = attr.value;
-                        });
+                        }
-                        return attributes;
+                        data.meta_tags.push(attributes);
                     });
-                }''')
 
-                # Extract Open Graph tags
+                    // Extract Open Graph tags
-                og_tags = await page.evaluate('''() => {
+                    document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
-                    const ogTags = {};
+                        data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content');
-                    document.querySelectorAll('meta[property^="og:"]').forEach(tag => {
-                        const property = tag.getAttribute('property');
-                        ogTags[property] = tag.getAttribute('content');
                     });
-                    return ogTags;
-                }''')
 
-                # Extract Twitter card tags
+                    // Extract Twitter card tags
-                twitter_tags = await page.evaluate('''() => {
+                    document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
-                    const twitterTags = {};
+                        data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content');
-                    document.querySelectorAll('meta[name^="twitter:"]').forEach(tag => {
-                        const name = tag.getAttribute('name');
-                        twitterTags[name] = tag.getAttribute('content');
                     });
-                    return twitterTags;
+
+                    return data;
                 }''')
 
                 result = {
                     "status": "success",
                     "url": decoded_url,
-                    "meta_tags": meta_tags,
+                    "meta_tags": meta_data['meta_tags'],
-                    "open_graph": og_tags,
+                    "open_graph": meta_data['open_graph'],
-                    "twitter_card": twitter_tags,
+                    "twitter_card": meta_data['twitter_card'],
-                    "title": await page.title()
+                    "title": meta_data['title']
                 }
 
                 return result
@@ -690,12 +664,8 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
                 print(f"Error during meta tag extraction: {e}")
                 return {"status": "error", "url": decoded_url, "error": str(e)}
 
-        # Perform the operation
         result = await safe_browser_operation(decoded_url, meta_operation)
-
-        # Save to cache
         save_to_cache(decoded_url, "meta", result)
-
         return result
 
     except Exception as e:
@@ -703,18 +673,22 @@ async def extract_meta_tags(url: str, x_api_key: Optional[str] = Header(None)):
 
 @app.get("/cache/clear")
 async def clear_cache(x_api_key: Optional[str] = Header(None)):
-    """Clear the entire cache database"""
+    """Clear all cached data"""
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
 
-    conn = sqlite3.connect(DB_PATH)
+    try:
-    cursor = conn.cursor()
+        conn = sqlite3.connect('/db/cache.db')
-    cursor.execute("DELETE FROM cache")
+        cursor = conn.cursor()
-    conn.commit()
+        cursor.execute('DELETE FROM cache')
-    conn.close()
+        deleted_count = cursor.rowcount
+        conn.commit()
+        conn.close()
 
-    return {"status": "success", "message": "Cache cleared successfully"}
+        return {"status": "success", "message": f"Cleared {deleted_count} cache entries"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
 
 @app.get("/cache/stats")
 async def cache_stats(x_api_key: Optional[str] = Header(None)):
@@ -724,79 +698,103 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)):
         raise HTTPException(status_code=401, detail="Invalid API key")
 
     try:
-        conn = sqlite3.connect(DB_PATH)
+        conn = sqlite3.connect('/db/cache.db')
         cursor = conn.cursor()
 
         # Get total count
-        cursor.execute("SELECT COUNT(*) FROM cache")
+        cursor.execute('SELECT COUNT(*) FROM cache')
         total_count = cursor.fetchone()[0]
 
         # Get count by route
-        cursor.execute("SELECT route, COUNT(*) FROM cache GROUP BY route")
+        cursor.execute('''
+            SELECT route, COUNT(*) as count
+            FROM cache
+            GROUP BY route
+        ''')
         route_counts = dict(cursor.fetchall())
 
         # Get oldest and newest entries
-        cursor.execute("SELECT MIN(timestamp), MAX(timestamp) FROM cache")
+        cursor.execute('''
-        min_time, max_time = cursor.fetchone()
+            SELECT MIN(created_at), MAX(created_at)
+            FROM cache
+        ''')
+        oldest, newest = cursor.fetchone()
+
+        # Get database size
+        cursor.execute('PRAGMA page_count')
+        page_count = cursor.fetchone()[0]
+        cursor.execute('PRAGMA page_size')
+        page_size = cursor.fetchone()[0]
+        db_size = page_count * page_size
 
         conn.close()
 
         return {
+            "status": "success",
             "total_entries": total_count,
             "route_counts": route_counts,
-            "oldest_entry": min_time,
+            "oldest_entry": oldest,
-            "newest_entry": max_time
+            "newest_entry": newest,
+            "database_size_bytes": db_size,
+            "cache_expiry_hours": CACHE_EXPIRY_HOURS
         }
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.get("/status")
 async def system_status(x_api_key: Optional[str] = Header(None)):
-    """Get system status and browser pool information"""
+    """Get system status and health information"""
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
 
     try:
         # Get system information
-        process = psutil.Process()
+        cpu_percent = psutil.cpu_percent(interval=1)
-        memory_info = process.memory_info()
+        memory = psutil.virtual_memory()
+        disk = psutil.disk_usage('/')
 
         # Get browser pool information
         pool_size = browser_pool.qsize()
         active_browser_count = len(active_browsers)
 
-        # Calculate browser ages
+        # Get cache statistics
-        browser_ages = []
+        conn = sqlite3.connect('/db/cache.db')
-        for browser, creation_time in browser_creation_times.items():
+        cursor = conn.cursor()
-            age = time.time() - creation_time
+        cursor.execute('SELECT COUNT(*) FROM cache')
-            browser_ages.append(age)
+        cache_count = cursor.fetchone()[0]
+        conn.close()
 
         return {
+            "status": "success",
             "system": {
-                "cpu_percent": process.cpu_percent(),
+                "cpu_percent": cpu_percent,
-                "memory_mb": memory_info.rss / 1024 / 1024,
+                "memory_percent": memory.percent,
-                "memory_percent": process.memory_percent(),
+                "memory_available_gb": round(memory.available / (1024**3), 2),
-                "open_files": len(process.open_files()),
+                "disk_percent": disk.percent,
-                "connections": len(process.connections()),
+                "disk_free_gb": round(disk.free / (1024**3), 2)
-                "threads": process.num_threads()
             },
             "browser_pool": {
                 "pool_size": pool_size,
-                "active_browsers": active_browser_count,
                 "max_browsers": MAX_BROWSERS,
-                "browser_ttl_seconds": BROWSER_TTL,
+                "active_browsers": active_browser_count,
-                "browser_ages_seconds": browser_ages,
+                "browser_ttl_seconds": BROWSER_TTL
-                "concurrent_operations_limit": MAX_CONCURRENT_OPERATIONS
             },
-            "timestamp": time.time()
+            "cache": {
+                "total_entries": cache_count,
+                "expiry_hours": CACHE_EXPIRY_HOURS
+            },
+            "rate_limiting": {
+                "requests_per_minute": RATE_LIMIT_MINUTE,
+                "window_seconds": RATE_LIMIT_WINDOW
+            }
         }
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.post("/emergency-cleanup")
 async def emergency_cleanup(x_api_key: Optional[str] = Header(None)):
-    """Force emergency cleanup of all browsers"""
+    """Emergency cleanup endpoint to force cleanup all browsers"""
     # Validate API key
     if not x_api_key or x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Invalid API key")
@@ -809,60 +807,14 @@ async def emergency_cleanup(x_api_key: Optional[str] = Header(None)):
 
 @asynccontextmanager
 async def get_browser():
-    """Get a browser from the pool or create a new one if needed"""
+    """Context manager for getting a browser from the pool"""
     browser = None
     try:
-        # Try to get a browser from the pool with timeout
+        browser = await browser_pool.get()
-        try:
-            browser = await asyncio.wait_for(browser_pool.get(), timeout=30.0)
-        except (asyncio.QueueEmpty, asyncio.TimeoutError):
-            # If pool is empty or timeout, create a new browser if under the limit
-            async with browser_lock:
-                current_browser_count = len(active_browsers)
-                if current_browser_count < MAX_BROWSERS:
-                    browser = await create_browser()
-                else:
-                    # If at limit, wait for a browser to become available with timeout
-                    try:
-                        browser = await asyncio.wait_for(browser_pool.get(), timeout=60.0)
-                    except asyncio.TimeoutError:
-                        # Emergency cleanup and create new browser
-                        print("Emergency: Timeout waiting for browser, forcing cleanup")
-                        await force_cleanup_all_browsers()
-                        browser = await create_browser()
-
-        yield browser
-    except Exception as e:
-        print(f"Error in get_browser: {e}")
-        # Emergency cleanup if we can't get a browser
-        await force_cleanup_all_browsers()
-        browser = await create_browser()
         yield browser
     finally:
-        # Return browser to pool if it's still viable
         if browser:
-            try:
+            await browser_pool.put(browser)
-                # Quick check if browser is still usable
-                await browser.pages()
-                # Check if browser is too old
-                if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL:
-                    print(f"Recycling old browser in get_browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)")
-                    await cleanup_browser(browser)
-                    browser = await create_browser()
-
-                # Only put back if pool is not full
-                if not browser_pool.full():
-                    await browser_pool.put(browser)
-                else:
-                    # Pool is full, cleanup this browser
-                    await cleanup_browser(browser)
-            except Exception as e:
-                print(f"Browser health check failed in get_browser: {e}")
-                # If browser is not usable, close it and create a new one
-                await cleanup_browser(browser)
-                if not browser_pool.full():
-                    new_browser = await create_browser()
-                    await browser_pool.put(new_browser)
 
 if __name__ == "__main__":
     import uvicorn

Dockers/puppeteer-api/requirements.txt

@@ -1,7 +1,7 @@
-fastapi==0.68.1
+fastapi==0.104.1
-uvicorn==0.15.0
+uvicorn==0.24.0
-pyppeteer==1.0.2
+playwright==1.40.0
 psutil==6.0.0
-apscheduler
+apscheduler==3.10.4
 aiohttp==3.9.1
 beautifulsoup4==4.12.2

Dockers/puppeteer-api/test_playwright_migration.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""
+Test script to verify Playwright migration works correctly
+"""
+
+import asyncio
+import sys
+import os
+
+# Add the current directory to Python path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from playwright.async_api import async_playwright
+
+async def test_playwright_installation():
+    """Test that Playwright is properly installed and can launch a browser"""
+    print("Testing Playwright installation...")
+
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context()
+            page = await context.new_page()
+
+            # Test basic navigation
+            response = await page.goto('https://example.com', wait_until='networkidle', timeout=10000)
+            title = await page.title()
+
+            print(f"✅ Successfully navigated to example.com")
+            print(f"   Title: {title}")
+            print(f"   Status: {response.status if response else 'No response'}")
+
+            await context.close()
+            await browser.close()
+
+            return True
+
+    except Exception as e:
+        print(f"❌ Playwright test failed: {e}")
+        return False
+
+async def test_browser_utils():
+    """Test the updated browser_utils module"""
+    print("\nTesting browser_utils module...")
+
+    try:
+        from app.utils.browser_utils import safe_browser_operation
+
+        async def test_operation(page):
+            await page.goto('https://example.com', wait_until='networkidle', timeout=10000)
+            title = await page.title()
+            return {"status": "success", "title": title}
+
+        result = await safe_browser_operation('https://example.com', test_operation)
+
+        if result.get('status') == 'success':
+            print(f"✅ browser_utils test passed")
+            print(f"   Title: {result.get('title')}")
+            return True
+        else:
+            print(f"❌ browser_utils test failed: {result}")
+            return False
+
+    except Exception as e:
+        print(f"❌ browser_utils test failed: {e}")
+        return False
+
+async def test_browser_service():
+    """Test the updated browser service"""
+    print("\nTesting browser service...")
+
+    try:
+        from app.services.browser import visit_url_service
+
+        result = await visit_url_service('https://example.com')
+
+        if result.get('status') == 'success':
+            print(f"✅ browser service test passed")
+            print(f"   Content length: {len(result.get('content', ''))}")
+            return True
+        else:
+            print(f"❌ browser service test failed: {result}")
+            return False
+
+    except Exception as e:
+        print(f"❌ browser service test failed: {e}")
+        return False
+
+async def main():
+    """Run all tests"""
+    print("🧪 Running Playwright migration tests...\n")
+
+    tests = [
+        test_playwright_installation,
+        test_browser_utils,
+        test_browser_service,
+    ]
+
+    results = []
+    for test in tests:
+        try:
+            result = await test()
+            results.append(result)
+        except Exception as e:
+            print(f"❌ Test {test.__name__} failed with exception: {e}")
+            results.append(False)
+
+    print(f"\n📊 Test Results:")
+    print(f"   Passed: {sum(results)}/{len(results)}")
+
+    if all(results):
+        print("🎉 All tests passed! Playwright migration is working correctly.")
+        return 0
+    else:
+        print("❌ Some tests failed. Please check the errors above.")
+        return 1
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)

Dockers/puppeteer-api/version

@@ -1 +1 @@
-latest
+3.0.0