basic playwright api

Dockers/playwright-node-api/scraper.js

@@ -0,0 +1,103 @@
+const dbOps = require('./database');
+const { getBrowser } = require('./browser');
+const config = require('./config');
+
+async function fetchHtml(url, skipCache = false) {
+    let page = null;
+    try {
+        // Check cache first (unless skipCache is true)
+        if (!skipCache) {
+            const now = Date.now();
+            const cached = await dbOps.getCached(url, now);
+            if (cached) {
+                console.log(`Cache hit for: ${url}`);
+                return cached;
+            }
+        }
+
+        console.log(`Fetching: ${url} (skipCache: ${skipCache})`);
+
+        // Get browser instance
+        const browserInstance = await getBrowser();
+
+        // Create new page
+        page = await browserInstance.newPage();
+
+        // Set reasonable timeouts
+        page.setDefaultTimeout(30000);
+        page.setDefaultNavigationTimeout(30000);
+
+        // Track network requests to ensure all are completed
+        const pendingRequests = new Set();
+        const finishedRequests = new Set();
+
+        // Monitor network requests
+        page.on('request', (request) => {
+            const requestId = request.url();
+            pendingRequests.add(requestId);
+        });
+
+        page.on('response', (response) => {
+            const requestId = response.url();
+            finishedRequests.add(requestId);
+            pendingRequests.delete(requestId);
+        });
+
+        page.on('requestfailed', (request) => {
+            const requestId = request.url();
+            pendingRequests.delete(requestId);
+        });
+
+        // Navigate to URL
+        await page.goto(url, {
+            waitUntil: 'domcontentloaded',
+            timeout: 30000,
+        });
+
+        // Wait for network to be idle (no requests for 500ms)
+        await page.waitForLoadState('networkidle', { timeout: 30000 });
+
+        // Additional wait to ensure all pending requests complete
+        const maxWaitTime = 10000; // 10 seconds max
+        const checkInterval = 100; // Check every 100ms
+        const startTime = Date.now();
+
+        while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
+            await new Promise(resolve => setTimeout(resolve, checkInterval));
+        }
+
+        if (pendingRequests.size > 0) {
+            console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
+        }
+
+        // Get full HTML
+        const html = await page.content();
+
+        // Store in cache
+        const now = Date.now();
+        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
+        await dbOps.setCache(url, html, now, expiresAt);
+        console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+
+        return html;
+    } catch (error) {
+        console.error(`Error fetching ${url}:`, error.message);
+
+        // If cache exists and we had an error, optionally return stale cache
+        // For now, we'll just throw the error
+        throw error;
+    } finally {
+        // Always close the page, even on error
+        if (page) {
+            try {
+                await page.close();
+            } catch (closeError) {
+                console.error('Error closing page:', closeError.message);
+            }
+        }
+    }
+}
+
+module.exports = {
+    fetchHtml,
+};