other routes

Dockers/playwright-node-api/database.js

@@ -287,6 +287,90 @@ const dbOps = {
         }
     },
 
+    async getCachedOutgoingCalls(url, now) {
+        const callsKey = `outgoing-calls:${url}`;
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'SELECT html FROM cache WHERE url = $1 AND expires_at > $2',
+                    [callsKey, now]
+                );
+                if (result.rows.length > 0) {
+                    return JSON.parse(result.rows[0].html);
+                }
+                return null;
+            } finally {
+                client.release();
+            }
+        } else {
+            const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?');
+            const result = getCached.get(callsKey, now);
+            return result ? JSON.parse(result.html) : null;
+        }
+    },
+
+    async setCacheOutgoingCalls(url, callsData, now, expiresAt) {
+        const callsKey = `outgoing-calls:${url}`;
+        const html = JSON.stringify(callsData);
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                await client.query(
+                    'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4',
+                    [callsKey, html, now, expiresAt]
+                );
+            } finally {
+                client.release();
+            }
+        } else {
+            const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)');
+            setCache.run(callsKey, html, now, expiresAt);
+        }
+    },
+
+    async getCachedResultingUrl(url, now) {
+        const resultingUrlKey = `resulting-url:${url}`;
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'SELECT html FROM cache WHERE url = $1 AND expires_at > $2',
+                    [resultingUrlKey, now]
+                );
+                if (result.rows.length > 0) {
+                    return JSON.parse(result.rows[0].html);
+                }
+                return null;
+            } finally {
+                client.release();
+            }
+        } else {
+            const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?');
+            const result = getCached.get(resultingUrlKey, now);
+            return result ? JSON.parse(result.html) : null;
+        }
+    },
+
+    async setCacheResultingUrl(url, resultingUrlData, now, expiresAt) {
+        const resultingUrlKey = `resulting-url:${url}`;
+        const html = JSON.stringify(resultingUrlData);
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                await client.query(
+                    'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4',
+                    [resultingUrlKey, html, now, expiresAt]
+                );
+            } finally {
+                client.release();
+            }
+        } else {
+            const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)');
+            setCache.run(resultingUrlKey, html, now, expiresAt);
+        }
+    },
+
     async deleteExpired(now) {
         if (dbType === 'postgres') {
             const client = await pgPool.connect();

Dockers/playwright-node-api/routes.js

@@ -1,5 +1,5 @@
 const express = require('express');
-const { fetchHtml, extractSeo, extractMetaTags } = require('./scraper');
+const { fetchHtml, extractSeo, extractMetaTags, extractOutgoingCalls, extractResultingUrl } = require('./scraper');
 const dbOps = require('./database');
 const { authenticateApiKey } = require('./middleware');
 const { isBrowserConnected } = require('./browser');
@@ -155,4 +155,78 @@ router.get('/meta', authenticateApiKey, async (req, res) => {
     }
 });
 
+// Outgoing calls capture endpoint
+router.get('/outgoing-calls', authenticateApiKey, async (req, res) => {
+    const { url, skipCache } = req.query;
+
+    // Validate URL parameter
+    if (!url) {
+        return res.status(400).json({
+            error: 'Missing required parameter: url',
+        });
+    }
+
+    // Validate URL format
+    let urlObj;
+    try {
+        urlObj = new URL(url);
+    } catch (error) {
+        return res.status(400).json({
+            error: 'Invalid URL format',
+        });
+    }
+
+    // Parse skipCache (accept 'true', '1', 'yes', etc.)
+    const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes';
+
+    try {
+        const callsData = await extractOutgoingCalls(url, shouldSkipCache);
+        res.json(callsData);
+    } catch (error) {
+        console.error('Outgoing calls capture error:', error);
+        res.status(500).json({
+            status: 'error',
+            url: url,
+            error: error.message,
+        });
+    }
+});
+
+// Resulting URL endpoint
+router.get('/resulting-url', authenticateApiKey, async (req, res) => {
+    const { url, skipCache } = req.query;
+
+    // Validate URL parameter
+    if (!url) {
+        return res.status(400).json({
+            error: 'Missing required parameter: url',
+        });
+    }
+
+    // Validate URL format
+    let urlObj;
+    try {
+        urlObj = new URL(url);
+    } catch (error) {
+        return res.status(400).json({
+            error: 'Invalid URL format',
+        });
+    }
+
+    // Parse skipCache (accept 'true', '1', 'yes', etc.)
+    const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes';
+
+    try {
+        const resultingUrlData = await extractResultingUrl(url, shouldSkipCache);
+        res.json(resultingUrlData);
+    } catch (error) {
+        console.error('Resulting URL error:', error);
+        res.status(500).json({
+            status: 'error',
+            original_url: url,
+            error: error.message,
+        });
+    }
+});
+
 module.exports = router;

Dockers/playwright-node-api/scraper.js

@@ -289,8 +289,283 @@ async function extractMetaTags(url, skipCache = false) {
     }
 }
 
+async function extractOutgoingCalls(url, skipCache = false) {
+    let page = null;
+    try {
+        // Check cache first (unless skipCache is true)
+        if (!skipCache) {
+            const now = Date.now();
+            const cached = await dbOps.getCachedOutgoingCalls(url, now);
+            if (cached) {
+                console.log(`Outgoing calls cache hit for: ${url}`);
+                return cached;
+            }
+        }
+
+        console.log(`Capturing outgoing calls from: ${url}`);
+
+        // Get browser instance
+        const browserInstance = await getBrowser();
+
+        // Create new page
+        page = await browserInstance.newPage();
+
+        // Set reasonable timeouts
+        page.setDefaultTimeout(30000);
+        page.setDefaultNavigationTimeout(30000);
+
+        // List to store all network requests
+        const networkRequests = [];
+
+        // Static asset extensions to skip
+        const staticExtensions = ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.eot'];
+
+        // Set up network request listener BEFORE navigation
+        page.on('request', (request) => {
+            const requestUrl = request.url();
+            const method = request.method();
+            const headers = request.headers();
+
+            // Skip static assets and common non-API requests
+            if (staticExtensions.some(ext => requestUrl.endsWith(ext))) {
+                return;
+            }
+
+            // Skip data URLs and blob URLs
+            if (requestUrl.startsWith('data:') || requestUrl.startsWith('blob:')) {
+                return;
+            }
+
+            // Skip same-origin requests that are likely static assets
+            if (requestUrl.startsWith(url) && staticExtensions.some(staticExt => requestUrl.toLowerCase().includes(staticExt))) {
+                return;
+            }
+
+            // Capture the request details
+            const requestData = {
+                url: requestUrl,
+                method: method,
+                headers: headers,
+                timestamp: Date.now()
+            };
+
+            networkRequests.push(requestData);
+        });
+
+        // Navigate to URL and wait for network to be idle
+        try {
+            await page.goto(url, {
+                waitUntil: 'networkidle',
+                timeout: 30000,
+            });
+        } catch (error) {
+            console.log(`Error during page navigation: ${error.message}`);
+            // Continue anyway to capture any requests that were made
+        }
+
+        // Wait a bit more to catch any delayed requests
+        await page.waitForTimeout(2000);
+
+        // Process and categorize the requests
+        const apiCalls = [];
+        for (const req of networkRequests) {
+            // Determine if this looks like an API call
+            let isApiCall = false;
+            let apiType = 'unknown';
+
+            const urlLower = req.url.toLowerCase();
+            const contentType = (req.headers['content-type'] || '').toLowerCase();
+
+            // Check for common API patterns
+            if (['/api/', '/rest/', '/graphql', '/json', '/xml'].some(pattern => urlLower.includes(pattern))) {
+                isApiCall = true;
+                apiType = 'rest';
+            } else if (req.url.endsWith('.json')) {
+                isApiCall = true;
+                apiType = 'json';
+            } else if (contentType.includes('application/json')) {
+                isApiCall = true;
+                apiType = 'json';
+            } else if (contentType.includes('application/xml')) {
+                isApiCall = true;
+                apiType = 'xml';
+            } else if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(req.method)) {
+                // These methods are typically API calls
+                isApiCall = true;
+                apiType = 'rest';
+            } else if (['api.', 'rest.', 'graphql.'].some(domain => req.url.includes(domain))) {
+                isApiCall = true;
+                apiType = 'rest';
+            }
+
+            // Filter headers (exclude common browser headers)
+            const filteredHeaders = {};
+            const excludedHeaders = ['user-agent', 'accept-encoding', 'accept-language', 'cache-control'];
+            for (const [key, value] of Object.entries(req.headers)) {
+                if (!excludedHeaders.includes(key.toLowerCase())) {
+                    filteredHeaders[key] = value;
+                }
+            }
+
+            // Include all requests but mark API calls specifically
+            const callInfo = {
+                url: req.url,
+                method: req.method,
+                is_api_call: isApiCall,
+                api_type: isApiCall ? apiType : null,
+                headers: filteredHeaders
+            };
+
+            apiCalls.push(callInfo);
+        }
+
+        // Sort by whether it's an API call (API calls first), then by URL
+        apiCalls.sort((a, b) => {
+            if (a.is_api_call !== b.is_api_call) {
+                // API calls first (true comes before false)
+                // If a is API call and b is not, a comes first (return -1)
+                // If a is not API call and b is, b comes first (return 1)
+                return a.is_api_call ? -1 : 1;
+            }
+            return a.url.localeCompare(b.url);
+        });
+
+        // Extract unique domains
+        const domains = new Set();
+        for (const call of apiCalls) {
+            try {
+                const urlObj = new URL(call.url);
+                domains.add(urlObj.hostname);
+            } catch (e) {
+                // Skip invalid URLs
+            }
+        }
+
+        const result = {
+            status: 'success',
+            url: url,
+            total_requests: apiCalls.length,
+            api_calls: apiCalls.filter(call => call.is_api_call),
+            other_requests: apiCalls.filter(call => !call.is_api_call),
+            summary: {
+                api_calls_count: apiCalls.filter(call => call.is_api_call).length,
+                other_requests_count: apiCalls.filter(call => !call.is_api_call).length,
+                unique_domains: domains.size
+            }
+        };
+
+        // Store in cache
+        const now = Date.now();
+        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
+        await dbOps.setCacheOutgoingCalls(url, result, now, expiresAt);
+        console.log(`Cached outgoing calls for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+
+        return result;
+    } catch (error) {
+        console.error(`Error capturing outgoing calls from ${url}:`, error.message);
+
+        return {
+            status: 'error',
+            url: url,
+            error: error.message
+        };
+    } finally {
+        // Always close the page, even on error
+        if (page) {
+            try {
+                await page.close();
+            } catch (closeError) {
+                console.error('Error closing page:', closeError.message);
+            }
+        }
+    }
+}
+
+async function extractResultingUrl(url, skipCache = false) {
+    let page = null;
+    try {
+        // Check cache first (unless skipCache is true)
+        if (!skipCache) {
+            const now = Date.now();
+            const cached = await dbOps.getCachedResultingUrl(url, now);
+            if (cached) {
+                console.log(`Resulting URL cache hit for: ${url}`);
+                return cached;
+            }
+        }
+
+        console.log(`Getting resulting URL for: ${url}`);
+
+        // Get browser instance
+        const browserInstance = await getBrowser();
+
+        // Create new page
+        page = await browserInstance.newPage();
+
+        // Set reasonable timeouts
+        page.setDefaultTimeout(30000);
+        page.setDefaultNavigationTimeout(30000);
+
+        // Navigate to URL and wait for network to be idle
+        let response = null;
+        try {
+            response = await page.goto(url, {
+                waitUntil: 'networkidle',
+                timeout: 30000,
+            });
+        } catch (error) {
+            console.log(`Error during page navigation: ${error.message}`);
+            // Continue anyway to get the final URL
+        }
+
+        // Get the final URL after any redirects
+        const finalUrl = page.url;
+
+        // Get response status if available
+        const statusCode = response ? response.status() : null;
+
+        // Check if there was a redirect
+        const redirected = finalUrl !== url;
+
+        const result = {
+            status: 'success',
+            original_url: url,
+            resulting_url: finalUrl,
+            status_code: statusCode,
+            redirected: redirected
+        };
+
+        // Store in cache
+        const now = Date.now();
+        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
+        await dbOps.setCacheResultingUrl(url, result, now, expiresAt);
+        console.log(`Cached resulting URL for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+
+        return result;
+    } catch (error) {
+        console.error(`Error getting resulting URL for ${url}:`, error.message);
+
+        return {
+            status: 'error',
+            original_url: url,
+            error: error.message
+        };
+    } finally {
+        // Always close the page, even on error
+        if (page) {
+            try {
+                await page.close();
+            } catch (closeError) {
+                console.error('Error closing page:', closeError.message);
+            }
+        }
+    }
+}
+
 module.exports = {
     fetchHtml,
     extractSeo,
     extractMetaTags,
+    extractOutgoingCalls,
+    extractResultingUrl,
 };