diff --git a/Dockers/playwright-node-api/database.js b/Dockers/playwright-node-api/database.js index 072a585..a835465 100644 --- a/Dockers/playwright-node-api/database.js +++ b/Dockers/playwright-node-api/database.js @@ -287,6 +287,90 @@ const dbOps = { } }, + async getCachedOutgoingCalls(url, now) { + const callsKey = `outgoing-calls:${url}`; + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'SELECT html FROM cache WHERE url = $1 AND expires_at > $2', + [callsKey, now] + ); + if (result.rows.length > 0) { + return JSON.parse(result.rows[0].html); + } + return null; + } finally { + client.release(); + } + } else { + const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?'); + const result = getCached.get(callsKey, now); + return result ? JSON.parse(result.html) : null; + } + }, + + async setCacheOutgoingCalls(url, callsData, now, expiresAt) { + const callsKey = `outgoing-calls:${url}`; + const html = JSON.stringify(callsData); + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + await client.query( + 'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4', + [callsKey, html, now, expiresAt] + ); + } finally { + client.release(); + } + } else { + const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)'); + setCache.run(callsKey, html, now, expiresAt); + } + }, + + async getCachedResultingUrl(url, now) { + const resultingUrlKey = `resulting-url:${url}`; + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'SELECT html FROM cache WHERE url = $1 AND expires_at > $2', + [resultingUrlKey, now] + ); + if (result.rows.length > 0) { + return JSON.parse(result.rows[0].html); + } + return null; + } finally { + client.release(); + } + } else { + const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?'); + const result = getCached.get(resultingUrlKey, now); + return result ? JSON.parse(result.html) : null; + } + }, + + async setCacheResultingUrl(url, resultingUrlData, now, expiresAt) { + const resultingUrlKey = `resulting-url:${url}`; + const html = JSON.stringify(resultingUrlData); + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + await client.query( + 'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4', + [resultingUrlKey, html, now, expiresAt] + ); + } finally { + client.release(); + } + } else { + const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)'); + setCache.run(resultingUrlKey, html, now, expiresAt); + } + }, + async deleteExpired(now) { if (dbType === 'postgres') { const client = await pgPool.connect(); diff --git a/Dockers/playwright-node-api/routes.js b/Dockers/playwright-node-api/routes.js index 1e1124b..691abac 100644 --- a/Dockers/playwright-node-api/routes.js +++ b/Dockers/playwright-node-api/routes.js @@ -1,5 +1,5 @@ const express = require('express'); -const { fetchHtml, extractSeo, extractMetaTags } = require('./scraper'); +const { fetchHtml, extractSeo, extractMetaTags, extractOutgoingCalls, extractResultingUrl } = require('./scraper'); const dbOps = require('./database'); const { authenticateApiKey } = require('./middleware'); const { isBrowserConnected } = require('./browser'); @@ -155,4 +155,78 @@ router.get('/meta', authenticateApiKey, async (req, res) => { } }); +// Outgoing calls capture endpoint +router.get('/outgoing-calls', authenticateApiKey, async (req, res) => { + const { url, skipCache } = req.query; + + // Validate URL parameter + if (!url) { + return res.status(400).json({ + error: 'Missing required parameter: url', + }); + } + + // Validate URL format + let urlObj; + try { + urlObj = new URL(url); + } catch (error) { + return res.status(400).json({ + error: 'Invalid URL format', + }); + } + + // Parse skipCache (accept 'true', '1', 'yes', etc.) + const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes'; + + try { + const callsData = await extractOutgoingCalls(url, shouldSkipCache); + res.json(callsData); + } catch (error) { + console.error('Outgoing calls capture error:', error); + res.status(500).json({ + status: 'error', + url: url, + error: error.message, + }); + } +}); + +// Resulting URL endpoint +router.get('/resulting-url', authenticateApiKey, async (req, res) => { + const { url, skipCache } = req.query; + + // Validate URL parameter + if (!url) { + return res.status(400).json({ + error: 'Missing required parameter: url', + }); + } + + // Validate URL format + let urlObj; + try { + urlObj = new URL(url); + } catch (error) { + return res.status(400).json({ + error: 'Invalid URL format', + }); + } + + // Parse skipCache (accept 'true', '1', 'yes', etc.) + const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes'; + + try { + const resultingUrlData = await extractResultingUrl(url, shouldSkipCache); + res.json(resultingUrlData); + } catch (error) { + console.error('Resulting URL error:', error); + res.status(500).json({ + status: 'error', + original_url: url, + error: error.message, + }); + } +}); + module.exports = router; diff --git a/Dockers/playwright-node-api/scraper.js b/Dockers/playwright-node-api/scraper.js index 717eb00..2b6d62c 100644 --- a/Dockers/playwright-node-api/scraper.js +++ b/Dockers/playwright-node-api/scraper.js @@ -289,8 +289,283 @@ async function extractMetaTags(url, skipCache = false) { } } +async function extractOutgoingCalls(url, skipCache = false) { + let page = null; + try { + // Check cache first (unless skipCache is true) + if (!skipCache) { + const now = Date.now(); + const cached = await dbOps.getCachedOutgoingCalls(url, now); + if (cached) { + console.log(`Outgoing calls cache hit for: ${url}`); + return cached; + } + } + + console.log(`Capturing outgoing calls from: ${url}`); + + // Get browser instance + const browserInstance = await getBrowser(); + + // Create new page + page = await browserInstance.newPage(); + + // Set reasonable timeouts + page.setDefaultTimeout(30000); + page.setDefaultNavigationTimeout(30000); + + // List to store all network requests + const networkRequests = []; + + // Static asset extensions to skip + const staticExtensions = ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.eot']; + + // Set up network request listener BEFORE navigation + page.on('request', (request) => { + const requestUrl = request.url(); + const method = request.method(); + const headers = request.headers(); + + // Skip static assets and common non-API requests + if (staticExtensions.some(ext => requestUrl.endsWith(ext))) { + return; + } + + // Skip data URLs and blob URLs + if (requestUrl.startsWith('data:') || requestUrl.startsWith('blob:')) { + return; + } + + // Skip same-origin requests that are likely static assets + if (requestUrl.startsWith(url) && staticExtensions.some(staticExt => requestUrl.toLowerCase().includes(staticExt))) { + return; + } + + // Capture the request details + const requestData = { + url: requestUrl, + method: method, + headers: headers, + timestamp: Date.now() + }; + + networkRequests.push(requestData); + }); + + // Navigate to URL and wait for network to be idle + try { + await page.goto(url, { + waitUntil: 'networkidle', + timeout: 30000, + }); + } catch (error) { + console.log(`Error during page navigation: ${error.message}`); + // Continue anyway to capture any requests that were made + } + + // Wait a bit more to catch any delayed requests + await page.waitForTimeout(2000); + + // Process and categorize the requests + const apiCalls = []; + for (const req of networkRequests) { + // Determine if this looks like an API call + let isApiCall = false; + let apiType = 'unknown'; + + const urlLower = req.url.toLowerCase(); + const contentType = (req.headers['content-type'] || '').toLowerCase(); + + // Check for common API patterns + if (['/api/', '/rest/', '/graphql', '/json', '/xml'].some(pattern => urlLower.includes(pattern))) { + isApiCall = true; + apiType = 'rest'; + } else if (req.url.endsWith('.json')) { + isApiCall = true; + apiType = 'json'; + } else if (contentType.includes('application/json')) { + isApiCall = true; + apiType = 'json'; + } else if (contentType.includes('application/xml')) { + isApiCall = true; + apiType = 'xml'; + } else if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(req.method)) { + // These methods are typically API calls + isApiCall = true; + apiType = 'rest'; + } else if (['api.', 'rest.', 'graphql.'].some(domain => req.url.includes(domain))) { + isApiCall = true; + apiType = 'rest'; + } + + // Filter headers (exclude common browser headers) + const filteredHeaders = {}; + const excludedHeaders = ['user-agent', 'accept-encoding', 'accept-language', 'cache-control']; + for (const [key, value] of Object.entries(req.headers)) { + if (!excludedHeaders.includes(key.toLowerCase())) { + filteredHeaders[key] = value; + } + } + + // Include all requests but mark API calls specifically + const callInfo = { + url: req.url, + method: req.method, + is_api_call: isApiCall, + api_type: isApiCall ? apiType : null, + headers: filteredHeaders + }; + + apiCalls.push(callInfo); + } + + // Sort by whether it's an API call (API calls first), then by URL + apiCalls.sort((a, b) => { + if (a.is_api_call !== b.is_api_call) { + // API calls first (true comes before false) + // If a is API call and b is not, a comes first (return -1) + // If a is not API call and b is, b comes first (return 1) + return a.is_api_call ? -1 : 1; + } + return a.url.localeCompare(b.url); + }); + + // Extract unique domains + const domains = new Set(); + for (const call of apiCalls) { + try { + const urlObj = new URL(call.url); + domains.add(urlObj.hostname); + } catch (e) { + // Skip invalid URLs + } + } + + const result = { + status: 'success', + url: url, + total_requests: apiCalls.length, + api_calls: apiCalls.filter(call => call.is_api_call), + other_requests: apiCalls.filter(call => !call.is_api_call), + summary: { + api_calls_count: apiCalls.filter(call => call.is_api_call).length, + other_requests_count: apiCalls.filter(call => !call.is_api_call).length, + unique_domains: domains.size + } + }; + + // Store in cache + const now = Date.now(); + const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000); + await dbOps.setCacheOutgoingCalls(url, result, now, expiresAt); + console.log(`Cached outgoing calls for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`); + + return result; + } catch (error) { + console.error(`Error capturing outgoing calls from ${url}:`, error.message); + + return { + status: 'error', + url: url, + error: error.message + }; + } finally { + // Always close the page, even on error + if (page) { + try { + await page.close(); + } catch (closeError) { + console.error('Error closing page:', closeError.message); + } + } + } +} + +async function extractResultingUrl(url, skipCache = false) { + let page = null; + try { + // Check cache first (unless skipCache is true) + if (!skipCache) { + const now = Date.now(); + const cached = await dbOps.getCachedResultingUrl(url, now); + if (cached) { + console.log(`Resulting URL cache hit for: ${url}`); + return cached; + } + } + + console.log(`Getting resulting URL for: ${url}`); + + // Get browser instance + const browserInstance = await getBrowser(); + + // Create new page + page = await browserInstance.newPage(); + + // Set reasonable timeouts + page.setDefaultTimeout(30000); + page.setDefaultNavigationTimeout(30000); + + // Navigate to URL and wait for network to be idle + let response = null; + try { + response = await page.goto(url, { + waitUntil: 'networkidle', + timeout: 30000, + }); + } catch (error) { + console.log(`Error during page navigation: ${error.message}`); + // Continue anyway to get the final URL + } + + // Get the final URL after any redirects + const finalUrl = page.url; + + // Get response status if available + const statusCode = response ? response.status() : null; + + // Check if there was a redirect + const redirected = finalUrl !== url; + + const result = { + status: 'success', + original_url: url, + resulting_url: finalUrl, + status_code: statusCode, + redirected: redirected + }; + + // Store in cache + const now = Date.now(); + const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000); + await dbOps.setCacheResultingUrl(url, result, now, expiresAt); + console.log(`Cached resulting URL for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`); + + return result; + } catch (error) { + console.error(`Error getting resulting URL for ${url}:`, error.message); + + return { + status: 'error', + original_url: url, + error: error.message + }; + } finally { + // Always close the page, even on error + if (page) { + try { + await page.close(); + } catch (closeError) { + console.error('Error closing page:', closeError.message); + } + } + } +} + module.exports = { fetchHtml, extractSeo, extractMetaTags, + extractOutgoingCalls, + extractResultingUrl, };