diff --git a/Dockers/playwright-node-api/database.js b/Dockers/playwright-node-api/database.js index d95797f..072a585 100644 --- a/Dockers/playwright-node-api/database.js +++ b/Dockers/playwright-node-api/database.js @@ -186,6 +186,29 @@ const dbOps = { } }, + async getCachedSeo(url, now) { + const seoKey = `seo:${url}`; + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'SELECT html FROM cache WHERE url = $1 AND expires_at > $2', + [seoKey, now] + ); + if (result.rows.length > 0) { + return JSON.parse(result.rows[0].html); + } + return null; + } finally { + client.release(); + } + } else { + const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?'); + const result = getCached.get(seoKey, now); + return result ? JSON.parse(result.html) : null; + } + }, + async setCache(url, html, now, expiresAt) { if (dbType === 'postgres') { const client = await pgPool.connect(); @@ -203,6 +226,67 @@ const dbOps = { } }, + async setCacheSeo(url, seoData, now, expiresAt) { + const seoKey = `seo:${url}`; + const html = JSON.stringify(seoData); + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + await client.query( + 'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4', + [seoKey, html, now, expiresAt] + ); + } finally { + client.release(); + } + } else { + const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)'); + setCache.run(seoKey, html, now, expiresAt); + } + }, + + async getCachedMeta(url, now) { + const metaKey = `meta:${url}`; + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'SELECT html FROM cache WHERE url = $1 AND expires_at > $2', + [metaKey, now] + ); + if (result.rows.length > 0) { + return JSON.parse(result.rows[0].html); + } + return null; + } finally { + client.release(); + } + } else { + const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?'); + const result = getCached.get(metaKey, now); + return result ? JSON.parse(result.html) : null; + } + }, + + async setCacheMeta(url, metaData, now, expiresAt) { + const metaKey = `meta:${url}`; + const html = JSON.stringify(metaData); + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + await client.query( + 'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4', + [metaKey, html, now, expiresAt] + ); + } finally { + client.release(); + } + } else { + const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)'); + setCache.run(metaKey, html, now, expiresAt); + } + }, + async deleteExpired(now) { if (dbType === 'postgres') { const client = await pgPool.connect(); diff --git a/Dockers/playwright-node-api/routes.js b/Dockers/playwright-node-api/routes.js index 203a5ca..1e1124b 100644 --- a/Dockers/playwright-node-api/routes.js +++ b/Dockers/playwright-node-api/routes.js @@ -1,5 +1,5 @@ const express = require('express'); -const { fetchHtml } = require('./scraper'); +const { fetchHtml, extractSeo, extractMetaTags } = require('./scraper'); const dbOps = require('./database'); const { authenticateApiKey } = require('./middleware'); const { isBrowserConnected } = require('./browser'); @@ -81,4 +81,78 @@ router.delete('/cache', authenticateApiKey, async (req, res) => { } }); +// SEO extraction endpoint +router.get('/seo', authenticateApiKey, async (req, res) => { + const { url, skipCache } = req.query; + + // Validate URL parameter + if (!url) { + return res.status(400).json({ + error: 'Missing required parameter: url', + }); + } + + // Validate URL format + let urlObj; + try { + urlObj = new URL(url); + } catch (error) { + return res.status(400).json({ + error: 'Invalid URL format', + }); + } + + // Parse skipCache (accept 'true', '1', 'yes', etc.) + const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes'; + + try { + const seoData = await extractSeo(url, shouldSkipCache); + res.json(seoData); + } catch (error) { + console.error('SEO extraction error:', error); + res.status(500).json({ + status: 'error', + url: url, + error: error.message, + }); + } +}); + +// Meta tags extraction endpoint +router.get('/meta', authenticateApiKey, async (req, res) => { + const { url, skipCache } = req.query; + + // Validate URL parameter + if (!url) { + return res.status(400).json({ + error: 'Missing required parameter: url', + }); + } + + // Validate URL format + let urlObj; + try { + urlObj = new URL(url); + } catch (error) { + return res.status(400).json({ + error: 'Invalid URL format', + }); + } + + // Parse skipCache (accept 'true', '1', 'yes', etc.) + const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes'; + + try { + const metaData = await extractMetaTags(url, shouldSkipCache); + res.json(metaData); + } catch (error) { + console.error('Meta tags extraction error:', error); + res.status(500).json({ + status: 'error', + url: url, + error: error.message, + }); + } +}); + module.exports = router; diff --git a/Dockers/playwright-node-api/scraper.js b/Dockers/playwright-node-api/scraper.js index ae0a643..717eb00 100644 --- a/Dockers/playwright-node-api/scraper.js +++ b/Dockers/playwright-node-api/scraper.js @@ -2,89 +2,133 @@ const dbOps = require('./database'); const { getBrowser } = require('./browser'); const config = require('./config'); -async function fetchHtml(url, skipCache = false) { +/** + * Reusable function to navigate to a URL and wait for all network requests to complete + * @param {string} url - The URL to navigate to + * @param {object} options - Options for navigation + * @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle') + * @returns {Promise} - The Playwright page object + */ +async function navigateAndWait(url, options = {}) { + const { waitUntil = 'networkidle' } = options; + + // Get browser instance + const browserInstance = await getBrowser(); + + // Create new page + const page = await browserInstance.newPage(); + + // Set reasonable timeouts + page.setDefaultTimeout(30000); + page.setDefaultNavigationTimeout(30000); + + // Track network requests to ensure all are completed + const pendingRequests = new Set(); + + // Monitor network requests + page.on('request', (request) => { + const requestId = request.url(); + pendingRequests.add(requestId); + }); + + page.on('response', (response) => { + const requestId = response.url(); + pendingRequests.delete(requestId); + }); + + page.on('requestfailed', (request) => { + const requestId = request.url(); + pendingRequests.delete(requestId); + }); + + // Navigate to URL + await page.goto(url, { + waitUntil: waitUntil, + timeout: 30000, + }); + + // If we used domcontentloaded, wait for networkidle separately + if (waitUntil === 'domcontentloaded') { + await page.waitForLoadState('networkidle', { timeout: 30000 }); + } + + // Additional wait to ensure all pending requests complete + const maxWaitTime = 10000; // 10 seconds max + const checkInterval = 100; // Check every 100ms + const startTime = Date.now(); + + while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) { + await new Promise(resolve => setTimeout(resolve, checkInterval)); + } + + if (pendingRequests.size > 0) { + console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`); + } + + return page; +} + +/** + * Reusable function to safely execute a scraping operation with proper cleanup + * @param {string} url - The URL to scrape + * @param {object} options - Options for the operation + * @param {Function} options.getCache - Function to get cached data + * @param {Function} options.setCache - Function to set cached data + * @param {Function} options.extractData - Function to extract data from the page + * @param {Function} options.formatResult - Function to format the result + * @param {boolean} options.skipCache - Whether to skip cache + * @param {string} options.logMessage - Log message for the operation + * @param {string} options.cacheKey - Cache key prefix for logging + * @returns {Promise} - The extracted data + */ +async function scrapeWithCache(url, options) { + const { + getCache, + setCache, + extractData, + formatResult, + skipCache = false, + logMessage, + cacheKey = 'cache', + waitUntil = 'networkidle', + } = options; + let page = null; try { // Check cache first (unless skipCache is true) - if (!skipCache) { + if (!skipCache && getCache) { const now = Date.now(); - const cached = await dbOps.getCached(url, now); + const cached = await getCache(url, now); if (cached) { - console.log(`Cache hit for: ${url}`); + console.log(`${cacheKey} hit for: ${url}`); return cached; } } - console.log(`Fetching: ${url} (skipCache: ${skipCache})`); - - // Get browser instance - const browserInstance = await getBrowser(); - - // Create new page - page = await browserInstance.newPage(); - - // Set reasonable timeouts - page.setDefaultTimeout(30000); - page.setDefaultNavigationTimeout(30000); - - // Track network requests to ensure all are completed - const pendingRequests = new Set(); - const finishedRequests = new Set(); - - // Monitor network requests - page.on('request', (request) => { - const requestId = request.url(); - pendingRequests.add(requestId); - }); - - page.on('response', (response) => { - const requestId = response.url(); - finishedRequests.add(requestId); - pendingRequests.delete(requestId); - }); - - page.on('requestfailed', (request) => { - const requestId = request.url(); - pendingRequests.delete(requestId); - }); - - // Navigate to URL - await page.goto(url, { - waitUntil: 'domcontentloaded', - timeout: 30000, - }); - - // Wait for network to be idle (no requests for 500ms) - await page.waitForLoadState('networkidle', { timeout: 30000 }); - - // Additional wait to ensure all pending requests complete - const maxWaitTime = 10000; // 10 seconds max - const checkInterval = 100; // Check every 100ms - const startTime = Date.now(); - - while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) { - await new Promise(resolve => setTimeout(resolve, checkInterval)); + if (logMessage) { + console.log(logMessage); } - if (pendingRequests.size > 0) { - console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`); - } + // Navigate and wait for network + page = await navigateAndWait(url, { waitUntil }); - // Get full HTML - const html = await page.content(); + // Extract data using the provided function + const extractedData = await extractData(page); + + // Format the result + const result = formatResult ? formatResult(extractedData, url) : extractedData; // Store in cache - const now = Date.now(); - const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000); - await dbOps.setCache(url, html, now, expiresAt); - console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`); + if (setCache) { + const now = Date.now(); + const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000); + await setCache(url, result, now, expiresAt); + console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`); + } - return html; + return result; } catch (error) { - console.error(`Error fetching ${url}:`, error.message); - - // If cache exists and we had an error, optionally return stale cache - // For now, we'll just throw the error + console.error(`Error processing ${url}:`, error.message); throw error; } finally { // Always close the page, even on error @@ -98,6 +142,155 @@ async function fetchHtml(url, skipCache = false) { } } +async function fetchHtml(url, skipCache = false) { + try { + return await scrapeWithCache(url, { + getCache: async (url, now) => await dbOps.getCached(url, now), + setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt), + extractData: async (page) => await page.content(), + formatResult: (html) => html, + skipCache, + logMessage: `Fetching: ${url} (skipCache: ${skipCache})`, + cacheKey: 'Cache', + waitUntil: 'domcontentloaded', + }); + } catch (error) { + console.error(`Error fetching ${url}:`, error.message); + throw error; + } +} + +async function extractSeo(url, skipCache = false) { + try { + return await scrapeWithCache(url, { + getCache: async (url, now) => await dbOps.getCachedSeo(url, now), + setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt), + extractData: async (page) => { + return await page.evaluate(() => { + const data = { + title: document.title || '', + description: '', + canonical: '', + h1: [], + h2: [], + images: 0, + links: 0 + }; + + // Get meta description + const metaDescription = document.querySelector('meta[name="description"]'); + if (metaDescription) { + data.description = metaDescription.getAttribute('content') || ''; + } + + // Get canonical link + const canonicalLink = document.querySelector('link[rel="canonical"]'); + if (canonicalLink) { + data.canonical = canonicalLink.getAttribute('href') || ''; + } + + // Get h1 tags + document.querySelectorAll('h1').forEach(h1 => { + const text = h1.innerText.trim(); + if (text) data.h1.push(text); + }); + + // Get h2 tags + document.querySelectorAll('h2').forEach(h2 => { + const text = h2.innerText.trim(); + if (text) data.h2.push(text); + }); + + // Count images + data.images = document.querySelectorAll('img').length; + + // Count links + data.links = document.querySelectorAll('a').length; + + return data; + }); + }, + formatResult: (seoData, url) => ({ + status: 'success', + url: url, + seo: seoData + }), + skipCache, + logMessage: `Extracting SEO from: ${url}`, + cacheKey: 'SEO cache', + waitUntil: 'networkidle', + }); + } catch (error) { + console.error(`Error extracting SEO from ${url}:`, error.message); + return { + status: 'error', + url: url, + error: error.message + }; + } +} + +async function extractMetaTags(url, skipCache = false) { + try { + return await scrapeWithCache(url, { + getCache: async (url, now) => await dbOps.getCachedMeta(url, now), + setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt), + extractData: async (page) => { + return await page.evaluate(() => { + const data = { + meta_tags: [], + open_graph: {}, + twitter_card: {}, + title: document.title || '' + }; + + // Extract all meta tags + document.querySelectorAll('meta').forEach(meta => { + const attributes = {}; + for (let attr of meta.attributes) { + attributes[attr.name] = attr.value; + } + data.meta_tags.push(attributes); + }); + + // Extract Open Graph tags + document.querySelectorAll('meta[property^="og:"]').forEach(meta => { + data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || ''; + }); + + // Extract Twitter card tags + document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => { + data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || ''; + }); + + return data; + }); + }, + formatResult: (metaData, url) => ({ + status: 'success', + url: url, + meta_tags: metaData.meta_tags, + open_graph: metaData.open_graph, + twitter_card: metaData.twitter_card, + title: metaData.title + }), + skipCache, + logMessage: `Extracting meta tags from: ${url}`, + cacheKey: 'Meta tags cache', + waitUntil: 'networkidle', + }); + } catch (error) { + console.error(`Error extracting meta tags from ${url}:`, error.message); + return { + status: 'error', + url: url, + error: error.message + }; + } +} + module.exports = { fetchHtml, + extractSeo, + extractMetaTags, };