seo and meta routes

Dockers/playwright-node-api/database.js

@@ -186,6 +186,29 @@ const dbOps = {
         }
     },
 
+    async getCachedSeo(url, now) {
+        const seoKey = `seo:${url}`;
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'SELECT html FROM cache WHERE url = $1 AND expires_at > $2',
+                    [seoKey, now]
+                );
+                if (result.rows.length > 0) {
+                    return JSON.parse(result.rows[0].html);
+                }
+                return null;
+            } finally {
+                client.release();
+            }
+        } else {
+            const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?');
+            const result = getCached.get(seoKey, now);
+            return result ? JSON.parse(result.html) : null;
+        }
+    },
+
     async setCache(url, html, now, expiresAt) {
         if (dbType === 'postgres') {
             const client = await pgPool.connect();
@@ -203,6 +226,67 @@ const dbOps = {
         }
     },
 
+    async setCacheSeo(url, seoData, now, expiresAt) {
+        const seoKey = `seo:${url}`;
+        const html = JSON.stringify(seoData);
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                await client.query(
+                    'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4',
+                    [seoKey, html, now, expiresAt]
+                );
+            } finally {
+                client.release();
+            }
+        } else {
+            const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)');
+            setCache.run(seoKey, html, now, expiresAt);
+        }
+    },
+
+    async getCachedMeta(url, now) {
+        const metaKey = `meta:${url}`;
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'SELECT html FROM cache WHERE url = $1 AND expires_at > $2',
+                    [metaKey, now]
+                );
+                if (result.rows.length > 0) {
+                    return JSON.parse(result.rows[0].html);
+                }
+                return null;
+            } finally {
+                client.release();
+            }
+        } else {
+            const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?');
+            const result = getCached.get(metaKey, now);
+            return result ? JSON.parse(result.html) : null;
+        }
+    },
+
+    async setCacheMeta(url, metaData, now, expiresAt) {
+        const metaKey = `meta:${url}`;
+        const html = JSON.stringify(metaData);
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                await client.query(
+                    'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4',
+                    [metaKey, html, now, expiresAt]
+                );
+            } finally {
+                client.release();
+            }
+        } else {
+            const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)');
+            setCache.run(metaKey, html, now, expiresAt);
+        }
+    },
+
     async deleteExpired(now) {
         if (dbType === 'postgres') {
             const client = await pgPool.connect();

Dockers/playwright-node-api/routes.js

@@ -1,5 +1,5 @@
 const express = require('express');
-const { fetchHtml } = require('./scraper');
+const { fetchHtml, extractSeo, extractMetaTags } = require('./scraper');
 const dbOps = require('./database');
 const { authenticateApiKey } = require('./middleware');
 const { isBrowserConnected } = require('./browser');
@@ -81,4 +81,78 @@ router.delete('/cache', authenticateApiKey, async (req, res) => {
     }
 });
 
+// SEO extraction endpoint
+router.get('/seo', authenticateApiKey, async (req, res) => {
+    const { url, skipCache } = req.query;
+
+    // Validate URL parameter
+    if (!url) {
+        return res.status(400).json({
+            error: 'Missing required parameter: url',
+        });
+    }
+
+    // Validate URL format
+    let urlObj;
+    try {
+        urlObj = new URL(url);
+    } catch (error) {
+        return res.status(400).json({
+            error: 'Invalid URL format',
+        });
+    }
+
+    // Parse skipCache (accept 'true', '1', 'yes', etc.)
+    const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes';
+
+    try {
+        const seoData = await extractSeo(url, shouldSkipCache);
+        res.json(seoData);
+    } catch (error) {
+        console.error('SEO extraction error:', error);
+        res.status(500).json({
+            status: 'error',
+            url: url,
+            error: error.message,
+        });
+    }
+});
+
+// Meta tags extraction endpoint
+router.get('/meta', authenticateApiKey, async (req, res) => {
+    const { url, skipCache } = req.query;
+
+    // Validate URL parameter
+    if (!url) {
+        return res.status(400).json({
+            error: 'Missing required parameter: url',
+        });
+    }
+
+    // Validate URL format
+    let urlObj;
+    try {
+        urlObj = new URL(url);
+    } catch (error) {
+        return res.status(400).json({
+            error: 'Invalid URL format',
+        });
+    }
+
+    // Parse skipCache (accept 'true', '1', 'yes', etc.)
+    const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes';
+
+    try {
+        const metaData = await extractMetaTags(url, shouldSkipCache);
+        res.json(metaData);
+    } catch (error) {
+        console.error('Meta tags extraction error:', error);
+        res.status(500).json({
+            status: 'error',
+            url: url,
+            error: error.message,
+        });
+    }
+});
+
 module.exports = router;

Dockers/playwright-node-api/scraper.js

@@ -2,89 +2,133 @@ const dbOps = require('./database');
 const { getBrowser } = require('./browser');
 const config = require('./config');
 
-async function fetchHtml(url, skipCache = false) {
+/**
+ * Reusable function to navigate to a URL and wait for all network requests to complete
+ * @param {string} url - The URL to navigate to
+ * @param {object} options - Options for navigation
+ * @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle')
+ * @returns {Promise<Page>} - The Playwright page object
+ */
+async function navigateAndWait(url, options = {}) {
+    const { waitUntil = 'networkidle' } = options;
+
+    // Get browser instance
+    const browserInstance = await getBrowser();
+
+    // Create new page
+    const page = await browserInstance.newPage();
+
+    // Set reasonable timeouts
+    page.setDefaultTimeout(30000);
+    page.setDefaultNavigationTimeout(30000);
+
+    // Track network requests to ensure all are completed
+    const pendingRequests = new Set();
+
+    // Monitor network requests
+    page.on('request', (request) => {
+        const requestId = request.url();
+        pendingRequests.add(requestId);
+    });
+
+    page.on('response', (response) => {
+        const requestId = response.url();
+        pendingRequests.delete(requestId);
+    });
+
+    page.on('requestfailed', (request) => {
+        const requestId = request.url();
+        pendingRequests.delete(requestId);
+    });
+
+    // Navigate to URL
+    await page.goto(url, {
+        waitUntil: waitUntil,
+        timeout: 30000,
+    });
+
+    // If we used domcontentloaded, wait for networkidle separately
+    if (waitUntil === 'domcontentloaded') {
+        await page.waitForLoadState('networkidle', { timeout: 30000 });
+    }
+
+    // Additional wait to ensure all pending requests complete
+    const maxWaitTime = 10000; // 10 seconds max
+    const checkInterval = 100; // Check every 100ms
+    const startTime = Date.now();
+
+    while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
+        await new Promise(resolve => setTimeout(resolve, checkInterval));
+    }
+
+    if (pendingRequests.size > 0) {
+        console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
+    }
+
+    return page;
+}
+
+/**
+ * Reusable function to safely execute a scraping operation with proper cleanup
+ * @param {string} url - The URL to scrape
+ * @param {object} options - Options for the operation
+ * @param {Function} options.getCache - Function to get cached data
+ * @param {Function} options.setCache - Function to set cached data
+ * @param {Function} options.extractData - Function to extract data from the page
+ * @param {Function} options.formatResult - Function to format the result
+ * @param {boolean} options.skipCache - Whether to skip cache
+ * @param {string} options.logMessage - Log message for the operation
+ * @param {string} options.cacheKey - Cache key prefix for logging
+ * @returns {Promise<any>} - The extracted data
+ */
+async function scrapeWithCache(url, options) {
+    const {
+        getCache,
+        setCache,
+        extractData,
+        formatResult,
+        skipCache = false,
+        logMessage,
+        cacheKey = 'cache',
+        waitUntil = 'networkidle',
+    } = options;
+
     let page = null;
     try {
         // Check cache first (unless skipCache is true)
-        if (!skipCache) {
+        if (!skipCache && getCache) {
             const now = Date.now();
-            const cached = await dbOps.getCached(url, now);
+            const cached = await getCache(url, now);
             if (cached) {
-                console.log(`Cache hit for: ${url}`);
+                console.log(`${cacheKey} hit for: ${url}`);
                 return cached;
             }
         }
 
-        console.log(`Fetching: ${url} (skipCache: ${skipCache})`);
+        if (logMessage) {
-
+            console.log(logMessage);
-        // Get browser instance
-        const browserInstance = await getBrowser();
-
-        // Create new page
-        page = await browserInstance.newPage();
-
-        // Set reasonable timeouts
-        page.setDefaultTimeout(30000);
-        page.setDefaultNavigationTimeout(30000);
-
-        // Track network requests to ensure all are completed
-        const pendingRequests = new Set();
-        const finishedRequests = new Set();
-
-        // Monitor network requests
-        page.on('request', (request) => {
-            const requestId = request.url();
-            pendingRequests.add(requestId);
-        });
-
-        page.on('response', (response) => {
-            const requestId = response.url();
-            finishedRequests.add(requestId);
-            pendingRequests.delete(requestId);
-        });
-
-        page.on('requestfailed', (request) => {
-            const requestId = request.url();
-            pendingRequests.delete(requestId);
-        });
-
-        // Navigate to URL
-        await page.goto(url, {
-            waitUntil: 'domcontentloaded',
-            timeout: 30000,
-        });
-
-        // Wait for network to be idle (no requests for 500ms)
-        await page.waitForLoadState('networkidle', { timeout: 30000 });
-
-        // Additional wait to ensure all pending requests complete
-        const maxWaitTime = 10000; // 10 seconds max
-        const checkInterval = 100; // Check every 100ms
-        const startTime = Date.now();
-
-        while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
-            await new Promise(resolve => setTimeout(resolve, checkInterval));
         }
 
-        if (pendingRequests.size > 0) {
+        // Navigate and wait for network
-            console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
+        page = await navigateAndWait(url, { waitUntil });
-        }
 
-        // Get full HTML
+        // Extract data using the provided function
-        const html = await page.content();
+        const extractedData = await extractData(page);
+
+        // Format the result
+        const result = formatResult ? formatResult(extractedData, url) : extractedData;
 
         // Store in cache
-        const now = Date.now();
+        if (setCache) {
-        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
+            const now = Date.now();
-        await dbOps.setCache(url, html, now, expiresAt);
+            const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
-        console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+            await setCache(url, result, now, expiresAt);
+            console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+        }
 
-        return html;
+        return result;
     } catch (error) {
-        console.error(`Error fetching ${url}:`, error.message);
+        console.error(`Error processing ${url}:`, error.message);
-
-        // If cache exists and we had an error, optionally return stale cache
-        // For now, we'll just throw the error
         throw error;
     } finally {
         // Always close the page, even on error
@@ -98,6 +142,155 @@ async function fetchHtml(url, skipCache = false) {
     }
 }
 
+async function fetchHtml(url, skipCache = false) {
+    try {
+        return await scrapeWithCache(url, {
+            getCache: async (url, now) => await dbOps.getCached(url, now),
+            setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt),
+            extractData: async (page) => await page.content(),
+            formatResult: (html) => html,
+            skipCache,
+            logMessage: `Fetching: ${url} (skipCache: ${skipCache})`,
+            cacheKey: 'Cache',
+            waitUntil: 'domcontentloaded',
+        });
+    } catch (error) {
+        console.error(`Error fetching ${url}:`, error.message);
+        throw error;
+    }
+}
+
+async function extractSeo(url, skipCache = false) {
+    try {
+        return await scrapeWithCache(url, {
+            getCache: async (url, now) => await dbOps.getCachedSeo(url, now),
+            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt),
+            extractData: async (page) => {
+                return await page.evaluate(() => {
+                    const data = {
+                        title: document.title || '',
+                        description: '',
+                        canonical: '',
+                        h1: [],
+                        h2: [],
+                        images: 0,
+                        links: 0
+                    };
+
+                    // Get meta description
+                    const metaDescription = document.querySelector('meta[name="description"]');
+                    if (metaDescription) {
+                        data.description = metaDescription.getAttribute('content') || '';
+                    }
+
+                    // Get canonical link
+                    const canonicalLink = document.querySelector('link[rel="canonical"]');
+                    if (canonicalLink) {
+                        data.canonical = canonicalLink.getAttribute('href') || '';
+                    }
+
+                    // Get h1 tags
+                    document.querySelectorAll('h1').forEach(h1 => {
+                        const text = h1.innerText.trim();
+                        if (text) data.h1.push(text);
+                    });
+
+                    // Get h2 tags
+                    document.querySelectorAll('h2').forEach(h2 => {
+                        const text = h2.innerText.trim();
+                        if (text) data.h2.push(text);
+                    });
+
+                    // Count images
+                    data.images = document.querySelectorAll('img').length;
+
+                    // Count links
+                    data.links = document.querySelectorAll('a').length;
+
+                    return data;
+                });
+            },
+            formatResult: (seoData, url) => ({
+                status: 'success',
+                url: url,
+                seo: seoData
+            }),
+            skipCache,
+            logMessage: `Extracting SEO from: ${url}`,
+            cacheKey: 'SEO cache',
+            waitUntil: 'networkidle',
+        });
+    } catch (error) {
+        console.error(`Error extracting SEO from ${url}:`, error.message);
+        return {
+            status: 'error',
+            url: url,
+            error: error.message
+        };
+    }
+}
+
+async function extractMetaTags(url, skipCache = false) {
+    try {
+        return await scrapeWithCache(url, {
+            getCache: async (url, now) => await dbOps.getCachedMeta(url, now),
+            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt),
+            extractData: async (page) => {
+                return await page.evaluate(() => {
+                    const data = {
+                        meta_tags: [],
+                        open_graph: {},
+                        twitter_card: {},
+                        title: document.title || ''
+                    };
+
+                    // Extract all meta tags
+                    document.querySelectorAll('meta').forEach(meta => {
+                        const attributes = {};
+                        for (let attr of meta.attributes) {
+                            attributes[attr.name] = attr.value;
+                        }
+                        data.meta_tags.push(attributes);
+                    });
+
+                    // Extract Open Graph tags
+                    document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
+                        data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
+                    });
+
+                    // Extract Twitter card tags
+                    document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
+                        data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
+                    });
+
+                    return data;
+                });
+            },
+            formatResult: (metaData, url) => ({
+                status: 'success',
+                url: url,
+                meta_tags: metaData.meta_tags,
+                open_graph: metaData.open_graph,
+                twitter_card: metaData.twitter_card,
+                title: metaData.title
+            }),
+            skipCache,
+            logMessage: `Extracting meta tags from: ${url}`,
+            cacheKey: 'Meta tags cache',
+            waitUntil: 'networkidle',
+        });
+    } catch (error) {
+        console.error(`Error extracting meta tags from ${url}:`, error.message);
+        return {
+            status: 'error',
+            url: url,
+            error: error.message
+        };
+    }
+}
+
 module.exports = {
     fetchHtml,
+    extractSeo,
+    extractMetaTags,
 };