seo and meta routes

Dockers/playwright-node-api/scraper.js

@@ -2,89 +2,133 @@ const dbOps = require('./database');
 const { getBrowser } = require('./browser');
 const config = require('./config');
 
-async function fetchHtml(url, skipCache = false) {
+/**
+ * Reusable function to navigate to a URL and wait for all network requests to complete
+ * @param {string} url - The URL to navigate to
+ * @param {object} options - Options for navigation
+ * @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle')
+ * @returns {Promise<Page>} - The Playwright page object
+ */
+async function navigateAndWait(url, options = {}) {
+    const { waitUntil = 'networkidle' } = options;
+
+    // Get browser instance
+    const browserInstance = await getBrowser();
+
+    // Create new page
+    const page = await browserInstance.newPage();
+
+    // Set reasonable timeouts
+    page.setDefaultTimeout(30000);
+    page.setDefaultNavigationTimeout(30000);
+
+    // Track network requests to ensure all are completed
+    const pendingRequests = new Set();
+
+    // Monitor network requests
+    page.on('request', (request) => {
+        const requestId = request.url();
+        pendingRequests.add(requestId);
+    });
+
+    page.on('response', (response) => {
+        const requestId = response.url();
+        pendingRequests.delete(requestId);
+    });
+
+    page.on('requestfailed', (request) => {
+        const requestId = request.url();
+        pendingRequests.delete(requestId);
+    });
+
+    // Navigate to URL
+    await page.goto(url, {
+        waitUntil: waitUntil,
+        timeout: 30000,
+    });
+
+    // If we used domcontentloaded, wait for networkidle separately
+    if (waitUntil === 'domcontentloaded') {
+        await page.waitForLoadState('networkidle', { timeout: 30000 });
+    }
+
+    // Additional wait to ensure all pending requests complete
+    const maxWaitTime = 10000; // 10 seconds max
+    const checkInterval = 100; // Check every 100ms
+    const startTime = Date.now();
+
+    while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
+        await new Promise(resolve => setTimeout(resolve, checkInterval));
+    }
+
+    if (pendingRequests.size > 0) {
+        console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
+    }
+
+    return page;
+}
+
+/**
+ * Reusable function to safely execute a scraping operation with proper cleanup
+ * @param {string} url - The URL to scrape
+ * @param {object} options - Options for the operation
+ * @param {Function} options.getCache - Function to get cached data
+ * @param {Function} options.setCache - Function to set cached data
+ * @param {Function} options.extractData - Function to extract data from the page
+ * @param {Function} options.formatResult - Function to format the result
+ * @param {boolean} options.skipCache - Whether to skip cache
+ * @param {string} options.logMessage - Log message for the operation
+ * @param {string} options.cacheKey - Cache key prefix for logging
+ * @returns {Promise<any>} - The extracted data
+ */
+async function scrapeWithCache(url, options) {
+    const {
+        getCache,
+        setCache,
+        extractData,
+        formatResult,
+        skipCache = false,
+        logMessage,
+        cacheKey = 'cache',
+        waitUntil = 'networkidle',
+    } = options;
+
     let page = null;
     try {
         // Check cache first (unless skipCache is true)
-        if (!skipCache) {
+        if (!skipCache && getCache) {
             const now = Date.now();
-            const cached = await dbOps.getCached(url, now);
+            const cached = await getCache(url, now);
             if (cached) {
-                console.log(`Cache hit for: ${url}`);
+                console.log(`${cacheKey} hit for: ${url}`);
                 return cached;
             }
         }
 
-        console.log(`Fetching: ${url} (skipCache: ${skipCache})`);
-
-        // Get browser instance
-        const browserInstance = await getBrowser();
-
-        // Create new page
-        page = await browserInstance.newPage();
-
-        // Set reasonable timeouts
-        page.setDefaultTimeout(30000);
-        page.setDefaultNavigationTimeout(30000);
-
-        // Track network requests to ensure all are completed
-        const pendingRequests = new Set();
-        const finishedRequests = new Set();
-
-        // Monitor network requests
-        page.on('request', (request) => {
-            const requestId = request.url();
-            pendingRequests.add(requestId);
-        });
-
-        page.on('response', (response) => {
-            const requestId = response.url();
-            finishedRequests.add(requestId);
-            pendingRequests.delete(requestId);
-        });
-
-        page.on('requestfailed', (request) => {
-            const requestId = request.url();
-            pendingRequests.delete(requestId);
-        });
-
-        // Navigate to URL
-        await page.goto(url, {
-            waitUntil: 'domcontentloaded',
-            timeout: 30000,
-        });
-
-        // Wait for network to be idle (no requests for 500ms)
-        await page.waitForLoadState('networkidle', { timeout: 30000 });
-
-        // Additional wait to ensure all pending requests complete
-        const maxWaitTime = 10000; // 10 seconds max
-        const checkInterval = 100; // Check every 100ms
-        const startTime = Date.now();
-
-        while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
-            await new Promise(resolve => setTimeout(resolve, checkInterval));
+        if (logMessage) {
+            console.log(logMessage);
         }
 
-        if (pendingRequests.size > 0) {
-            console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
-        }
+        // Navigate and wait for network
+        page = await navigateAndWait(url, { waitUntil });
 
-        // Get full HTML
-        const html = await page.content();
+        // Extract data using the provided function
+        const extractedData = await extractData(page);
+
+        // Format the result
+        const result = formatResult ? formatResult(extractedData, url) : extractedData;
 
         // Store in cache
-        const now = Date.now();
-        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
-        await dbOps.setCache(url, html, now, expiresAt);
-        console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+        if (setCache) {
+            const now = Date.now();
+            const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
+            await setCache(url, result, now, expiresAt);
+            console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+        }
 
-        return html;
+        return result;
     } catch (error) {
-        console.error(`Error fetching ${url}:`, error.message);
-
-        // If cache exists and we had an error, optionally return stale cache
-        // For now, we'll just throw the error
+        console.error(`Error processing ${url}:`, error.message);
         throw error;
     } finally {
         // Always close the page, even on error
@@ -98,6 +142,155 @@ async function fetchHtml(url, skipCache = false) {
     }
 }
 
+async function fetchHtml(url, skipCache = false) {
+    try {
+        return await scrapeWithCache(url, {
+            getCache: async (url, now) => await dbOps.getCached(url, now),
+            setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt),
+            extractData: async (page) => await page.content(),
+            formatResult: (html) => html,
+            skipCache,
+            logMessage: `Fetching: ${url} (skipCache: ${skipCache})`,
+            cacheKey: 'Cache',
+            waitUntil: 'domcontentloaded',
+        });
+    } catch (error) {
+        console.error(`Error fetching ${url}:`, error.message);
+        throw error;
+    }
+}
+
+async function extractSeo(url, skipCache = false) {
+    try {
+        return await scrapeWithCache(url, {
+            getCache: async (url, now) => await dbOps.getCachedSeo(url, now),
+            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt),
+            extractData: async (page) => {
+                return await page.evaluate(() => {
+                    const data = {
+                        title: document.title || '',
+                        description: '',
+                        canonical: '',
+                        h1: [],
+                        h2: [],
+                        images: 0,
+                        links: 0
+                    };
+
+                    // Get meta description
+                    const metaDescription = document.querySelector('meta[name="description"]');
+                    if (metaDescription) {
+                        data.description = metaDescription.getAttribute('content') || '';
+                    }
+
+                    // Get canonical link
+                    const canonicalLink = document.querySelector('link[rel="canonical"]');
+                    if (canonicalLink) {
+                        data.canonical = canonicalLink.getAttribute('href') || '';
+                    }
+
+                    // Get h1 tags
+                    document.querySelectorAll('h1').forEach(h1 => {
+                        const text = h1.innerText.trim();
+                        if (text) data.h1.push(text);
+                    });
+
+                    // Get h2 tags
+                    document.querySelectorAll('h2').forEach(h2 => {
+                        const text = h2.innerText.trim();
+                        if (text) data.h2.push(text);
+                    });
+
+                    // Count images
+                    data.images = document.querySelectorAll('img').length;
+
+                    // Count links
+                    data.links = document.querySelectorAll('a').length;
+
+                    return data;
+                });
+            },
+            formatResult: (seoData, url) => ({
+                status: 'success',
+                url: url,
+                seo: seoData
+            }),
+            skipCache,
+            logMessage: `Extracting SEO from: ${url}`,
+            cacheKey: 'SEO cache',
+            waitUntil: 'networkidle',
+        });
+    } catch (error) {
+        console.error(`Error extracting SEO from ${url}:`, error.message);
+        return {
+            status: 'error',
+            url: url,
+            error: error.message
+        };
+    }
+}
+
+async function extractMetaTags(url, skipCache = false) {
+    try {
+        return await scrapeWithCache(url, {
+            getCache: async (url, now) => await dbOps.getCachedMeta(url, now),
+            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt),
+            extractData: async (page) => {
+                return await page.evaluate(() => {
+                    const data = {
+                        meta_tags: [],
+                        open_graph: {},
+                        twitter_card: {},
+                        title: document.title || ''
+                    };
+
+                    // Extract all meta tags
+                    document.querySelectorAll('meta').forEach(meta => {
+                        const attributes = {};
+                        for (let attr of meta.attributes) {
+                            attributes[attr.name] = attr.value;
+                        }
+                        data.meta_tags.push(attributes);
+                    });
+
+                    // Extract Open Graph tags
+                    document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
+                        data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
+                    });
+
+                    // Extract Twitter card tags
+                    document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
+                        data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
+                    });
+
+                    return data;
+                });
+            },
+            formatResult: (metaData, url) => ({
+                status: 'success',
+                url: url,
+                meta_tags: metaData.meta_tags,
+                open_graph: metaData.open_graph,
+                twitter_card: metaData.twitter_card,
+                title: metaData.title
+            }),
+            skipCache,
+            logMessage: `Extracting meta tags from: ${url}`,
+            cacheKey: 'Meta tags cache',
+            waitUntil: 'networkidle',
+        });
+    } catch (error) {
+        console.error(`Error extracting meta tags from ${url}:`, error.message);
+        return {
+            status: 'error',
+            url: url,
+            error: error.message
+        };
+    }
+}
+
 module.exports = {
     fetchHtml,
+    extractSeo,
+    extractMetaTags,
 };