297 lines
10 KiB
JavaScript
297 lines
10 KiB
JavaScript
const dbOps = require('./database');
|
|
const { getBrowser } = require('./browser');
|
|
const config = require('./config');
|
|
|
|
/**
|
|
* Reusable function to navigate to a URL and wait for all network requests to complete
|
|
* @param {string} url - The URL to navigate to
|
|
* @param {object} options - Options for navigation
|
|
* @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle')
|
|
* @returns {Promise<Page>} - The Playwright page object
|
|
*/
|
|
async function navigateAndWait(url, options = {}) {
|
|
const { waitUntil = 'networkidle' } = options;
|
|
|
|
// Get browser instance
|
|
const browserInstance = await getBrowser();
|
|
|
|
// Create new page
|
|
const page = await browserInstance.newPage();
|
|
|
|
// Set reasonable timeouts
|
|
page.setDefaultTimeout(30000);
|
|
page.setDefaultNavigationTimeout(30000);
|
|
|
|
// Track network requests to ensure all are completed
|
|
const pendingRequests = new Set();
|
|
|
|
// Monitor network requests
|
|
page.on('request', (request) => {
|
|
const requestId = request.url();
|
|
pendingRequests.add(requestId);
|
|
});
|
|
|
|
page.on('response', (response) => {
|
|
const requestId = response.url();
|
|
pendingRequests.delete(requestId);
|
|
});
|
|
|
|
page.on('requestfailed', (request) => {
|
|
const requestId = request.url();
|
|
pendingRequests.delete(requestId);
|
|
});
|
|
|
|
// Navigate to URL
|
|
await page.goto(url, {
|
|
waitUntil: waitUntil,
|
|
timeout: 30000,
|
|
});
|
|
|
|
// If we used domcontentloaded, wait for networkidle separately
|
|
if (waitUntil === 'domcontentloaded') {
|
|
await page.waitForLoadState('networkidle', { timeout: 30000 });
|
|
}
|
|
|
|
// Additional wait to ensure all pending requests complete
|
|
const maxWaitTime = 10000; // 10 seconds max
|
|
const checkInterval = 100; // Check every 100ms
|
|
const startTime = Date.now();
|
|
|
|
while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
|
|
await new Promise(resolve => setTimeout(resolve, checkInterval));
|
|
}
|
|
|
|
if (pendingRequests.size > 0) {
|
|
console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
|
|
}
|
|
|
|
return page;
|
|
}
|
|
|
|
/**
|
|
* Reusable function to safely execute a scraping operation with proper cleanup
|
|
* @param {string} url - The URL to scrape
|
|
* @param {object} options - Options for the operation
|
|
* @param {Function} options.getCache - Function to get cached data
|
|
* @param {Function} options.setCache - Function to set cached data
|
|
* @param {Function} options.extractData - Function to extract data from the page
|
|
* @param {Function} options.formatResult - Function to format the result
|
|
* @param {boolean} options.skipCache - Whether to skip cache
|
|
* @param {string} options.logMessage - Log message for the operation
|
|
* @param {string} options.cacheKey - Cache key prefix for logging
|
|
* @returns {Promise<any>} - The extracted data
|
|
*/
|
|
async function scrapeWithCache(url, options) {
|
|
const {
|
|
getCache,
|
|
setCache,
|
|
extractData,
|
|
formatResult,
|
|
skipCache = false,
|
|
logMessage,
|
|
cacheKey = 'cache',
|
|
waitUntil = 'networkidle',
|
|
} = options;
|
|
|
|
let page = null;
|
|
try {
|
|
// Check cache first (unless skipCache is true)
|
|
if (!skipCache && getCache) {
|
|
const now = Date.now();
|
|
const cached = await getCache(url, now);
|
|
if (cached) {
|
|
console.log(`${cacheKey} hit for: ${url}`);
|
|
return cached;
|
|
}
|
|
}
|
|
|
|
if (logMessage) {
|
|
console.log(logMessage);
|
|
}
|
|
|
|
// Navigate and wait for network
|
|
page = await navigateAndWait(url, { waitUntil });
|
|
|
|
// Extract data using the provided function
|
|
const extractedData = await extractData(page);
|
|
|
|
// Format the result
|
|
const result = formatResult ? formatResult(extractedData, url) : extractedData;
|
|
|
|
// Store in cache
|
|
if (setCache) {
|
|
const now = Date.now();
|
|
const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
|
|
await setCache(url, result, now, expiresAt);
|
|
console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
|
|
}
|
|
|
|
return result;
|
|
} catch (error) {
|
|
console.error(`Error processing ${url}:`, error.message);
|
|
throw error;
|
|
} finally {
|
|
// Always close the page, even on error
|
|
if (page) {
|
|
try {
|
|
await page.close();
|
|
} catch (closeError) {
|
|
console.error('Error closing page:', closeError.message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function fetchHtml(url, skipCache = false) {
|
|
try {
|
|
return await scrapeWithCache(url, {
|
|
getCache: async (url, now) => await dbOps.getCached(url, now),
|
|
setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt),
|
|
extractData: async (page) => await page.content(),
|
|
formatResult: (html) => html,
|
|
skipCache,
|
|
logMessage: `Fetching: ${url} (skipCache: ${skipCache})`,
|
|
cacheKey: 'Cache',
|
|
waitUntil: 'domcontentloaded',
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error fetching ${url}:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function extractSeo(url, skipCache = false) {
|
|
try {
|
|
return await scrapeWithCache(url, {
|
|
getCache: async (url, now) => await dbOps.getCachedSeo(url, now),
|
|
setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt),
|
|
extractData: async (page) => {
|
|
return await page.evaluate(() => {
|
|
const data = {
|
|
title: document.title || '',
|
|
description: '',
|
|
canonical: '',
|
|
h1: [],
|
|
h2: [],
|
|
images: 0,
|
|
links: 0
|
|
};
|
|
|
|
// Get meta description
|
|
const metaDescription = document.querySelector('meta[name="description"]');
|
|
if (metaDescription) {
|
|
data.description = metaDescription.getAttribute('content') || '';
|
|
}
|
|
|
|
// Get canonical link
|
|
const canonicalLink = document.querySelector('link[rel="canonical"]');
|
|
if (canonicalLink) {
|
|
data.canonical = canonicalLink.getAttribute('href') || '';
|
|
}
|
|
|
|
// Get h1 tags
|
|
document.querySelectorAll('h1').forEach(h1 => {
|
|
const text = h1.innerText.trim();
|
|
if (text) data.h1.push(text);
|
|
});
|
|
|
|
// Get h2 tags
|
|
document.querySelectorAll('h2').forEach(h2 => {
|
|
const text = h2.innerText.trim();
|
|
if (text) data.h2.push(text);
|
|
});
|
|
|
|
// Count images
|
|
data.images = document.querySelectorAll('img').length;
|
|
|
|
// Count links
|
|
data.links = document.querySelectorAll('a').length;
|
|
|
|
return data;
|
|
});
|
|
},
|
|
formatResult: (seoData, url) => ({
|
|
status: 'success',
|
|
url: url,
|
|
seo: seoData
|
|
}),
|
|
skipCache,
|
|
logMessage: `Extracting SEO from: ${url}`,
|
|
cacheKey: 'SEO cache',
|
|
waitUntil: 'networkidle',
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error extracting SEO from ${url}:`, error.message);
|
|
return {
|
|
status: 'error',
|
|
url: url,
|
|
error: error.message
|
|
};
|
|
}
|
|
}
|
|
|
|
async function extractMetaTags(url, skipCache = false) {
|
|
try {
|
|
return await scrapeWithCache(url, {
|
|
getCache: async (url, now) => await dbOps.getCachedMeta(url, now),
|
|
setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt),
|
|
extractData: async (page) => {
|
|
return await page.evaluate(() => {
|
|
const data = {
|
|
meta_tags: [],
|
|
open_graph: {},
|
|
twitter_card: {},
|
|
title: document.title || ''
|
|
};
|
|
|
|
// Extract all meta tags
|
|
document.querySelectorAll('meta').forEach(meta => {
|
|
const attributes = {};
|
|
for (let attr of meta.attributes) {
|
|
attributes[attr.name] = attr.value;
|
|
}
|
|
data.meta_tags.push(attributes);
|
|
});
|
|
|
|
// Extract Open Graph tags
|
|
document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
|
|
data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
|
|
});
|
|
|
|
// Extract Twitter card tags
|
|
document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
|
|
data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
|
|
});
|
|
|
|
return data;
|
|
});
|
|
},
|
|
formatResult: (metaData, url) => ({
|
|
status: 'success',
|
|
url: url,
|
|
meta_tags: metaData.meta_tags,
|
|
open_graph: metaData.open_graph,
|
|
twitter_card: metaData.twitter_card,
|
|
title: metaData.title
|
|
}),
|
|
skipCache,
|
|
logMessage: `Extracting meta tags from: ${url}`,
|
|
cacheKey: 'Meta tags cache',
|
|
waitUntil: 'networkidle',
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error extracting meta tags from ${url}:`, error.message);
|
|
return {
|
|
status: 'error',
|
|
url: url,
|
|
error: error.message
|
|
};
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
fetchHtml,
|
|
extractSeo,
|
|
extractMetaTags,
|
|
};
|