572 lines
20 KiB
JavaScript
572 lines
20 KiB
JavaScript
const dbOps = require('./database');
|
|
const { getBrowser } = require('./browser');
|
|
const config = require('./config');
|
|
|
|
/**
|
|
* Reusable function to navigate to a URL and wait for all network requests to complete
|
|
* @param {string} url - The URL to navigate to
|
|
* @param {object} options - Options for navigation
|
|
* @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle')
|
|
* @returns {Promise<Page>} - The Playwright page object
|
|
*/
|
|
async function navigateAndWait(url, options = {}) {
|
|
const { waitUntil = 'networkidle' } = options;
|
|
|
|
// Get browser instance
|
|
const browserInstance = await getBrowser();
|
|
|
|
// Create new page
|
|
const page = await browserInstance.newPage();
|
|
|
|
// Set reasonable timeouts
|
|
page.setDefaultTimeout(30000);
|
|
page.setDefaultNavigationTimeout(30000);
|
|
|
|
// Track network requests to ensure all are completed
|
|
const pendingRequests = new Set();
|
|
|
|
// Monitor network requests
|
|
page.on('request', (request) => {
|
|
const requestId = request.url();
|
|
pendingRequests.add(requestId);
|
|
});
|
|
|
|
page.on('response', (response) => {
|
|
const requestId = response.url();
|
|
pendingRequests.delete(requestId);
|
|
});
|
|
|
|
page.on('requestfailed', (request) => {
|
|
const requestId = request.url();
|
|
pendingRequests.delete(requestId);
|
|
});
|
|
|
|
// Navigate to URL
|
|
await page.goto(url, {
|
|
waitUntil: waitUntil,
|
|
timeout: 30000,
|
|
});
|
|
|
|
// If we used domcontentloaded, wait for networkidle separately
|
|
if (waitUntil === 'domcontentloaded') {
|
|
await page.waitForLoadState('networkidle', { timeout: 30000 });
|
|
}
|
|
|
|
// Additional wait to ensure all pending requests complete
|
|
const maxWaitTime = 10000; // 10 seconds max
|
|
const checkInterval = 100; // Check every 100ms
|
|
const startTime = Date.now();
|
|
|
|
while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
|
|
await new Promise(resolve => setTimeout(resolve, checkInterval));
|
|
}
|
|
|
|
if (pendingRequests.size > 0) {
|
|
console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
|
|
}
|
|
|
|
return page;
|
|
}
|
|
|
|
/**
|
|
* Reusable function to safely execute a scraping operation with proper cleanup
|
|
* @param {string} url - The URL to scrape
|
|
* @param {object} options - Options for the operation
|
|
* @param {Function} options.getCache - Function to get cached data
|
|
* @param {Function} options.setCache - Function to set cached data
|
|
* @param {Function} options.extractData - Function to extract data from the page
|
|
* @param {Function} options.formatResult - Function to format the result
|
|
* @param {boolean} options.skipCache - Whether to skip cache
|
|
* @param {string} options.logMessage - Log message for the operation
|
|
* @param {string} options.cacheKey - Cache key prefix for logging
|
|
* @returns {Promise<any>} - The extracted data
|
|
*/
|
|
async function scrapeWithCache(url, options) {
|
|
const {
|
|
getCache,
|
|
setCache,
|
|
extractData,
|
|
formatResult,
|
|
skipCache = false,
|
|
logMessage,
|
|
cacheKey = 'cache',
|
|
waitUntil = 'networkidle',
|
|
} = options;
|
|
|
|
let page = null;
|
|
try {
|
|
// Check cache first (unless skipCache is true)
|
|
if (!skipCache && getCache) {
|
|
const now = Date.now();
|
|
const cached = await getCache(url, now);
|
|
if (cached) {
|
|
console.log(`${cacheKey} hit for: ${url}`);
|
|
return cached;
|
|
}
|
|
}
|
|
|
|
if (logMessage) {
|
|
console.log(logMessage);
|
|
}
|
|
|
|
// Navigate and wait for network
|
|
page = await navigateAndWait(url, { waitUntil });
|
|
|
|
// Extract data using the provided function
|
|
const extractedData = await extractData(page);
|
|
|
|
// Format the result
|
|
const result = formatResult ? formatResult(extractedData, url) : extractedData;
|
|
|
|
// Store in cache
|
|
if (setCache) {
|
|
const now = Date.now();
|
|
const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
|
|
await setCache(url, result, now, expiresAt);
|
|
console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
|
|
}
|
|
|
|
return result;
|
|
} catch (error) {
|
|
console.error(`Error processing ${url}:`, error.message);
|
|
throw error;
|
|
} finally {
|
|
// Always close the page, even on error
|
|
if (page) {
|
|
try {
|
|
await page.close();
|
|
} catch (closeError) {
|
|
console.error('Error closing page:', closeError.message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function fetchHtml(url, skipCache = false) {
|
|
try {
|
|
return await scrapeWithCache(url, {
|
|
getCache: async (url, now) => await dbOps.getCached(url, now),
|
|
setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt),
|
|
extractData: async (page) => await page.content(),
|
|
formatResult: (html) => html,
|
|
skipCache,
|
|
logMessage: `Fetching: ${url} (skipCache: ${skipCache})`,
|
|
cacheKey: 'Cache',
|
|
waitUntil: 'domcontentloaded',
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error fetching ${url}:`, error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function extractSeo(url, skipCache = false) {
|
|
try {
|
|
return await scrapeWithCache(url, {
|
|
getCache: async (url, now) => await dbOps.getCachedSeo(url, now),
|
|
setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt),
|
|
extractData: async (page) => {
|
|
return await page.evaluate(() => {
|
|
const data = {
|
|
title: document.title || '',
|
|
description: '',
|
|
canonical: '',
|
|
h1: [],
|
|
h2: [],
|
|
images: 0,
|
|
links: 0
|
|
};
|
|
|
|
// Get meta description
|
|
const metaDescription = document.querySelector('meta[name="description"]');
|
|
if (metaDescription) {
|
|
data.description = metaDescription.getAttribute('content') || '';
|
|
}
|
|
|
|
// Get canonical link
|
|
const canonicalLink = document.querySelector('link[rel="canonical"]');
|
|
if (canonicalLink) {
|
|
data.canonical = canonicalLink.getAttribute('href') || '';
|
|
}
|
|
|
|
// Get h1 tags
|
|
document.querySelectorAll('h1').forEach(h1 => {
|
|
const text = h1.innerText.trim();
|
|
if (text) data.h1.push(text);
|
|
});
|
|
|
|
// Get h2 tags
|
|
document.querySelectorAll('h2').forEach(h2 => {
|
|
const text = h2.innerText.trim();
|
|
if (text) data.h2.push(text);
|
|
});
|
|
|
|
// Count images
|
|
data.images = document.querySelectorAll('img').length;
|
|
|
|
// Count links
|
|
data.links = document.querySelectorAll('a').length;
|
|
|
|
return data;
|
|
});
|
|
},
|
|
formatResult: (seoData, url) => ({
|
|
status: 'success',
|
|
url: url,
|
|
seo: seoData
|
|
}),
|
|
skipCache,
|
|
logMessage: `Extracting SEO from: ${url}`,
|
|
cacheKey: 'SEO cache',
|
|
waitUntil: 'networkidle',
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error extracting SEO from ${url}:`, error.message);
|
|
return {
|
|
status: 'error',
|
|
url: url,
|
|
error: error.message
|
|
};
|
|
}
|
|
}
|
|
|
|
async function extractMetaTags(url, skipCache = false) {
|
|
try {
|
|
return await scrapeWithCache(url, {
|
|
getCache: async (url, now) => await dbOps.getCachedMeta(url, now),
|
|
setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt),
|
|
extractData: async (page) => {
|
|
return await page.evaluate(() => {
|
|
const data = {
|
|
meta_tags: [],
|
|
open_graph: {},
|
|
twitter_card: {},
|
|
title: document.title || ''
|
|
};
|
|
|
|
// Extract all meta tags
|
|
document.querySelectorAll('meta').forEach(meta => {
|
|
const attributes = {};
|
|
for (let attr of meta.attributes) {
|
|
attributes[attr.name] = attr.value;
|
|
}
|
|
data.meta_tags.push(attributes);
|
|
});
|
|
|
|
// Extract Open Graph tags
|
|
document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
|
|
data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
|
|
});
|
|
|
|
// Extract Twitter card tags
|
|
document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
|
|
data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
|
|
});
|
|
|
|
return data;
|
|
});
|
|
},
|
|
formatResult: (metaData, url) => ({
|
|
status: 'success',
|
|
url: url,
|
|
meta_tags: metaData.meta_tags,
|
|
open_graph: metaData.open_graph,
|
|
twitter_card: metaData.twitter_card,
|
|
title: metaData.title
|
|
}),
|
|
skipCache,
|
|
logMessage: `Extracting meta tags from: ${url}`,
|
|
cacheKey: 'Meta tags cache',
|
|
waitUntil: 'networkidle',
|
|
});
|
|
} catch (error) {
|
|
console.error(`Error extracting meta tags from ${url}:`, error.message);
|
|
return {
|
|
status: 'error',
|
|
url: url,
|
|
error: error.message
|
|
};
|
|
}
|
|
}
|
|
|
|
async function extractOutgoingCalls(url, skipCache = false) {
|
|
let page = null;
|
|
try {
|
|
// Check cache first (unless skipCache is true)
|
|
if (!skipCache) {
|
|
const now = Date.now();
|
|
const cached = await dbOps.getCachedOutgoingCalls(url, now);
|
|
if (cached) {
|
|
console.log(`Outgoing calls cache hit for: ${url}`);
|
|
return cached;
|
|
}
|
|
}
|
|
|
|
console.log(`Capturing outgoing calls from: ${url}`);
|
|
|
|
// Get browser instance
|
|
const browserInstance = await getBrowser();
|
|
|
|
// Create new page
|
|
page = await browserInstance.newPage();
|
|
|
|
// Set reasonable timeouts
|
|
page.setDefaultTimeout(30000);
|
|
page.setDefaultNavigationTimeout(30000);
|
|
|
|
// List to store all network requests
|
|
const networkRequests = [];
|
|
|
|
// Static asset extensions to skip
|
|
const staticExtensions = ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.eot'];
|
|
|
|
// Set up network request listener BEFORE navigation
|
|
page.on('request', (request) => {
|
|
const requestUrl = request.url();
|
|
const method = request.method();
|
|
const headers = request.headers();
|
|
|
|
// Skip static assets and common non-API requests
|
|
if (staticExtensions.some(ext => requestUrl.endsWith(ext))) {
|
|
return;
|
|
}
|
|
|
|
// Skip data URLs and blob URLs
|
|
if (requestUrl.startsWith('data:') || requestUrl.startsWith('blob:')) {
|
|
return;
|
|
}
|
|
|
|
// Skip same-origin requests that are likely static assets
|
|
if (requestUrl.startsWith(url) && staticExtensions.some(staticExt => requestUrl.toLowerCase().includes(staticExt))) {
|
|
return;
|
|
}
|
|
|
|
// Capture the request details
|
|
const requestData = {
|
|
url: requestUrl,
|
|
method: method,
|
|
headers: headers,
|
|
timestamp: Date.now()
|
|
};
|
|
|
|
networkRequests.push(requestData);
|
|
});
|
|
|
|
// Navigate to URL and wait for network to be idle
|
|
try {
|
|
await page.goto(url, {
|
|
waitUntil: 'networkidle',
|
|
timeout: 30000,
|
|
});
|
|
} catch (error) {
|
|
console.log(`Error during page navigation: ${error.message}`);
|
|
// Continue anyway to capture any requests that were made
|
|
}
|
|
|
|
// Wait a bit more to catch any delayed requests
|
|
await page.waitForTimeout(2000);
|
|
|
|
// Process and categorize the requests
|
|
const apiCalls = [];
|
|
for (const req of networkRequests) {
|
|
// Determine if this looks like an API call
|
|
let isApiCall = false;
|
|
let apiType = 'unknown';
|
|
|
|
const urlLower = req.url.toLowerCase();
|
|
const contentType = (req.headers['content-type'] || '').toLowerCase();
|
|
|
|
// Check for common API patterns
|
|
if (['/api/', '/rest/', '/graphql', '/json', '/xml'].some(pattern => urlLower.includes(pattern))) {
|
|
isApiCall = true;
|
|
apiType = 'rest';
|
|
} else if (req.url.endsWith('.json')) {
|
|
isApiCall = true;
|
|
apiType = 'json';
|
|
} else if (contentType.includes('application/json')) {
|
|
isApiCall = true;
|
|
apiType = 'json';
|
|
} else if (contentType.includes('application/xml')) {
|
|
isApiCall = true;
|
|
apiType = 'xml';
|
|
} else if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(req.method)) {
|
|
// These methods are typically API calls
|
|
isApiCall = true;
|
|
apiType = 'rest';
|
|
} else if (['api.', 'rest.', 'graphql.'].some(domain => req.url.includes(domain))) {
|
|
isApiCall = true;
|
|
apiType = 'rest';
|
|
}
|
|
|
|
// Filter headers (exclude common browser headers)
|
|
const filteredHeaders = {};
|
|
const excludedHeaders = ['user-agent', 'accept-encoding', 'accept-language', 'cache-control'];
|
|
for (const [key, value] of Object.entries(req.headers)) {
|
|
if (!excludedHeaders.includes(key.toLowerCase())) {
|
|
filteredHeaders[key] = value;
|
|
}
|
|
}
|
|
|
|
// Include all requests but mark API calls specifically
|
|
const callInfo = {
|
|
url: req.url,
|
|
method: req.method,
|
|
is_api_call: isApiCall,
|
|
api_type: isApiCall ? apiType : null,
|
|
headers: filteredHeaders
|
|
};
|
|
|
|
apiCalls.push(callInfo);
|
|
}
|
|
|
|
// Sort by whether it's an API call (API calls first), then by URL
|
|
apiCalls.sort((a, b) => {
|
|
if (a.is_api_call !== b.is_api_call) {
|
|
// API calls first (true comes before false)
|
|
// If a is API call and b is not, a comes first (return -1)
|
|
// If a is not API call and b is, b comes first (return 1)
|
|
return a.is_api_call ? -1 : 1;
|
|
}
|
|
return a.url.localeCompare(b.url);
|
|
});
|
|
|
|
// Extract unique domains
|
|
const domains = new Set();
|
|
for (const call of apiCalls) {
|
|
try {
|
|
const urlObj = new URL(call.url);
|
|
domains.add(urlObj.hostname);
|
|
} catch (e) {
|
|
// Skip invalid URLs
|
|
}
|
|
}
|
|
|
|
const result = {
|
|
status: 'success',
|
|
url: url,
|
|
total_requests: apiCalls.length,
|
|
api_calls: apiCalls.filter(call => call.is_api_call),
|
|
other_requests: apiCalls.filter(call => !call.is_api_call),
|
|
summary: {
|
|
api_calls_count: apiCalls.filter(call => call.is_api_call).length,
|
|
other_requests_count: apiCalls.filter(call => !call.is_api_call).length,
|
|
unique_domains: domains.size
|
|
}
|
|
};
|
|
|
|
// Store in cache
|
|
const now = Date.now();
|
|
const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
|
|
await dbOps.setCacheOutgoingCalls(url, result, now, expiresAt);
|
|
console.log(`Cached outgoing calls for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
|
|
|
|
return result;
|
|
} catch (error) {
|
|
console.error(`Error capturing outgoing calls from ${url}:`, error.message);
|
|
|
|
return {
|
|
status: 'error',
|
|
url: url,
|
|
error: error.message
|
|
};
|
|
} finally {
|
|
// Always close the page, even on error
|
|
if (page) {
|
|
try {
|
|
await page.close();
|
|
} catch (closeError) {
|
|
console.error('Error closing page:', closeError.message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function extractResultingUrl(url, skipCache = false) {
|
|
let page = null;
|
|
try {
|
|
// Check cache first (unless skipCache is true)
|
|
if (!skipCache) {
|
|
const now = Date.now();
|
|
const cached = await dbOps.getCachedResultingUrl(url, now);
|
|
if (cached) {
|
|
console.log(`Resulting URL cache hit for: ${url}`);
|
|
return cached;
|
|
}
|
|
}
|
|
|
|
console.log(`Getting resulting URL for: ${url}`);
|
|
|
|
// Get browser instance
|
|
const browserInstance = await getBrowser();
|
|
|
|
// Create new page
|
|
page = await browserInstance.newPage();
|
|
|
|
// Set reasonable timeouts
|
|
page.setDefaultTimeout(30000);
|
|
page.setDefaultNavigationTimeout(30000);
|
|
|
|
// Navigate to URL and wait for network to be idle
|
|
let response = null;
|
|
try {
|
|
response = await page.goto(url, {
|
|
waitUntil: 'networkidle',
|
|
timeout: 30000,
|
|
});
|
|
} catch (error) {
|
|
console.log(`Error during page navigation: ${error.message}`);
|
|
// Continue anyway to get the final URL
|
|
}
|
|
|
|
// Get the final URL after any redirects
|
|
const finalUrl = page.url;
|
|
|
|
// Get response status if available
|
|
const statusCode = response ? response.status() : null;
|
|
|
|
// Check if there was a redirect
|
|
const redirected = finalUrl !== url;
|
|
|
|
const result = {
|
|
status: 'success',
|
|
original_url: url,
|
|
resulting_url: finalUrl,
|
|
status_code: statusCode,
|
|
redirected: redirected
|
|
};
|
|
|
|
// Store in cache
|
|
const now = Date.now();
|
|
const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
|
|
await dbOps.setCacheResultingUrl(url, result, now, expiresAt);
|
|
console.log(`Cached resulting URL for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
|
|
|
|
return result;
|
|
} catch (error) {
|
|
console.error(`Error getting resulting URL for ${url}:`, error.message);
|
|
|
|
return {
|
|
status: 'error',
|
|
original_url: url,
|
|
error: error.message
|
|
};
|
|
} finally {
|
|
// Always close the page, even on error
|
|
if (page) {
|
|
try {
|
|
await page.close();
|
|
} catch (closeError) {
|
|
console.error('Error closing page:', closeError.message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
fetchHtml,
|
|
extractSeo,
|
|
extractMetaTags,
|
|
extractOutgoingCalls,
|
|
extractResultingUrl,
|
|
};
|