104 lines
3.2 KiB
JavaScript
104 lines
3.2 KiB
JavaScript
const dbOps = require('./database');
|
|
const { getBrowser } = require('./browser');
|
|
const config = require('./config');
|
|
|
|
async function fetchHtml(url, skipCache = false) {
|
|
let page = null;
|
|
try {
|
|
// Check cache first (unless skipCache is true)
|
|
if (!skipCache) {
|
|
const now = Date.now();
|
|
const cached = await dbOps.getCached(url, now);
|
|
if (cached) {
|
|
console.log(`Cache hit for: ${url}`);
|
|
return cached;
|
|
}
|
|
}
|
|
|
|
console.log(`Fetching: ${url} (skipCache: ${skipCache})`);
|
|
|
|
// Get browser instance
|
|
const browserInstance = await getBrowser();
|
|
|
|
// Create new page
|
|
page = await browserInstance.newPage();
|
|
|
|
// Set reasonable timeouts
|
|
page.setDefaultTimeout(30000);
|
|
page.setDefaultNavigationTimeout(30000);
|
|
|
|
// Track network requests to ensure all are completed
|
|
const pendingRequests = new Set();
|
|
const finishedRequests = new Set();
|
|
|
|
// Monitor network requests
|
|
page.on('request', (request) => {
|
|
const requestId = request.url();
|
|
pendingRequests.add(requestId);
|
|
});
|
|
|
|
page.on('response', (response) => {
|
|
const requestId = response.url();
|
|
finishedRequests.add(requestId);
|
|
pendingRequests.delete(requestId);
|
|
});
|
|
|
|
page.on('requestfailed', (request) => {
|
|
const requestId = request.url();
|
|
pendingRequests.delete(requestId);
|
|
});
|
|
|
|
// Navigate to URL
|
|
await page.goto(url, {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: 30000,
|
|
});
|
|
|
|
// Wait for network to be idle (no requests for 500ms)
|
|
await page.waitForLoadState('networkidle', { timeout: 30000 });
|
|
|
|
// Additional wait to ensure all pending requests complete
|
|
const maxWaitTime = 10000; // 10 seconds max
|
|
const checkInterval = 100; // Check every 100ms
|
|
const startTime = Date.now();
|
|
|
|
while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
|
|
await new Promise(resolve => setTimeout(resolve, checkInterval));
|
|
}
|
|
|
|
if (pendingRequests.size > 0) {
|
|
console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
|
|
}
|
|
|
|
// Get full HTML
|
|
const html = await page.content();
|
|
|
|
// Store in cache
|
|
const now = Date.now();
|
|
const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
|
|
await dbOps.setCache(url, html, now, expiresAt);
|
|
console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
|
|
|
|
return html;
|
|
} catch (error) {
|
|
console.error(`Error fetching ${url}:`, error.message);
|
|
|
|
// If cache exists and we had an error, optionally return stale cache
|
|
// For now, we'll just throw the error
|
|
throw error;
|
|
} finally {
|
|
// Always close the page, even on error
|
|
if (page) {
|
|
try {
|
|
await page.close();
|
|
} catch (closeError) {
|
|
console.error('Error closing page:', closeError.message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
fetchHtml,
|
|
};
|