const dbOps = require('./database');
const { getBrowser } = require('./browser');
const config = require('./config');

async function fetchHtml(url, skipCache = false) {
    let page = null;
    try {
        // Check cache first (unless skipCache is true)
        if (!skipCache) {
            const now = Date.now();
            const cached = await dbOps.getCached(url, now);
            if (cached) {
                console.log(`Cache hit for: ${url}`);
                return cached;
            }
        }

        console.log(`Fetching: ${url} (skipCache: ${skipCache})`);

        // Get browser instance
        const browserInstance = await getBrowser();

        // Create new page
        page = await browserInstance.newPage();

        // Set reasonable timeouts
        page.setDefaultTimeout(30000);
        page.setDefaultNavigationTimeout(30000);

        // Track network requests to ensure all are completed
        const pendingRequests = new Set();
        const finishedRequests = new Set();

        // Monitor network requests
        page.on('request', (request) => {
            const requestId = request.url();
            pendingRequests.add(requestId);
        });

        page.on('response', (response) => {
            const requestId = response.url();
            finishedRequests.add(requestId);
            pendingRequests.delete(requestId);
        });

        page.on('requestfailed', (request) => {
            const requestId = request.url();
            pendingRequests.delete(requestId);
        });

        // Navigate to URL
        await page.goto(url, {
            waitUntil: 'domcontentloaded',
            timeout: 30000,
        });

        // Wait for network to be idle (no requests for 500ms)
        await page.waitForLoadState('networkidle', { timeout: 30000 });

        // Additional wait to ensure all pending requests complete
        const maxWaitTime = 10000; // 10 seconds max
        const checkInterval = 100; // Check every 100ms
        const startTime = Date.now();

        while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
            await new Promise(resolve => setTimeout(resolve, checkInterval));
        }

        if (pendingRequests.size > 0) {
            console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
        }

        // Get full HTML
        const html = await page.content();

        // Store in cache
        const now = Date.now();
        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
        await dbOps.setCache(url, html, now, expiresAt);
        console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);

        return html;
    } catch (error) {
        console.error(`Error fetching ${url}:`, error.message);

        // If cache exists and we had an error, optionally return stale cache
        // For now, we'll just throw the error
        throw error;
    } finally {
        // Always close the page, even on error
        if (page) {
            try {
                await page.close();
            } catch (closeError) {
                console.error('Error closing page:', closeError.message);
            }
        }
    }
}

module.exports = {
    fetchHtml,
};