const dbOps = require('./database');
const { getBrowser } = require('./browser');
const config = require('./config');

/**
 * Reusable function to navigate to a URL and wait for all network requests to complete
 * @param {string} url - The URL to navigate to
 * @param {object} options - Options for navigation
 * @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle')
 * @returns {Promise<Page>} - The Playwright page object
 */
async function navigateAndWait(url, options = {}) {
    const { waitUntil = 'networkidle' } = options;

    // Get browser instance
    const browserInstance = await getBrowser();

    // Create new page
    const page = await browserInstance.newPage();

    // Set reasonable timeouts
    page.setDefaultTimeout(30000);
    page.setDefaultNavigationTimeout(30000);

    // Track network requests to ensure all are completed
    const pendingRequests = new Set();

    // Monitor network requests
    page.on('request', (request) => {
        const requestId = request.url();
        pendingRequests.add(requestId);
    });

    page.on('response', (response) => {
        const requestId = response.url();
        pendingRequests.delete(requestId);
    });

    page.on('requestfailed', (request) => {
        const requestId = request.url();
        pendingRequests.delete(requestId);
    });

    // Navigate to URL
    await page.goto(url, {
        waitUntil: waitUntil,
        timeout: 30000,
    });

    // If we used domcontentloaded, wait for networkidle separately
    if (waitUntil === 'domcontentloaded') {
        await page.waitForLoadState('networkidle', { timeout: 30000 });
    }

    // Additional wait to ensure all pending requests complete
    const maxWaitTime = 10000; // 10 seconds max
    const checkInterval = 100; // Check every 100ms
    const startTime = Date.now();

    while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
        await new Promise(resolve => setTimeout(resolve, checkInterval));
    }

    if (pendingRequests.size > 0) {
        console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
    }

    return page;
}

/**
 * Reusable function to safely execute a scraping operation with proper cleanup
 * @param {string} url - The URL to scrape
 * @param {object} options - Options for the operation
 * @param {Function} options.getCache - Function to get cached data
 * @param {Function} options.setCache - Function to set cached data
 * @param {Function} options.extractData - Function to extract data from the page
 * @param {Function} options.formatResult - Function to format the result
 * @param {boolean} options.skipCache - Whether to skip cache
 * @param {string} options.logMessage - Log message for the operation
 * @param {string} options.cacheKey - Cache key prefix for logging
 * @returns {Promise<any>} - The extracted data
 */
async function scrapeWithCache(url, options) {
    const {
        getCache,
        setCache,
        extractData,
        formatResult,
        skipCache = false,
        logMessage,
        cacheKey = 'cache',
        waitUntil = 'networkidle',
    } = options;

    let page = null;
    try {
        // Check cache first (unless skipCache is true)
        if (!skipCache && getCache) {
            const now = Date.now();
            const cached = await getCache(url, now);
            if (cached) {
                console.log(`${cacheKey} hit for: ${url}`);
                return cached;
            }
        }

        if (logMessage) {
            console.log(logMessage);
        }

        // Navigate and wait for network
        page = await navigateAndWait(url, { waitUntil });

        // Extract data using the provided function
        const extractedData = await extractData(page);

        // Format the result
        const result = formatResult ? formatResult(extractedData, url) : extractedData;

        // Store in cache
        if (setCache) {
            const now = Date.now();
            const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
            await setCache(url, result, now, expiresAt);
            console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
        }

        return result;
    } catch (error) {
        console.error(`Error processing ${url}:`, error.message);
        throw error;
    } finally {
        // Always close the page, even on error
        if (page) {
            try {
                await page.close();
            } catch (closeError) {
                console.error('Error closing page:', closeError.message);
            }
        }
    }
}

async function fetchHtml(url, skipCache = false) {
    try {
        return await scrapeWithCache(url, {
            getCache: async (url, now) => await dbOps.getCached(url, now),
            setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt),
            extractData: async (page) => await page.content(),
            formatResult: (html) => html,
            skipCache,
            logMessage: `Fetching: ${url} (skipCache: ${skipCache})`,
            cacheKey: 'Cache',
            waitUntil: 'domcontentloaded',
        });
    } catch (error) {
        console.error(`Error fetching ${url}:`, error.message);
        throw error;
    }
}

async function extractSeo(url, skipCache = false) {
    try {
        return await scrapeWithCache(url, {
            getCache: async (url, now) => await dbOps.getCachedSeo(url, now),
            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt),
            extractData: async (page) => {
                return await page.evaluate(() => {
                    const data = {
                        title: document.title || '',
                        description: '',
                        canonical: '',
                        h1: [],
                        h2: [],
                        images: 0,
                        links: 0
                    };

                    // Get meta description
                    const metaDescription = document.querySelector('meta[name="description"]');
                    if (metaDescription) {
                        data.description = metaDescription.getAttribute('content') || '';
                    }

                    // Get canonical link
                    const canonicalLink = document.querySelector('link[rel="canonical"]');
                    if (canonicalLink) {
                        data.canonical = canonicalLink.getAttribute('href') || '';
                    }

                    // Get h1 tags
                    document.querySelectorAll('h1').forEach(h1 => {
                        const text = h1.innerText.trim();
                        if (text) data.h1.push(text);
                    });

                    // Get h2 tags
                    document.querySelectorAll('h2').forEach(h2 => {
                        const text = h2.innerText.trim();
                        if (text) data.h2.push(text);
                    });

                    // Count images
                    data.images = document.querySelectorAll('img').length;

                    // Count links
                    data.links = document.querySelectorAll('a').length;

                    return data;
                });
            },
            formatResult: (seoData, url) => ({
                status: 'success',
                url: url,
                seo: seoData
            }),
            skipCache,
            logMessage: `Extracting SEO from: ${url}`,
            cacheKey: 'SEO cache',
            waitUntil: 'networkidle',
        });
    } catch (error) {
        console.error(`Error extracting SEO from ${url}:`, error.message);
        return {
            status: 'error',
            url: url,
            error: error.message
        };
    }
}

async function extractMetaTags(url, skipCache = false) {
    try {
        return await scrapeWithCache(url, {
            getCache: async (url, now) => await dbOps.getCachedMeta(url, now),
            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt),
            extractData: async (page) => {
                return await page.evaluate(() => {
                    const data = {
                        meta_tags: [],
                        open_graph: {},
                        twitter_card: {},
                        title: document.title || ''
                    };

                    // Extract all meta tags
                    document.querySelectorAll('meta').forEach(meta => {
                        const attributes = {};
                        for (let attr of meta.attributes) {
                            attributes[attr.name] = attr.value;
                        }
                        data.meta_tags.push(attributes);
                    });

                    // Extract Open Graph tags
                    document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
                        data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
                    });

                    // Extract Twitter card tags
                    document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
                        data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
                    });

                    return data;
                });
            },
            formatResult: (metaData, url) => ({
                status: 'success',
                url: url,
                meta_tags: metaData.meta_tags,
                open_graph: metaData.open_graph,
                twitter_card: metaData.twitter_card,
                title: metaData.title
            }),
            skipCache,
            logMessage: `Extracting meta tags from: ${url}`,
            cacheKey: 'Meta tags cache',
            waitUntil: 'networkidle',
        });
    } catch (error) {
        console.error(`Error extracting meta tags from ${url}:`, error.message);
        return {
            status: 'error',
            url: url,
            error: error.message
        };
    }
}

module.exports = {
    fetchHtml,
    extractSeo,
    extractMetaTags,
};