const dbOps = require('./database');
const { getBrowser } = require('./browser');
const config = require('./config');

/**
 * Reusable function to navigate to a URL and wait for all network requests to complete
 * @param {string} url - The URL to navigate to
 * @param {object} options - Options for navigation
 * @param {string} options.waitUntil - Wait condition ('domcontentloaded' or 'networkidle')
 * @returns {Promise<Page>} - The Playwright page object
 */
async function navigateAndWait(url, options = {}) {
    const { waitUntil = 'networkidle' } = options;

    // Get browser instance
    const browserInstance = await getBrowser();

    // Create new page
    const page = await browserInstance.newPage();

    // Set reasonable timeouts
    page.setDefaultTimeout(30000);
    page.setDefaultNavigationTimeout(30000);

    // Track network requests to ensure all are completed
    const pendingRequests = new Set();

    // Monitor network requests
    page.on('request', (request) => {
        const requestId = request.url();
        pendingRequests.add(requestId);
    });

    page.on('response', (response) => {
        const requestId = response.url();
        pendingRequests.delete(requestId);
    });

    page.on('requestfailed', (request) => {
        const requestId = request.url();
        pendingRequests.delete(requestId);
    });

    // Navigate to URL
    await page.goto(url, {
        waitUntil: waitUntil,
        timeout: 30000,
    });

    // If we used domcontentloaded, wait for networkidle separately
    if (waitUntil === 'domcontentloaded') {
        await page.waitForLoadState('networkidle', { timeout: 30000 });
    }

    // Additional wait to ensure all pending requests complete
    const maxWaitTime = 10000; // 10 seconds max
    const checkInterval = 100; // Check every 100ms
    const startTime = Date.now();

    while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
        await new Promise(resolve => setTimeout(resolve, checkInterval));
    }

    if (pendingRequests.size > 0) {
        console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
    }

    return page;
}

/**
 * Reusable function to safely execute a scraping operation with proper cleanup
 * @param {string} url - The URL to scrape
 * @param {object} options - Options for the operation
 * @param {Function} options.getCache - Function to get cached data
 * @param {Function} options.setCache - Function to set cached data
 * @param {Function} options.extractData - Function to extract data from the page
 * @param {Function} options.formatResult - Function to format the result
 * @param {boolean} options.skipCache - Whether to skip cache
 * @param {string} options.logMessage - Log message for the operation
 * @param {string} options.cacheKey - Cache key prefix for logging
 * @returns {Promise<any>} - The extracted data
 */
async function scrapeWithCache(url, options) {
    const {
        getCache,
        setCache,
        extractData,
        formatResult,
        skipCache = false,
        logMessage,
        cacheKey = 'cache',
        waitUntil = 'networkidle',
    } = options;

    let page = null;
    try {
        // Check cache first (unless skipCache is true)
        if (!skipCache && getCache) {
            const now = Date.now();
            const cached = await getCache(url, now);
            if (cached) {
                console.log(`${cacheKey} hit for: ${url}`);
                return cached;
            }
        }

        if (logMessage) {
            console.log(logMessage);
        }

        // Navigate and wait for network
        page = await navigateAndWait(url, { waitUntil });

        // Extract data using the provided function
        const extractedData = await extractData(page);

        // Format the result
        const result = formatResult ? formatResult(extractedData, url) : extractedData;

        // Store in cache
        if (setCache) {
            const now = Date.now();
            const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
            await setCache(url, result, now, expiresAt);
            console.log(`Cached ${cacheKey} for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
        }

        return result;
    } catch (error) {
        console.error(`Error processing ${url}:`, error.message);
        throw error;
    } finally {
        // Always close the page, even on error
        if (page) {
            try {
                await page.close();
            } catch (closeError) {
                console.error('Error closing page:', closeError.message);
            }
        }
    }
}

async function fetchHtml(url, skipCache = false) {
    try {
        return await scrapeWithCache(url, {
            getCache: async (url, now) => await dbOps.getCached(url, now),
            setCache: async (url, html, now, expiresAt) => await dbOps.setCache(url, html, now, expiresAt),
            extractData: async (page) => await page.content(),
            formatResult: (html) => html,
            skipCache,
            logMessage: `Fetching: ${url} (skipCache: ${skipCache})`,
            cacheKey: 'Cache',
            waitUntil: 'domcontentloaded',
        });
    } catch (error) {
        console.error(`Error fetching ${url}:`, error.message);
        throw error;
    }
}

async function extractSeo(url, skipCache = false) {
    try {
        return await scrapeWithCache(url, {
            getCache: async (url, now) => await dbOps.getCachedSeo(url, now),
            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheSeo(url, data, now, expiresAt),
            extractData: async (page) => {
                return await page.evaluate(() => {
                    const data = {
                        title: document.title || '',
                        description: '',
                        canonical: '',
                        h1: [],
                        h2: [],
                        images: 0,
                        links: 0
                    };

                    // Get meta description
                    const metaDescription = document.querySelector('meta[name="description"]');
                    if (metaDescription) {
                        data.description = metaDescription.getAttribute('content') || '';
                    }

                    // Get canonical link
                    const canonicalLink = document.querySelector('link[rel="canonical"]');
                    if (canonicalLink) {
                        data.canonical = canonicalLink.getAttribute('href') || '';
                    }

                    // Get h1 tags
                    document.querySelectorAll('h1').forEach(h1 => {
                        const text = h1.innerText.trim();
                        if (text) data.h1.push(text);
                    });

                    // Get h2 tags
                    document.querySelectorAll('h2').forEach(h2 => {
                        const text = h2.innerText.trim();
                        if (text) data.h2.push(text);
                    });

                    // Count images
                    data.images = document.querySelectorAll('img').length;

                    // Count links
                    data.links = document.querySelectorAll('a').length;

                    return data;
                });
            },
            formatResult: (seoData, url) => ({
                status: 'success',
                url: url,
                seo: seoData
            }),
            skipCache,
            logMessage: `Extracting SEO from: ${url}`,
            cacheKey: 'SEO cache',
            waitUntil: 'networkidle',
        });
    } catch (error) {
        console.error(`Error extracting SEO from ${url}:`, error.message);
        return {
            status: 'error',
            url: url,
            error: error.message
        };
    }
}

async function extractMetaTags(url, skipCache = false) {
    try {
        return await scrapeWithCache(url, {
            getCache: async (url, now) => await dbOps.getCachedMeta(url, now),
            setCache: async (url, data, now, expiresAt) => await dbOps.setCacheMeta(url, data, now, expiresAt),
            extractData: async (page) => {
                return await page.evaluate(() => {
                    const data = {
                        meta_tags: [],
                        open_graph: {},
                        twitter_card: {},
                        title: document.title || ''
                    };

                    // Extract all meta tags
                    document.querySelectorAll('meta').forEach(meta => {
                        const attributes = {};
                        for (let attr of meta.attributes) {
                            attributes[attr.name] = attr.value;
                        }
                        data.meta_tags.push(attributes);
                    });

                    // Extract Open Graph tags
                    document.querySelectorAll('meta[property^="og:"]').forEach(meta => {
                        data.open_graph[meta.getAttribute('property')] = meta.getAttribute('content') || '';
                    });

                    // Extract Twitter card tags
                    document.querySelectorAll('meta[name^="twitter:"]').forEach(meta => {
                        data.twitter_card[meta.getAttribute('name')] = meta.getAttribute('content') || '';
                    });

                    return data;
                });
            },
            formatResult: (metaData, url) => ({
                status: 'success',
                url: url,
                meta_tags: metaData.meta_tags,
                open_graph: metaData.open_graph,
                twitter_card: metaData.twitter_card,
                title: metaData.title
            }),
            skipCache,
            logMessage: `Extracting meta tags from: ${url}`,
            cacheKey: 'Meta tags cache',
            waitUntil: 'networkidle',
        });
    } catch (error) {
        console.error(`Error extracting meta tags from ${url}:`, error.message);
        return {
            status: 'error',
            url: url,
            error: error.message
        };
    }
}

async function extractOutgoingCalls(url, skipCache = false) {
    let page = null;
    try {
        // Check cache first (unless skipCache is true)
        if (!skipCache) {
            const now = Date.now();
            const cached = await dbOps.getCachedOutgoingCalls(url, now);
            if (cached) {
                console.log(`Outgoing calls cache hit for: ${url}`);
                return cached;
            }
        }

        console.log(`Capturing outgoing calls from: ${url}`);

        // Get browser instance
        const browserInstance = await getBrowser();

        // Create new page
        page = await browserInstance.newPage();

        // Set reasonable timeouts
        page.setDefaultTimeout(30000);
        page.setDefaultNavigationTimeout(30000);

        // List to store all network requests
        const networkRequests = [];

        // Static asset extensions to skip
        const staticExtensions = ['.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.eot'];

        // Set up network request listener BEFORE navigation
        page.on('request', (request) => {
            const requestUrl = request.url();
            const method = request.method();
            const headers = request.headers();

            // Skip static assets and common non-API requests
            if (staticExtensions.some(ext => requestUrl.endsWith(ext))) {
                return;
            }

            // Skip data URLs and blob URLs
            if (requestUrl.startsWith('data:') || requestUrl.startsWith('blob:')) {
                return;
            }

            // Skip same-origin requests that are likely static assets
            if (requestUrl.startsWith(url) && staticExtensions.some(staticExt => requestUrl.toLowerCase().includes(staticExt))) {
                return;
            }

            // Capture the request details
            const requestData = {
                url: requestUrl,
                method: method,
                headers: headers,
                timestamp: Date.now()
            };

            networkRequests.push(requestData);
        });

        // Navigate to URL and wait for network to be idle
        try {
            await page.goto(url, {
                waitUntil: 'networkidle',
                timeout: 30000,
            });
        } catch (error) {
            console.log(`Error during page navigation: ${error.message}`);
            // Continue anyway to capture any requests that were made
        }

        // Wait a bit more to catch any delayed requests
        await page.waitForTimeout(2000);

        // Process and categorize the requests
        const apiCalls = [];
        for (const req of networkRequests) {
            // Determine if this looks like an API call
            let isApiCall = false;
            let apiType = 'unknown';

            const urlLower = req.url.toLowerCase();
            const contentType = (req.headers['content-type'] || '').toLowerCase();

            // Check for common API patterns
            if (['/api/', '/rest/', '/graphql', '/json', '/xml'].some(pattern => urlLower.includes(pattern))) {
                isApiCall = true;
                apiType = 'rest';
            } else if (req.url.endsWith('.json')) {
                isApiCall = true;
                apiType = 'json';
            } else if (contentType.includes('application/json')) {
                isApiCall = true;
                apiType = 'json';
            } else if (contentType.includes('application/xml')) {
                isApiCall = true;
                apiType = 'xml';
            } else if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(req.method)) {
                // These methods are typically API calls
                isApiCall = true;
                apiType = 'rest';
            } else if (['api.', 'rest.', 'graphql.'].some(domain => req.url.includes(domain))) {
                isApiCall = true;
                apiType = 'rest';
            }

            // Filter headers (exclude common browser headers)
            const filteredHeaders = {};
            const excludedHeaders = ['user-agent', 'accept-encoding', 'accept-language', 'cache-control'];
            for (const [key, value] of Object.entries(req.headers)) {
                if (!excludedHeaders.includes(key.toLowerCase())) {
                    filteredHeaders[key] = value;
                }
            }

            // Include all requests but mark API calls specifically
            const callInfo = {
                url: req.url,
                method: req.method,
                is_api_call: isApiCall,
                api_type: isApiCall ? apiType : null,
                headers: filteredHeaders
            };

            apiCalls.push(callInfo);
        }

        // Sort by whether it's an API call (API calls first), then by URL
        apiCalls.sort((a, b) => {
            if (a.is_api_call !== b.is_api_call) {
                // API calls first (true comes before false)
                // If a is API call and b is not, a comes first (return -1)
                // If a is not API call and b is, b comes first (return 1)
                return a.is_api_call ? -1 : 1;
            }
            return a.url.localeCompare(b.url);
        });

        // Extract unique domains
        const domains = new Set();
        for (const call of apiCalls) {
            try {
                const urlObj = new URL(call.url);
                domains.add(urlObj.hostname);
            } catch (e) {
                // Skip invalid URLs
            }
        }

        const result = {
            status: 'success',
            url: url,
            total_requests: apiCalls.length,
            api_calls: apiCalls.filter(call => call.is_api_call),
            other_requests: apiCalls.filter(call => !call.is_api_call),
            summary: {
                api_calls_count: apiCalls.filter(call => call.is_api_call).length,
                other_requests_count: apiCalls.filter(call => !call.is_api_call).length,
                unique_domains: domains.size
            }
        };

        // Store in cache
        const now = Date.now();
        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
        await dbOps.setCacheOutgoingCalls(url, result, now, expiresAt);
        console.log(`Cached outgoing calls for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);

        return result;
    } catch (error) {
        console.error(`Error capturing outgoing calls from ${url}:`, error.message);

        return {
            status: 'error',
            url: url,
            error: error.message
        };
    } finally {
        // Always close the page, even on error
        if (page) {
            try {
                await page.close();
            } catch (closeError) {
                console.error('Error closing page:', closeError.message);
            }
        }
    }
}

async function extractResultingUrl(url, skipCache = false) {
    let page = null;
    try {
        // Check cache first (unless skipCache is true)
        if (!skipCache) {
            const now = Date.now();
            const cached = await dbOps.getCachedResultingUrl(url, now);
            if (cached) {
                console.log(`Resulting URL cache hit for: ${url}`);
                return cached;
            }
        }

        console.log(`Getting resulting URL for: ${url}`);

        // Get browser instance
        const browserInstance = await getBrowser();

        // Create new page
        page = await browserInstance.newPage();

        // Set reasonable timeouts
        page.setDefaultTimeout(30000);
        page.setDefaultNavigationTimeout(30000);

        // Navigate to URL and wait for network to be idle
        let response = null;
        try {
            response = await page.goto(url, {
                waitUntil: 'networkidle',
                timeout: 30000,
            });
        } catch (error) {
            console.log(`Error during page navigation: ${error.message}`);
            // Continue anyway to get the final URL
        }

        // Get the final URL after any redirects
        const finalUrl = page.url;

        // Get response status if available
        const statusCode = response ? response.status() : null;

        // Check if there was a redirect
        const redirected = finalUrl !== url;

        const result = {
            status: 'success',
            original_url: url,
            resulting_url: finalUrl,
            status_code: statusCode,
            redirected: redirected
        };

        // Store in cache
        const now = Date.now();
        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
        await dbOps.setCacheResultingUrl(url, result, now, expiresAt);
        console.log(`Cached resulting URL for: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);

        return result;
    } catch (error) {
        console.error(`Error getting resulting URL for ${url}:`, error.message);

        return {
            status: 'error',
            original_url: url,
            error: error.message
        };
    } finally {
        // Always close the page, even on error
        if (page) {
            try {
                await page.close();
            } catch (closeError) {
                console.error('Error closing page:', closeError.message);
            }
        }
    }
}

module.exports = {
    fetchHtml,
    extractSeo,
    extractMetaTags,
    extractOutgoingCalls,
    extractResultingUrl,
};