basic playwright api

Dockers/playwright-node-api/.dockerignore

@@ -0,0 +1,10 @@
+node_modules
+npm-debug.log
+.git
+.gitignore
+README.md
+.env
+*.db
+*.db-journal
+*.db-wal
+*.db-shm

Dockers/playwright-node-api/Dockerfile

@@ -0,0 +1,60 @@
+FROM node:18-slim
+
+# Install system dependencies for Playwright
+RUN apt-get update && apt-get install -y \
+    wget \
+    gnupg2 \
+    ca-certificates \
+    fonts-liberation \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libatspi2.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libnspr4 \
+    libnss3 \
+    libwayland-client0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxkbcommon0 \
+    libxrandr2 \
+    xdg-utils \
+    libu2f-udev \
+    libvulkan1 \
+    --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Copy package files
+COPY package*.json ./
+
+# Install Node.js dependencies
+RUN npm install
+
+# Install Playwright browsers
+RUN npx playwright install chromium
+RUN npx playwright install-deps chromium
+
+# Copy application files
+COPY . .
+
+# Create database directory
+RUN mkdir -p /db && chmod 777 /db
+
+# Set environment variables
+ENV PORT=3000
+ENV CACHE_EXPIRY_HOURS=24
+ENV DB_PATH=/db/cache.db
+
+# Expose port
+EXPOSE 3000
+
+# Run the application
+CMD ["npm", "start"]

Dockers/playwright-node-api/README.md

@@ -0,0 +1,117 @@
+# Playwright Node.js API
+
+A Node.js API service that uses Playwright to fetch webpage HTML with SQLite or PostgreSQL-based caching.
+
+## Features
+
+- Fetch full HTML of any webpage using Playwright
+- SQLite or PostgreSQL-based caching with configurable expiry (default: 24 hours)
+- API key authentication (optional, via `x-api-key` header)
+- Automatic cache cleanup and garbage collection
+- Proper browser/page cleanup on errors
+- Health check and cache statistics endpoints
+
+## API Endpoints
+
+### GET `/`
+
+Fetch HTML content of a webpage.
+
+**Query Parameters:**
+
+- `url` (required): The URL to fetch
+- `skipCache` (optional): Set to `true`, `1`, or `yes` to bypass cache
+
+**Headers:**
+
+- `x-api-key` (required if `API_KEY` env var is set): API key for authentication
+
+**Example:**
+
+```bash
+# Without API key (if API_KEY env var is not set)
+curl "http://localhost:3000/?url=https://example.com"
+
+# With API key
+curl -H "x-api-key: your-api-key" "http://localhost:3000/?url=https://example.com"
+curl -H "x-api-key: your-api-key" "http://localhost:3000/?url=https://example.com&skipCache=true"
+```
+
+### GET `/health`
+
+Health check endpoint. Returns API status and configuration.
+
+**Note:** This endpoint is not protected by API key authentication.
+
+### GET `/cache/stats`
+
+Get cache statistics (total entries, valid entries).
+
+### DELETE `/cache`
+
+Clear cache entries.
+
+**Query Parameters:**
+
+- `url` (optional): Clear specific URL from cache. If omitted, clears all cache.
+
+## Environment Variables
+
+- `PORT`: Server port (default: `3000`)
+- `CACHE_EXPIRY_HOURS`: Cache expiry time in hours (default: `24`)
+- `API_KEY`: API key for authentication (optional). If set, all endpoints except `/health` require the `x-api-key` header
+- `DB_PATH`: Path to SQLite database file (default: `/db/cache.db`) - only used if PostgreSQL is not configured
+
+### PostgreSQL Configuration (optional)
+
+If the following environment variables are set, the API will use PostgreSQL instead of SQLite:
+
+- `POSTGRES_HOST`: PostgreSQL host (e.g., `puppeteer-postgres`)
+- `POSTGRES_PORT`: PostgreSQL port (default: `5432`)
+- `POSTGRES_USER`: PostgreSQL username (e.g., `postgres`)
+- `POSTGRES_PASSWORD`: PostgreSQL password
+- `POSTGRES_DB`: PostgreSQL database name (e.g., `puppeteer`)
+
+If any of these PostgreSQL variables are missing, the API will fall back to SQLite.
+
+## Docker
+
+Build the image:
+
+```bash
+docker build -t playwright-node-api .
+```
+
+Run the container with SQLite:
+
+```bash
+docker run -d \
+  -p 3000:3000 \
+  -v /path/to/db:/db \
+  -e CACHE_EXPIRY_HOURS=24 \
+  -e API_KEY=your-secret-api-key \
+  playwright-node-api
+```
+
+Run the container with PostgreSQL:
+
+```bash
+docker run -d \
+  -p 3000:3000 \
+  -e POSTGRES_HOST=puppeteer-postgres \
+  -e POSTGRES_PORT=5432 \
+  -e POSTGRES_USER=postgres \
+  -e POSTGRES_PASSWORD=${PUPPETEER_DB_PASSWORD} \
+  -e POSTGRES_DB=puppeteer \
+  -e CACHE_EXPIRY_HOURS=24 \
+  -e API_KEY=your-secret-api-key \
+  playwright-node-api
+```
+
+## Notes
+
+- The service automatically cleans up expired cache entries every hour
+- Browser instances are reused for better performance
+- Pages are always closed after use, even on errors
+- Graceful shutdown is handled on SIGTERM/SIGINT
+- API key authentication is optional: if `API_KEY` is not set, all endpoints are publicly accessible (except `/health` which is always public)

Dockers/playwright-node-api/browser.js

@@ -0,0 +1,51 @@
+const { chromium } = require('playwright');
+
+let browser = null;
+let browserLaunchPromise = null;
+
+async function getBrowser() {
+    if (browser && browser.isConnected()) {
+        return browser;
+    }
+
+    if (browserLaunchPromise) {
+        return browserLaunchPromise;
+    }
+
+    browserLaunchPromise = chromium.launch({
+        headless: true,
+        args: [
+            '--no-sandbox',
+            '--disable-setuid-sandbox',
+            '--disable-dev-shm-usage',
+            '--disable-accelerated-2d-canvas',
+            '--disable-gpu',
+        ],
+    });
+
+    try {
+        browser = await browserLaunchPromise;
+        browserLaunchPromise = null;
+        return browser;
+    } catch (error) {
+        browserLaunchPromise = null;
+        throw error;
+    }
+}
+
+async function closeBrowser() {
+    if (browser) {
+        await browser.close();
+        browser = null;
+    }
+}
+
+function isBrowserConnected() {
+    return browser && browser.isConnected();
+}
+
+module.exports = {
+    getBrowser,
+    closeBrowser,
+    isBrowserConnected,
+};

Dockers/playwright-node-api/config.js

@@ -0,0 +1,17 @@
+module.exports = {
+    PORT: process.env.PORT || 3000,
+    CACHE_EXPIRY_HOURS: parseInt(process.env.CACHE_EXPIRY_HOURS || '24', 10),
+    API_KEY: process.env.API_KEY,
+
+    // Database configuration
+    POSTGRES_HOST: process.env.POSTGRES_HOST,
+    POSTGRES_PORT: parseInt(process.env.POSTGRES_PORT || '5432', 10),
+    POSTGRES_USER: process.env.POSTGRES_USER,
+    POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD,
+    POSTGRES_DB: process.env.POSTGRES_DB,
+    DB_PATH: process.env.DB_PATH || '/db/cache.db',
+
+    // Determine which database to use
+    USE_POSTGRES: !!(process.env.POSTGRES_HOST && process.env.POSTGRES_USER &&
+        process.env.POSTGRES_PASSWORD && process.env.POSTGRES_DB),
+};

Dockers/playwright-node-api/database.js

@@ -0,0 +1,298 @@
+const Database = require('better-sqlite3');
+const { Pool } = require('pg');
+const path = require('path');
+const fs = require('fs');
+const config = require('./config');
+
+let db = null;
+let pgPool = null;
+let dbType = 'sqlite';
+
+const dbOps = {
+    async init() {
+        if (config.USE_POSTGRES) {
+            await this.initPostgres();
+        } else {
+            this.initSqlite();
+        }
+    },
+
+    async initPostgres() {
+        dbType = 'postgres';
+        console.log(`Initializing PostgreSQL connection to ${config.POSTGRES_HOST}:${config.POSTGRES_PORT}/${config.POSTGRES_DB}`);
+
+        // Wait for PostgreSQL to be ready
+        await this.waitForPostgres();
+
+        // Create database if it doesn't exist
+        await this.createDatabaseIfNotExists();
+
+        // Create connection pool
+        pgPool = new Pool({
+            host: config.POSTGRES_HOST,
+            port: config.POSTGRES_PORT,
+            user: config.POSTGRES_USER,
+            password: config.POSTGRES_PASSWORD,
+            database: config.POSTGRES_DB,
+            max: 20,
+            idleTimeoutMillis: 30000,
+            connectionTimeoutMillis: 10000,
+        });
+
+        // Test connection
+        try {
+            const client = await pgPool.connect();
+            await client.query('SELECT 1');
+            client.release();
+            console.log('PostgreSQL connection established');
+        } catch (error) {
+            console.error('PostgreSQL connection failed:', error.message);
+            console.log('Falling back to SQLite');
+            this.initSqlite();
+            return;
+        }
+
+        // Create cache table
+        const client = await pgPool.connect();
+        try {
+            await client.query(`
+                CREATE TABLE IF NOT EXISTS cache (
+                    url TEXT PRIMARY KEY,
+                    html TEXT NOT NULL,
+                    created_at BIGINT NOT NULL,
+                    expires_at BIGINT NOT NULL
+                );
+
+                CREATE INDEX IF NOT EXISTS idx_expires_at ON cache(expires_at);
+            `);
+            await client.query('COMMIT');
+        } catch (error) {
+            console.error('Error creating PostgreSQL tables:', error);
+        } finally {
+            client.release();
+        }
+    },
+
+    async waitForPostgres(maxRetries = 30, retryDelay = 2000) {
+        console.log(`Waiting for PostgreSQL at ${config.POSTGRES_HOST}:${config.POSTGRES_PORT}...`);
+
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                const tempPool = new Pool({
+                    host: config.POSTGRES_HOST,
+                    port: config.POSTGRES_PORT,
+                    user: config.POSTGRES_USER,
+                    password: config.POSTGRES_PASSWORD,
+                    database: 'postgres',
+                    connectionTimeoutMillis: 5000,
+                });
+                const client = await tempPool.connect();
+                await client.query('SELECT 1');
+                client.release();
+                await tempPool.end();
+                console.log(`PostgreSQL is ready after ${attempt + 1} attempts`);
+                return true;
+            } catch (error) {
+                if (attempt < maxRetries - 1) {
+                    console.log(`PostgreSQL not ready (attempt ${attempt + 1}/${maxRetries}): ${error.message}`);
+                    await new Promise(resolve => setTimeout(resolve, retryDelay));
+                } else {
+                    console.error(`PostgreSQL connection failed after ${maxRetries} attempts: ${error.message}`);
+                    return false;
+                }
+            }
+        }
+        return false;
+    },
+
+    async createDatabaseIfNotExists() {
+        try {
+            const tempPool = new Pool({
+                host: config.POSTGRES_HOST,
+                port: config.POSTGRES_PORT,
+                user: config.POSTGRES_USER,
+                password: config.POSTGRES_PASSWORD,
+                database: 'postgres',
+            });
+            const client = await tempPool.connect();
+
+            // Check if database exists
+            const result = await client.query(
+                'SELECT 1 FROM pg_database WHERE datname = $1',
+                [config.POSTGRES_DB]
+            );
+
+            if (result.rows.length === 0) {
+                console.log(`Creating database '${config.POSTGRES_DB}'...`);
+                // Note: CREATE DATABASE cannot be run in a transaction
+                await client.query(`CREATE DATABASE ${config.POSTGRES_DB}`);
+                console.log(`Database '${config.POSTGRES_DB}' created successfully`);
+            } else {
+                console.log(`Database '${config.POSTGRES_DB}' already exists`);
+            }
+
+            client.release();
+            await tempPool.end();
+            return true;
+        } catch (error) {
+            console.error(`Error creating database: ${error.message}`);
+            return false;
+        }
+    },
+
+    initSqlite() {
+        dbType = 'sqlite';
+        console.log(`Initializing SQLite database at ${config.DB_PATH}`);
+
+        // Ensure database directory exists
+        const dbDir = path.dirname(config.DB_PATH);
+        if (!fs.existsSync(dbDir)) {
+            fs.mkdirSync(dbDir, { recursive: true });
+        }
+
+        // Initialize SQLite database
+        db = new Database(config.DB_PATH);
+        db.pragma('journal_mode = WAL');
+
+        // Create cache table if it doesn't exist
+        db.exec(`
+            CREATE TABLE IF NOT EXISTS cache (
+                url TEXT PRIMARY KEY,
+                html TEXT NOT NULL,
+                created_at INTEGER NOT NULL,
+                expires_at INTEGER NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_expires_at ON cache(expires_at);
+        `);
+    },
+
+    async getCached(url, now) {
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'SELECT html FROM cache WHERE url = $1 AND expires_at > $2',
+                    [url, now]
+                );
+                return result.rows.length > 0 ? result.rows[0].html : null;
+            } finally {
+                client.release();
+            }
+        } else {
+            const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?');
+            const result = getCached.get(url, now);
+            return result ? result.html : null;
+        }
+    },
+
+    async setCache(url, html, now, expiresAt) {
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                await client.query(
+                    'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4',
+                    [url, html, now, expiresAt]
+                );
+            } finally {
+                client.release();
+            }
+        } else {
+            const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)');
+            setCache.run(url, html, now, expiresAt);
+        }
+    },
+
+    async deleteExpired(now) {
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'DELETE FROM cache WHERE expires_at <= $1',
+                    [now]
+                );
+                return result.rowCount;
+            } finally {
+                client.release();
+            }
+        } else {
+            const deleteExpired = db.prepare('DELETE FROM cache WHERE expires_at <= ?');
+            const result = deleteExpired.run(now);
+            return result.changes;
+        }
+    },
+
+    async deleteByUrl(url) {
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query(
+                    'DELETE FROM cache WHERE url = $1',
+                    [url]
+                );
+                return result.rowCount;
+            } finally {
+                client.release();
+            }
+        } else {
+            const deleteByUrl = db.prepare('DELETE FROM cache WHERE url = ?');
+            const result = deleteByUrl.run(url);
+            return result.changes;
+        }
+    },
+
+    async deleteAll() {
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const result = await client.query('DELETE FROM cache');
+                return result.rowCount;
+            } finally {
+                client.release();
+            }
+        } else {
+            const result = db.prepare('DELETE FROM cache').run();
+            return result.changes;
+        }
+    },
+
+    async getStats() {
+        const now = Date.now();
+        if (dbType === 'postgres') {
+            const client = await pgPool.connect();
+            try {
+                const totalResult = await client.query('SELECT COUNT(*) as total FROM cache');
+                const validResult = await client.query(
+                    'SELECT COUNT(*) as valid FROM cache WHERE expires_at > $1',
+                    [now]
+                );
+                return {
+                    total: parseInt(totalResult.rows[0].total, 10),
+                    valid: parseInt(validResult.rows[0].valid, 10),
+                };
+            } finally {
+                client.release();
+            }
+        } else {
+            const stats = db.prepare('SELECT COUNT(*) as total, COUNT(CASE WHEN expires_at > ? THEN 1 END) as valid FROM cache').get(now);
+            return {
+                total: stats.total,
+                valid: stats.valid,
+            };
+        }
+    },
+
+    async close() {
+        if (dbType === 'postgres' && pgPool) {
+            await pgPool.end();
+        } else if (db) {
+            db.close();
+        }
+    },
+
+    getDbType() {
+        return dbType;
+    },
+};
+
+module.exports = dbOps;

Dockers/playwright-node-api/index.js

@@ -0,0 +1,59 @@
+const express = require('express');
+const dbOps = require('./database');
+const { closeBrowser } = require('./browser');
+const routes = require('./routes');
+const config = require('./config');
+
+const app = express();
+
+// Use routes
+app.use('/', routes);
+
+// Cleanup expired cache entries
+async function cleanupExpiredCache() {
+    const now = Date.now();
+    const deleted = await dbOps.deleteExpired(now);
+    if (deleted > 0) {
+        console.log(`Cleaned up ${deleted} expired cache entries`);
+    }
+}
+
+// Initialize database and start server
+(async () => {
+    try {
+        await dbOps.init();
+        console.log(`Database initialized: ${dbOps.getDbType()}`);
+
+        // Periodic cache cleanup (every hour)
+        setInterval(cleanupExpiredCache, 60 * 60 * 1000);
+
+        // Initial cleanup on startup
+        await cleanupExpiredCache();
+
+        // Start server
+        app.listen(config.PORT, '0.0.0.0', () => {
+            console.log(`Playwright API server listening on port ${config.PORT}`);
+            console.log(`Cache expiry: ${config.CACHE_EXPIRY_HOURS} hours`);
+            console.log(`Database: ${dbOps.getDbType()}${dbOps.getDbType() === 'postgres' ? ` (${config.POSTGRES_HOST}:${config.POSTGRES_PORT}/${config.POSTGRES_DB})` : ` (${config.DB_PATH})`}`);
+            console.log(`API Key authentication: ${config.API_KEY ? 'enabled' : 'disabled'}`);
+        });
+    } catch (error) {
+        console.error('Failed to initialize:', error);
+        process.exit(1);
+    }
+})();
+
+// Graceful shutdown
+process.on('SIGTERM', async () => {
+    console.log('SIGTERM received, shutting down gracefully...');
+    await closeBrowser();
+    await dbOps.close();
+    process.exit(0);
+});
+
+process.on('SIGINT', async () => {
+    console.log('SIGINT received, shutting down gracefully...');
+    await closeBrowser();
+    await dbOps.close();
+    process.exit(0);
+});

Dockers/playwright-node-api/middleware.js

@@ -0,0 +1,30 @@
+const config = require('./config');
+
+function authenticateApiKey(req, res, next) {
+    // If no API key is configured, skip authentication
+    if (!config.API_KEY) {
+        return next();
+    }
+
+    const providedKey = req.headers['x-api-key'];
+
+    if (!providedKey) {
+        return res.status(401).json({
+            error: 'Missing API key',
+            message: 'Please provide an API key in the x-api-key header',
+        });
+    }
+
+    if (providedKey !== config.API_KEY) {
+        return res.status(403).json({
+            error: 'Invalid API key',
+            message: 'The provided API key is not valid',
+        });
+    }
+
+    next();
+}
+
+module.exports = {
+    authenticateApiKey,
+};

Dockers/playwright-node-api/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "playwright-node-api",
+    "version": "1.0.0",
+    "description": "Node.js API using Playwright to fetch webpage HTML with caching",
+    "main": "index.js",
+    "author": "Bram Kelchtermans",
+    "license": "MIT",
+    "scripts": {
+        "start": "node index.js"
+    },
+    "dependencies": {
+        "express": "^4.18.2",
+        "playwright": "^1.40.0",
+        "better-sqlite3": "^9.2.2",
+        "pg": "^8.11.3"
+    }
+}

Dockers/playwright-node-api/routes.js

@@ -0,0 +1,84 @@
+const express = require('express');
+const { fetchHtml } = require('./scraper');
+const dbOps = require('./database');
+const { authenticateApiKey } = require('./middleware');
+const { isBrowserConnected } = require('./browser');
+const config = require('./config');
+
+const router = express.Router();
+
+// Main API endpoint
+router.get('/', authenticateApiKey, async (req, res) => {
+    const { url, skipCache } = req.query;
+
+    // Validate URL parameter
+    if (!url) {
+        return res.status(400).json({
+            error: 'Missing required parameter: url',
+        });
+    }
+
+    // Validate URL format
+    let urlObj;
+    try {
+        urlObj = new URL(url);
+    } catch (error) {
+        return res.status(400).json({
+            error: 'Invalid URL format',
+        });
+    }
+
+    // Parse skipCache (accept 'true', '1', 'yes', etc.)
+    const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes';
+
+    try {
+        const html = await fetchHtml(url, shouldSkipCache);
+        res.setHeader('Content-Type', 'text/html; charset=utf-8');
+        res.send(html);
+    } catch (error) {
+        console.error('Request error:', error);
+        res.status(500).json({
+            error: 'Failed to fetch webpage',
+            message: error.message,
+        });
+    }
+});
+
+// Health check endpoint
+router.get('/health', async (req, res) => {
+    res.json({
+        status: 'ok',
+        cacheExpiryHours: config.CACHE_EXPIRY_HOURS,
+        browserConnected: isBrowserConnected(),
+        database: dbOps.getDbType(),
+    });
+});
+
+// Cache stats endpoint
+router.get('/cache/stats', authenticateApiKey, async (req, res) => {
+    try {
+        const stats = await dbOps.getStats();
+        res.json(stats);
+    } catch (error) {
+        res.status(500).json({ error: error.message });
+    }
+});
+
+// Clear cache endpoint (optional, for maintenance)
+router.delete('/cache', authenticateApiKey, async (req, res) => {
+    try {
+        const { url } = req.query;
+        if (url) {
+            const deleted = await dbOps.deleteByUrl(url);
+            res.json({ deleted });
+        } else {
+            // Clear all cache
+            const deleted = await dbOps.deleteAll();
+            res.json({ deleted });
+        }
+    } catch (error) {
+        res.status(500).json({ error: error.message });
+    }
+});
+
+module.exports = router;

Dockers/playwright-node-api/scraper.js

@@ -0,0 +1,103 @@
+const dbOps = require('./database');
+const { getBrowser } = require('./browser');
+const config = require('./config');
+
+async function fetchHtml(url, skipCache = false) {
+    let page = null;
+    try {
+        // Check cache first (unless skipCache is true)
+        if (!skipCache) {
+            const now = Date.now();
+            const cached = await dbOps.getCached(url, now);
+            if (cached) {
+                console.log(`Cache hit for: ${url}`);
+                return cached;
+            }
+        }
+
+        console.log(`Fetching: ${url} (skipCache: ${skipCache})`);
+
+        // Get browser instance
+        const browserInstance = await getBrowser();
+
+        // Create new page
+        page = await browserInstance.newPage();
+
+        // Set reasonable timeouts
+        page.setDefaultTimeout(30000);
+        page.setDefaultNavigationTimeout(30000);
+
+        // Track network requests to ensure all are completed
+        const pendingRequests = new Set();
+        const finishedRequests = new Set();
+
+        // Monitor network requests
+        page.on('request', (request) => {
+            const requestId = request.url();
+            pendingRequests.add(requestId);
+        });
+
+        page.on('response', (response) => {
+            const requestId = response.url();
+            finishedRequests.add(requestId);
+            pendingRequests.delete(requestId);
+        });
+
+        page.on('requestfailed', (request) => {
+            const requestId = request.url();
+            pendingRequests.delete(requestId);
+        });
+
+        // Navigate to URL
+        await page.goto(url, {
+            waitUntil: 'domcontentloaded',
+            timeout: 30000,
+        });
+
+        // Wait for network to be idle (no requests for 500ms)
+        await page.waitForLoadState('networkidle', { timeout: 30000 });
+
+        // Additional wait to ensure all pending requests complete
+        const maxWaitTime = 10000; // 10 seconds max
+        const checkInterval = 100; // Check every 100ms
+        const startTime = Date.now();
+
+        while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) {
+            await new Promise(resolve => setTimeout(resolve, checkInterval));
+        }
+
+        if (pendingRequests.size > 0) {
+            console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`);
+        }
+
+        // Get full HTML
+        const html = await page.content();
+
+        // Store in cache
+        const now = Date.now();
+        const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000);
+        await dbOps.setCache(url, html, now, expiresAt);
+        console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`);
+
+        return html;
+    } catch (error) {
+        console.error(`Error fetching ${url}:`, error.message);
+
+        // If cache exists and we had an error, optionally return stale cache
+        // For now, we'll just throw the error
+        throw error;
+    } finally {
+        // Always close the page, even on error
+        if (page) {
+            try {
+                await page.close();
+            } catch (closeError) {
+                console.error('Error closing page:', closeError.message);
+            }
+        }
+    }
+}
+
+module.exports = {
+    fetchHtml,
+};

Dockers/playwright-node-api/version

@@ -0,0 +1 @@
+1.0.0