diff --git a/Dockers/playwright-node-api/.dockerignore b/Dockers/playwright-node-api/.dockerignore new file mode 100644 index 0000000..1c5414f --- /dev/null +++ b/Dockers/playwright-node-api/.dockerignore @@ -0,0 +1,10 @@ +node_modules +npm-debug.log +.git +.gitignore +README.md +.env +*.db +*.db-journal +*.db-wal +*.db-shm diff --git a/Dockers/playwright-node-api/Dockerfile b/Dockers/playwright-node-api/Dockerfile new file mode 100644 index 0000000..7936b1e --- /dev/null +++ b/Dockers/playwright-node-api/Dockerfile @@ -0,0 +1,60 @@ +FROM node:18-slim + +# Install system dependencies for Playwright +RUN apt-get update && apt-get install -y \ + wget \ + gnupg2 \ + ca-certificates \ + fonts-liberation \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libwayland-client0 \ + libxcomposite1 \ + libxdamage1 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + xdg-utils \ + libu2f-udev \ + libvulkan1 \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install Node.js dependencies +RUN npm install + +# Install Playwright browsers +RUN npx playwright install chromium +RUN npx playwright install-deps chromium + +# Copy application files +COPY . . + +# Create database directory +RUN mkdir -p /db && chmod 777 /db + +# Set environment variables +ENV PORT=3000 +ENV CACHE_EXPIRY_HOURS=24 +ENV DB_PATH=/db/cache.db + +# Expose port +EXPOSE 3000 + +# Run the application +CMD ["npm", "start"] diff --git a/Dockers/playwright-node-api/README.md b/Dockers/playwright-node-api/README.md new file mode 100644 index 0000000..dba8647 --- /dev/null +++ b/Dockers/playwright-node-api/README.md @@ -0,0 +1,117 @@ +# Playwright Node.js API + +A Node.js API service that uses Playwright to fetch webpage HTML with SQLite or PostgreSQL-based caching. + +## Features + +- Fetch full HTML of any webpage using Playwright +- SQLite or PostgreSQL-based caching with configurable expiry (default: 24 hours) +- API key authentication (optional, via `x-api-key` header) +- Automatic cache cleanup and garbage collection +- Proper browser/page cleanup on errors +- Health check and cache statistics endpoints + +## API Endpoints + +### GET `/` + +Fetch HTML content of a webpage. + +**Query Parameters:** + +- `url` (required): The URL to fetch +- `skipCache` (optional): Set to `true`, `1`, or `yes` to bypass cache + +**Headers:** + +- `x-api-key` (required if `API_KEY` env var is set): API key for authentication + +**Example:** + +```bash +# Without API key (if API_KEY env var is not set) +curl "http://localhost:3000/?url=https://example.com" + +# With API key +curl -H "x-api-key: your-api-key" "http://localhost:3000/?url=https://example.com" +curl -H "x-api-key: your-api-key" "http://localhost:3000/?url=https://example.com&skipCache=true" +``` + +### GET `/health` + +Health check endpoint. Returns API status and configuration. + +**Note:** This endpoint is not protected by API key authentication. + +### GET `/cache/stats` + +Get cache statistics (total entries, valid entries). + +### DELETE `/cache` + +Clear cache entries. + +**Query Parameters:** + +- `url` (optional): Clear specific URL from cache. If omitted, clears all cache. + +## Environment Variables + +- `PORT`: Server port (default: `3000`) +- `CACHE_EXPIRY_HOURS`: Cache expiry time in hours (default: `24`) +- `API_KEY`: API key for authentication (optional). If set, all endpoints except `/health` require the `x-api-key` header +- `DB_PATH`: Path to SQLite database file (default: `/db/cache.db`) - only used if PostgreSQL is not configured + +### PostgreSQL Configuration (optional) + +If the following environment variables are set, the API will use PostgreSQL instead of SQLite: + +- `POSTGRES_HOST`: PostgreSQL host (e.g., `puppeteer-postgres`) +- `POSTGRES_PORT`: PostgreSQL port (default: `5432`) +- `POSTGRES_USER`: PostgreSQL username (e.g., `postgres`) +- `POSTGRES_PASSWORD`: PostgreSQL password +- `POSTGRES_DB`: PostgreSQL database name (e.g., `puppeteer`) + +If any of these PostgreSQL variables are missing, the API will fall back to SQLite. + +## Docker + +Build the image: + +```bash +docker build -t playwright-node-api . +``` + +Run the container with SQLite: + +```bash +docker run -d \ + -p 3000:3000 \ + -v /path/to/db:/db \ + -e CACHE_EXPIRY_HOURS=24 \ + -e API_KEY=your-secret-api-key \ + playwright-node-api +``` + +Run the container with PostgreSQL: + +```bash +docker run -d \ + -p 3000:3000 \ + -e POSTGRES_HOST=puppeteer-postgres \ + -e POSTGRES_PORT=5432 \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=${PUPPETEER_DB_PASSWORD} \ + -e POSTGRES_DB=puppeteer \ + -e CACHE_EXPIRY_HOURS=24 \ + -e API_KEY=your-secret-api-key \ + playwright-node-api +``` + +## Notes + +- The service automatically cleans up expired cache entries every hour +- Browser instances are reused for better performance +- Pages are always closed after use, even on errors +- Graceful shutdown is handled on SIGTERM/SIGINT +- API key authentication is optional: if `API_KEY` is not set, all endpoints are publicly accessible (except `/health` which is always public) diff --git a/Dockers/playwright-node-api/browser.js b/Dockers/playwright-node-api/browser.js new file mode 100644 index 0000000..327468c --- /dev/null +++ b/Dockers/playwright-node-api/browser.js @@ -0,0 +1,51 @@ +const { chromium } = require('playwright'); + +let browser = null; +let browserLaunchPromise = null; + +async function getBrowser() { + if (browser && browser.isConnected()) { + return browser; + } + + if (browserLaunchPromise) { + return browserLaunchPromise; + } + + browserLaunchPromise = chromium.launch({ + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--disable-gpu', + ], + }); + + try { + browser = await browserLaunchPromise; + browserLaunchPromise = null; + return browser; + } catch (error) { + browserLaunchPromise = null; + throw error; + } +} + +async function closeBrowser() { + if (browser) { + await browser.close(); + browser = null; + } +} + +function isBrowserConnected() { + return browser && browser.isConnected(); +} + +module.exports = { + getBrowser, + closeBrowser, + isBrowserConnected, +}; diff --git a/Dockers/playwright-node-api/config.js b/Dockers/playwright-node-api/config.js new file mode 100644 index 0000000..ff14449 --- /dev/null +++ b/Dockers/playwright-node-api/config.js @@ -0,0 +1,17 @@ +module.exports = { + PORT: process.env.PORT || 3000, + CACHE_EXPIRY_HOURS: parseInt(process.env.CACHE_EXPIRY_HOURS || '24', 10), + API_KEY: process.env.API_KEY, + + // Database configuration + POSTGRES_HOST: process.env.POSTGRES_HOST, + POSTGRES_PORT: parseInt(process.env.POSTGRES_PORT || '5432', 10), + POSTGRES_USER: process.env.POSTGRES_USER, + POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD, + POSTGRES_DB: process.env.POSTGRES_DB, + DB_PATH: process.env.DB_PATH || '/db/cache.db', + + // Determine which database to use + USE_POSTGRES: !!(process.env.POSTGRES_HOST && process.env.POSTGRES_USER && + process.env.POSTGRES_PASSWORD && process.env.POSTGRES_DB), +}; diff --git a/Dockers/playwright-node-api/database.js b/Dockers/playwright-node-api/database.js new file mode 100644 index 0000000..d95797f --- /dev/null +++ b/Dockers/playwright-node-api/database.js @@ -0,0 +1,298 @@ +const Database = require('better-sqlite3'); +const { Pool } = require('pg'); +const path = require('path'); +const fs = require('fs'); +const config = require('./config'); + +let db = null; +let pgPool = null; +let dbType = 'sqlite'; + +const dbOps = { + async init() { + if (config.USE_POSTGRES) { + await this.initPostgres(); + } else { + this.initSqlite(); + } + }, + + async initPostgres() { + dbType = 'postgres'; + console.log(`Initializing PostgreSQL connection to ${config.POSTGRES_HOST}:${config.POSTGRES_PORT}/${config.POSTGRES_DB}`); + + // Wait for PostgreSQL to be ready + await this.waitForPostgres(); + + // Create database if it doesn't exist + await this.createDatabaseIfNotExists(); + + // Create connection pool + pgPool = new Pool({ + host: config.POSTGRES_HOST, + port: config.POSTGRES_PORT, + user: config.POSTGRES_USER, + password: config.POSTGRES_PASSWORD, + database: config.POSTGRES_DB, + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 10000, + }); + + // Test connection + try { + const client = await pgPool.connect(); + await client.query('SELECT 1'); + client.release(); + console.log('PostgreSQL connection established'); + } catch (error) { + console.error('PostgreSQL connection failed:', error.message); + console.log('Falling back to SQLite'); + this.initSqlite(); + return; + } + + // Create cache table + const client = await pgPool.connect(); + try { + await client.query(` + CREATE TABLE IF NOT EXISTS cache ( + url TEXT PRIMARY KEY, + html TEXT NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_expires_at ON cache(expires_at); + `); + await client.query('COMMIT'); + } catch (error) { + console.error('Error creating PostgreSQL tables:', error); + } finally { + client.release(); + } + }, + + async waitForPostgres(maxRetries = 30, retryDelay = 2000) { + console.log(`Waiting for PostgreSQL at ${config.POSTGRES_HOST}:${config.POSTGRES_PORT}...`); + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + const tempPool = new Pool({ + host: config.POSTGRES_HOST, + port: config.POSTGRES_PORT, + user: config.POSTGRES_USER, + password: config.POSTGRES_PASSWORD, + database: 'postgres', + connectionTimeoutMillis: 5000, + }); + const client = await tempPool.connect(); + await client.query('SELECT 1'); + client.release(); + await tempPool.end(); + console.log(`PostgreSQL is ready after ${attempt + 1} attempts`); + return true; + } catch (error) { + if (attempt < maxRetries - 1) { + console.log(`PostgreSQL not ready (attempt ${attempt + 1}/${maxRetries}): ${error.message}`); + await new Promise(resolve => setTimeout(resolve, retryDelay)); + } else { + console.error(`PostgreSQL connection failed after ${maxRetries} attempts: ${error.message}`); + return false; + } + } + } + return false; + }, + + async createDatabaseIfNotExists() { + try { + const tempPool = new Pool({ + host: config.POSTGRES_HOST, + port: config.POSTGRES_PORT, + user: config.POSTGRES_USER, + password: config.POSTGRES_PASSWORD, + database: 'postgres', + }); + const client = await tempPool.connect(); + + // Check if database exists + const result = await client.query( + 'SELECT 1 FROM pg_database WHERE datname = $1', + [config.POSTGRES_DB] + ); + + if (result.rows.length === 0) { + console.log(`Creating database '${config.POSTGRES_DB}'...`); + // Note: CREATE DATABASE cannot be run in a transaction + await client.query(`CREATE DATABASE ${config.POSTGRES_DB}`); + console.log(`Database '${config.POSTGRES_DB}' created successfully`); + } else { + console.log(`Database '${config.POSTGRES_DB}' already exists`); + } + + client.release(); + await tempPool.end(); + return true; + } catch (error) { + console.error(`Error creating database: ${error.message}`); + return false; + } + }, + + initSqlite() { + dbType = 'sqlite'; + console.log(`Initializing SQLite database at ${config.DB_PATH}`); + + // Ensure database directory exists + const dbDir = path.dirname(config.DB_PATH); + if (!fs.existsSync(dbDir)) { + fs.mkdirSync(dbDir, { recursive: true }); + } + + // Initialize SQLite database + db = new Database(config.DB_PATH); + db.pragma('journal_mode = WAL'); + + // Create cache table if it doesn't exist + db.exec(` + CREATE TABLE IF NOT EXISTS cache ( + url TEXT PRIMARY KEY, + html TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_expires_at ON cache(expires_at); + `); + }, + + async getCached(url, now) { + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'SELECT html FROM cache WHERE url = $1 AND expires_at > $2', + [url, now] + ); + return result.rows.length > 0 ? result.rows[0].html : null; + } finally { + client.release(); + } + } else { + const getCached = db.prepare('SELECT html FROM cache WHERE url = ? AND expires_at > ?'); + const result = getCached.get(url, now); + return result ? result.html : null; + } + }, + + async setCache(url, html, now, expiresAt) { + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + await client.query( + 'INSERT INTO cache (url, html, created_at, expires_at) VALUES ($1, $2, $3, $4) ON CONFLICT (url) DO UPDATE SET html = $2, created_at = $3, expires_at = $4', + [url, html, now, expiresAt] + ); + } finally { + client.release(); + } + } else { + const setCache = db.prepare('INSERT OR REPLACE INTO cache (url, html, created_at, expires_at) VALUES (?, ?, ?, ?)'); + setCache.run(url, html, now, expiresAt); + } + }, + + async deleteExpired(now) { + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'DELETE FROM cache WHERE expires_at <= $1', + [now] + ); + return result.rowCount; + } finally { + client.release(); + } + } else { + const deleteExpired = db.prepare('DELETE FROM cache WHERE expires_at <= ?'); + const result = deleteExpired.run(now); + return result.changes; + } + }, + + async deleteByUrl(url) { + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query( + 'DELETE FROM cache WHERE url = $1', + [url] + ); + return result.rowCount; + } finally { + client.release(); + } + } else { + const deleteByUrl = db.prepare('DELETE FROM cache WHERE url = ?'); + const result = deleteByUrl.run(url); + return result.changes; + } + }, + + async deleteAll() { + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const result = await client.query('DELETE FROM cache'); + return result.rowCount; + } finally { + client.release(); + } + } else { + const result = db.prepare('DELETE FROM cache').run(); + return result.changes; + } + }, + + async getStats() { + const now = Date.now(); + if (dbType === 'postgres') { + const client = await pgPool.connect(); + try { + const totalResult = await client.query('SELECT COUNT(*) as total FROM cache'); + const validResult = await client.query( + 'SELECT COUNT(*) as valid FROM cache WHERE expires_at > $1', + [now] + ); + return { + total: parseInt(totalResult.rows[0].total, 10), + valid: parseInt(validResult.rows[0].valid, 10), + }; + } finally { + client.release(); + } + } else { + const stats = db.prepare('SELECT COUNT(*) as total, COUNT(CASE WHEN expires_at > ? THEN 1 END) as valid FROM cache').get(now); + return { + total: stats.total, + valid: stats.valid, + }; + } + }, + + async close() { + if (dbType === 'postgres' && pgPool) { + await pgPool.end(); + } else if (db) { + db.close(); + } + }, + + getDbType() { + return dbType; + }, +}; + +module.exports = dbOps; diff --git a/Dockers/playwright-node-api/index.js b/Dockers/playwright-node-api/index.js new file mode 100644 index 0000000..c376964 --- /dev/null +++ b/Dockers/playwright-node-api/index.js @@ -0,0 +1,59 @@ +const express = require('express'); +const dbOps = require('./database'); +const { closeBrowser } = require('./browser'); +const routes = require('./routes'); +const config = require('./config'); + +const app = express(); + +// Use routes +app.use('/', routes); + +// Cleanup expired cache entries +async function cleanupExpiredCache() { + const now = Date.now(); + const deleted = await dbOps.deleteExpired(now); + if (deleted > 0) { + console.log(`Cleaned up ${deleted} expired cache entries`); + } +} + +// Initialize database and start server +(async () => { + try { + await dbOps.init(); + console.log(`Database initialized: ${dbOps.getDbType()}`); + + // Periodic cache cleanup (every hour) + setInterval(cleanupExpiredCache, 60 * 60 * 1000); + + // Initial cleanup on startup + await cleanupExpiredCache(); + + // Start server + app.listen(config.PORT, '0.0.0.0', () => { + console.log(`Playwright API server listening on port ${config.PORT}`); + console.log(`Cache expiry: ${config.CACHE_EXPIRY_HOURS} hours`); + console.log(`Database: ${dbOps.getDbType()}${dbOps.getDbType() === 'postgres' ? ` (${config.POSTGRES_HOST}:${config.POSTGRES_PORT}/${config.POSTGRES_DB})` : ` (${config.DB_PATH})`}`); + console.log(`API Key authentication: ${config.API_KEY ? 'enabled' : 'disabled'}`); + }); + } catch (error) { + console.error('Failed to initialize:', error); + process.exit(1); + } +})(); + +// Graceful shutdown +process.on('SIGTERM', async () => { + console.log('SIGTERM received, shutting down gracefully...'); + await closeBrowser(); + await dbOps.close(); + process.exit(0); +}); + +process.on('SIGINT', async () => { + console.log('SIGINT received, shutting down gracefully...'); + await closeBrowser(); + await dbOps.close(); + process.exit(0); +}); diff --git a/Dockers/playwright-node-api/middleware.js b/Dockers/playwright-node-api/middleware.js new file mode 100644 index 0000000..23c1a43 --- /dev/null +++ b/Dockers/playwright-node-api/middleware.js @@ -0,0 +1,30 @@ +const config = require('./config'); + +function authenticateApiKey(req, res, next) { + // If no API key is configured, skip authentication + if (!config.API_KEY) { + return next(); + } + + const providedKey = req.headers['x-api-key']; + + if (!providedKey) { + return res.status(401).json({ + error: 'Missing API key', + message: 'Please provide an API key in the x-api-key header', + }); + } + + if (providedKey !== config.API_KEY) { + return res.status(403).json({ + error: 'Invalid API key', + message: 'The provided API key is not valid', + }); + } + + next(); +} + +module.exports = { + authenticateApiKey, +}; diff --git a/Dockers/playwright-node-api/package.json b/Dockers/playwright-node-api/package.json new file mode 100644 index 0000000..9305592 --- /dev/null +++ b/Dockers/playwright-node-api/package.json @@ -0,0 +1,17 @@ +{ + "name": "playwright-node-api", + "version": "1.0.0", + "description": "Node.js API using Playwright to fetch webpage HTML with caching", + "main": "index.js", + "author": "Bram Kelchtermans", + "license": "MIT", + "scripts": { + "start": "node index.js" + }, + "dependencies": { + "express": "^4.18.2", + "playwright": "^1.40.0", + "better-sqlite3": "^9.2.2", + "pg": "^8.11.3" + } +} \ No newline at end of file diff --git a/Dockers/playwright-node-api/routes.js b/Dockers/playwright-node-api/routes.js new file mode 100644 index 0000000..203a5ca --- /dev/null +++ b/Dockers/playwright-node-api/routes.js @@ -0,0 +1,84 @@ +const express = require('express'); +const { fetchHtml } = require('./scraper'); +const dbOps = require('./database'); +const { authenticateApiKey } = require('./middleware'); +const { isBrowserConnected } = require('./browser'); +const config = require('./config'); + +const router = express.Router(); + +// Main API endpoint +router.get('/', authenticateApiKey, async (req, res) => { + const { url, skipCache } = req.query; + + // Validate URL parameter + if (!url) { + return res.status(400).json({ + error: 'Missing required parameter: url', + }); + } + + // Validate URL format + let urlObj; + try { + urlObj = new URL(url); + } catch (error) { + return res.status(400).json({ + error: 'Invalid URL format', + }); + } + + // Parse skipCache (accept 'true', '1', 'yes', etc.) + const shouldSkipCache = skipCache === 'true' || skipCache === '1' || skipCache === 'yes'; + + try { + const html = await fetchHtml(url, shouldSkipCache); + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.send(html); + } catch (error) { + console.error('Request error:', error); + res.status(500).json({ + error: 'Failed to fetch webpage', + message: error.message, + }); + } +}); + +// Health check endpoint +router.get('/health', async (req, res) => { + res.json({ + status: 'ok', + cacheExpiryHours: config.CACHE_EXPIRY_HOURS, + browserConnected: isBrowserConnected(), + database: dbOps.getDbType(), + }); +}); + +// Cache stats endpoint +router.get('/cache/stats', authenticateApiKey, async (req, res) => { + try { + const stats = await dbOps.getStats(); + res.json(stats); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +// Clear cache endpoint (optional, for maintenance) +router.delete('/cache', authenticateApiKey, async (req, res) => { + try { + const { url } = req.query; + if (url) { + const deleted = await dbOps.deleteByUrl(url); + res.json({ deleted }); + } else { + // Clear all cache + const deleted = await dbOps.deleteAll(); + res.json({ deleted }); + } + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +module.exports = router; diff --git a/Dockers/playwright-node-api/scraper.js b/Dockers/playwright-node-api/scraper.js new file mode 100644 index 0000000..ae0a643 --- /dev/null +++ b/Dockers/playwright-node-api/scraper.js @@ -0,0 +1,103 @@ +const dbOps = require('./database'); +const { getBrowser } = require('./browser'); +const config = require('./config'); + +async function fetchHtml(url, skipCache = false) { + let page = null; + try { + // Check cache first (unless skipCache is true) + if (!skipCache) { + const now = Date.now(); + const cached = await dbOps.getCached(url, now); + if (cached) { + console.log(`Cache hit for: ${url}`); + return cached; + } + } + + console.log(`Fetching: ${url} (skipCache: ${skipCache})`); + + // Get browser instance + const browserInstance = await getBrowser(); + + // Create new page + page = await browserInstance.newPage(); + + // Set reasonable timeouts + page.setDefaultTimeout(30000); + page.setDefaultNavigationTimeout(30000); + + // Track network requests to ensure all are completed + const pendingRequests = new Set(); + const finishedRequests = new Set(); + + // Monitor network requests + page.on('request', (request) => { + const requestId = request.url(); + pendingRequests.add(requestId); + }); + + page.on('response', (response) => { + const requestId = response.url(); + finishedRequests.add(requestId); + pendingRequests.delete(requestId); + }); + + page.on('requestfailed', (request) => { + const requestId = request.url(); + pendingRequests.delete(requestId); + }); + + // Navigate to URL + await page.goto(url, { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); + + // Wait for network to be idle (no requests for 500ms) + await page.waitForLoadState('networkidle', { timeout: 30000 }); + + // Additional wait to ensure all pending requests complete + const maxWaitTime = 10000; // 10 seconds max + const checkInterval = 100; // Check every 100ms + const startTime = Date.now(); + + while (pendingRequests.size > 0 && (Date.now() - startTime) < maxWaitTime) { + await new Promise(resolve => setTimeout(resolve, checkInterval)); + } + + if (pendingRequests.size > 0) { + console.warn(`Warning: ${pendingRequests.size} network requests still pending after waiting`); + } + + // Get full HTML + const html = await page.content(); + + // Store in cache + const now = Date.now(); + const expiresAt = now + (config.CACHE_EXPIRY_HOURS * 60 * 60 * 1000); + await dbOps.setCache(url, html, now, expiresAt); + console.log(`Cached: ${url} (expires in ${config.CACHE_EXPIRY_HOURS}h)`); + + return html; + } catch (error) { + console.error(`Error fetching ${url}:`, error.message); + + // If cache exists and we had an error, optionally return stale cache + // For now, we'll just throw the error + throw error; + } finally { + // Always close the page, even on error + if (page) { + try { + await page.close(); + } catch (closeError) { + console.error('Error closing page:', closeError.message); + } + } + } +} + +module.exports = { + fetchHtml, +}; diff --git a/Dockers/playwright-node-api/version b/Dockers/playwright-node-api/version new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/Dockers/playwright-node-api/version @@ -0,0 +1 @@ +1.0.0