From 96eb344191a2526f267c07262d3ebe83c4d29f64 Mon Sep 17 00:00:00 2001 From: Bram Date: Tue, 1 Jul 2025 09:54:11 +0200 Subject: [PATCH] try another fix --- Dockers/puppeteer-api/README.md | 154 ++++++++++++ Dockers/puppeteer-api/main.py | 396 +++++++++++++++++++++---------- Dockers/puppeteer-api/monitor.py | 140 +++++++++++ 3 files changed, 562 insertions(+), 128 deletions(-) create mode 100644 Dockers/puppeteer-api/README.md create mode 100755 Dockers/puppeteer-api/monitor.py diff --git a/Dockers/puppeteer-api/README.md b/Dockers/puppeteer-api/README.md new file mode 100644 index 0000000..b870742 --- /dev/null +++ b/Dockers/puppeteer-api/README.md @@ -0,0 +1,154 @@ +# Puppeteer API Server + +A FastAPI-based server that uses Puppeteer (via pyppeteer) to scrape websites and extract various types of data. + +## Recent Fixes (v2.0) + +### Issues Fixed + +- **Resource temporarily unavailable (Errno 11)** - Fixed by implementing proper browser cleanup and resource limits +- **100% CPU usage** - Fixed by reducing browser pool size, adding operation limits, and improving cleanup +- **Server hanging** - Fixed by adding timeouts, emergency cleanup, and better error handling + +### Key Improvements + +#### 1. Resource Management + +- **Reduced browser pool size**: From 5 to 3 browsers maximum +- **Shorter browser TTL**: From 1 hour to 30 minutes +- **Concurrent operation limits**: Maximum 5 concurrent operations +- **Queue size limits**: Prevent unbounded growth + +#### 2. Better Cleanup + +- **Automatic browser recycling**: Browsers are recycled every 30 minutes +- **Emergency cleanup**: Force cleanup all browsers when needed +- **Proper page cleanup**: Ensure pages are closed after each operation +- **Signal handlers**: Graceful shutdown on SIGTERM/SIGINT + +#### 3. Monitoring & Debugging + +- **System status endpoint**: `/status` - Monitor CPU, memory, file descriptors +- **Browser pool monitoring**: Track active browsers and their ages +- **Emergency cleanup endpoint**: `/emergency-cleanup` - Force cleanup when needed +- **Monitoring script**: `monitor.py` - Real-time system monitoring + +#### 4. Error Handling + +- **Timeout handling**: 30s timeout for getting browsers from pool +- **Emergency recovery**: Auto-cleanup when timeouts occur +- **Better exception handling**: More detailed error logging + +## Environment Variables + +```bash +# Required +API_KEY=your-api-key-here + +# Optional (with defaults) +MAX_BROWSERS=3 # Maximum browser instances (default: 3) +BROWSER_TTL=1800 # Browser time-to-live in seconds (default: 1800 = 30min) +MAX_CONCURRENT_OPERATIONS=5 # Max concurrent operations (default: 5) +CACHE_EXPIRY_HOURS=36 # Cache expiry in hours (default: 36) +CLEANUP_CRON=0 3 * * * # Cache cleanup schedule (default: daily at 3 AM) +RATE_LIMIT_MINUTE=60 # Requests per minute (default: 60) +``` + +## API Endpoints + +### Core Endpoints + +- `GET /` - Visit URL and get HTML content +- `GET /seo` - Extract SEO information +- `GET /meta` - Extract meta tags and Open Graph data + +### Management Endpoints + +- `HEAD /` - Health check +- `GET /status` - System status and browser pool information +- `POST /emergency-cleanup` - Force cleanup all browsers +- `GET /cache/stats` - Cache statistics +- `GET /cache/clear` - Clear all cache + +## Usage Examples + +### Basic Usage + +```bash +# Visit a URL +curl -H "X-API-Key: your-api-key" "http://localhost:8000/?url=https://example.com" + +# Extract SEO data +curl -H "X-API-Key: your-api-key" "http://localhost:8000/seo?url=https://example.com" + +# Get system status +curl -H "X-API-Key: your-api-key" "http://localhost:8000/status" +``` + +### Monitoring + +```bash +# Monitor system every 30 seconds +python monitor.py your-api-key 30 + +# Monitor with custom URL +python monitor.py your-api-key 30 http://your-server:8000 + +# Trigger emergency cleanup +python monitor.py your-api-key 30 http://localhost:8000 cleanup +``` + +## Docker Usage + +```bash +# Build the image +docker build -t puppeteer-api . + +# Run with environment variables +docker run -d \ + --name puppeteer-api \ + -p 8000:8000 \ + -e API_KEY=your-api-key \ + -e MAX_BROWSERS=3 \ + -e BROWSER_TTL=1800 \ + -v /path/to/cache:/db \ + puppeteer-api +``` + +## Troubleshooting + +### High CPU Usage + +1. Check system status: `GET /status` +2. If browser pool is at capacity, trigger cleanup: `POST /emergency-cleanup` +3. Consider reducing `MAX_BROWSERS` or `MAX_CONCURRENT_OPERATIONS` + +### Resource Errors (Errno 11) + +1. The system now automatically handles this with better cleanup +2. Monitor with `python monitor.py` to track resource usage +3. If persistent, trigger emergency cleanup + +### Server Hanging + +1. Check for stuck operations with monitoring script +2. Trigger emergency cleanup +3. Restart the container if needed + +## Performance Tips + +1. **Use caching**: The API caches results for 36 hours by default +2. **Monitor resources**: Use the monitoring script to track usage +3. **Adjust limits**: Tune `MAX_BROWSERS` and `MAX_CONCURRENT_OPERATIONS` based on your server capacity +4. **Regular cleanup**: The system automatically recycles browsers every 30 minutes + +## Logs + +The server provides detailed logging for: + +- Browser creation and cleanup +- Resource usage warnings +- Error conditions +- Cache operations + +Monitor logs to identify patterns and adjust configuration accordingly. diff --git a/Dockers/puppeteer-api/main.py b/Dockers/puppeteer-api/main.py index 206c5f7..d3deb41 100644 --- a/Dockers/puppeteer-api/main.py +++ b/Dockers/puppeteer-api/main.py @@ -6,6 +6,8 @@ import json import re import sqlite3 import time +import signal +import psutil from datetime import datetime, timedelta from typing import Optional, Dict, List, Any from urllib.parse import unquote @@ -14,7 +16,7 @@ from apscheduler.triggers.cron import CronTrigger from fastapi.middleware.base import BaseHTTPMiddleware # Add imports for browser pool -from asyncio import Queue, Lock +from asyncio import Queue, Lock, Semaphore from contextlib import asynccontextmanager app = FastAPI() @@ -33,12 +35,17 @@ CLEANUP_CRON = os.getenv('CLEANUP_CRON', '0 3 * * *') # Define custom user agent CUSTOM_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36' -# Browser pool configuration -MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '5')) # Maximum number of browser instances -BROWSER_TTL = int(os.getenv('BROWSER_TTL', '3600')) # Time to live for browser instances in seconds -browser_pool = Queue() +# Browser pool configuration - Reduced for better resource management +MAX_BROWSERS = int(os.getenv('MAX_BROWSERS', '3')) # Reduced from 5 to 3 +BROWSER_TTL = int(os.getenv('BROWSER_TTL', '1800')) # Reduced from 3600 to 1800 seconds (30 minutes) +MAX_CONCURRENT_OPERATIONS = int(os.getenv('MAX_CONCURRENT_OPERATIONS', '5')) # New: limit concurrent operations + +# Browser pool management +browser_pool = Queue(maxsize=MAX_BROWSERS) # Add maxsize to prevent unbounded growth browser_lock = Lock() browser_creation_times = {} +active_browsers = set() # Track active browsers +operation_semaphore = Semaphore(MAX_CONCURRENT_OPERATIONS) # Limit concurrent operations # Rate limiting configuration RATE_LIMIT_MINUTE = int(os.getenv('RATE_LIMIT_MINUTE', '60')) # requests per minute @@ -87,105 +94,164 @@ class RateLimitMiddleware(BaseHTTPMiddleware): app.add_middleware(RateLimitMiddleware) async def create_browser(): - """Create a new browser instance""" - browser = await launch( - headless=True, - executablePath='/usr/bin/google-chrome', - args=[ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--disable-gpu', - '--disable-extensions', - '--disable-sync', - '--disable-background-networking', - '--disable-default-apps', - '--disable-translate', - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-client-side-phishing-detection', - '--disable-features=site-per-process', - '--disable-hang-monitor', - '--disable-ipc-flooding-protection', - '--disable-popup-blocking', - '--disable-prompt-on-repost', - '--disable-renderer-backgrounding', - '--memory-pressure-off', - '--no-first-run', - '--safebrowsing-disable-auto-update', - ], - handleSIGINT=False, - handleSIGTERM=False, - handleSIGHUP=False, - ignoreHTTPSErrors=True - ) - browser_creation_times[browser] = time.time() - return browser + """Create a new browser instance with improved resource management""" + try: + browser = await launch( + headless=True, + executablePath='/usr/bin/google-chrome', + args=[ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--disable-gpu', + '--disable-extensions', + '--disable-sync', + '--disable-background-networking', + '--disable-default-apps', + '--disable-translate', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-client-side-phishing-detection', + '--disable-features=site-per-process', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--memory-pressure-off', + '--no-first-run', + '--safebrowsing-disable-auto-update', + '--max_old_space_size=512', # Limit memory usage + '--single-process', # Use single process to reduce resource usage + '--disable-web-security', + '--disable-features=VizDisplayCompositor', + ], + handleSIGINT=False, + handleSIGTERM=False, + handleSIGHUP=False, + ignoreHTTPSErrors=True, + autoClose=True, # Ensure browser closes automatically + ) + + browser_creation_times[browser] = time.time() + active_browsers.add(browser) + print(f"Created new browser instance. Total active browsers: {len(active_browsers)}") + return browser + except Exception as e: + print(f"Error creating browser: {e}") + raise + +async def cleanup_browser(browser): + """Safely cleanup a browser instance""" + try: + if browser in active_browsers: + active_browsers.remove(browser) + + if browser in browser_creation_times: + del browser_creation_times[browser] + + # Close all pages first + pages = await browser.pages() + for page in pages: + try: + await page.close() + except Exception as e: + print(f"Error closing page: {e}") + + # Close browser + await browser.close() + print(f"Browser cleaned up. Total active browsers: {len(active_browsers)}") + except Exception as e: + print(f"Error during browser cleanup: {e}") async def check_browser_health(): """Check browser health and recycle if needed""" while True: try: - # Sleep for 5 minutes between checks - await asyncio.sleep(300) + # Sleep for 2 minutes between checks (reduced from 5 minutes) + await asyncio.sleep(120) async with browser_lock: # Get all browsers from the pool browsers = [] while not browser_pool.empty(): - browsers.append(await browser_pool.get()) + try: + browsers.append(await browser_pool.get_nowait()) + except asyncio.QueueEmpty: + break # Check each browser for browser in browsers: try: # Check if browser is too old if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL: - await browser.close() - del browser_creation_times[browser] + print(f"Recycling old browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)") + await cleanup_browser(browser) browser = await create_browser() else: # Quick health check await browser.pages() # Put back in pool if healthy - await browser_pool.put(browser) - except Exception: + if not browser_pool.full(): + await browser_pool.put(browser) + else: + # Pool is full, cleanup this browser + await cleanup_browser(browser) + except Exception as e: + print(f"Browser health check failed: {e}") # If unhealthy, close and create new - try: - await browser.close() - except: - pass - if browser in browser_creation_times: - del browser_creation_times[browser] - new_browser = await create_browser() - await browser_pool.put(new_browser) + await cleanup_browser(browser) + if not browser_pool.full(): + new_browser = await create_browser() + await browser_pool.put(new_browser) except Exception as e: print(f"Error in browser health check: {str(e)}") +async def force_cleanup_all_browsers(): + """Force cleanup all browsers in emergency situations""" + print("Force cleaning up all browsers...") + + # Clean up pool + while not browser_pool.empty(): + try: + browser = await browser_pool.get_nowait() + await cleanup_browser(browser) + except asyncio.QueueEmpty: + break + + # Clean up active browsers + for browser in list(active_browsers): + await cleanup_browser(browser) + # Initialize browser pool @app.on_event("startup") async def init_browser_pool(): """Initialize the browser pool with some browsers""" - for _ in range(min(3, MAX_BROWSERS)): # Start with 3 browsers or MAX_BROWSERS, whichever is smaller - browser = await create_browser() - await browser_pool.put(browser) - print(f"Browser pool initialized with {browser_pool.qsize()} browsers") + try: + for _ in range(min(2, MAX_BROWSERS)): # Start with 2 browsers instead of 3 + browser = await create_browser() + await browser_pool.put(browser) + print(f"Browser pool initialized with {browser_pool.qsize()} browsers") - # Start browser health check task - asyncio.create_task(check_browser_health()) + # Start browser health check task + asyncio.create_task(check_browser_health()) + except Exception as e: + print(f"Error initializing browser pool: {e}") @app.on_event("shutdown") async def cleanup_browser_pool(): """Clean up all browsers in the pool""" - while not browser_pool.empty(): - try: - browser = await browser_pool.get_nowait() - await browser.close() - if browser in browser_creation_times: - del browser_creation_times[browser] - except: - pass + await force_cleanup_all_browsers() + +# Signal handlers for graceful shutdown +def signal_handler(signum, frame): + print(f"Received signal {signum}, shutting down gracefully...") + asyncio.create_task(force_cleanup_all_browsers()) + +signal.signal(signal.SIGTERM, signal_handler) +signal.signal(signal.SIGINT, signal_handler) # Initialize SQLite database def init_db(): @@ -371,47 +437,49 @@ async def health_check(): return {"status": "ok"} async def safe_browser_operation(url, operation_func): - """Safely perform a browser operation with proper cleanup""" - async with get_browser() as browser: - try: - # Create a new page - page = await browser.newPage() - - # Set reasonable viewport - await page.setViewport({'width': 1280, 'height': 800}) - - # Set user agent - await page.setUserAgent(CUSTOM_USER_AGENT) - - # Set reasonable timeout - page.setDefaultNavigationTimeout(30000) - - # Enable request interception to block unnecessary resources - await page.setRequestInterception(True) - - async def intercept(request): - # Block unnecessary resource types - if request.resourceType in ['image', 'media', 'font', 'stylesheet']: - await request.abort() - else: - await request.continue_() - - page.on('request', lambda req: asyncio.ensure_future(intercept(req))) - - # Perform the operation - result = await operation_func(page) - - return result - except Exception as e: - print(f"Error during browser operation: {str(e)}") - raise - finally: + """Safely perform a browser operation with proper cleanup and resource limits""" + async with operation_semaphore: # Limit concurrent operations + async with get_browser() as browser: + page = None try: - # Ensure page is properly closed - if 'page' in locals(): - await page.close() + # Create a new page + page = await browser.newPage() + + # Set reasonable viewport + await page.setViewport({'width': 1280, 'height': 800}) + + # Set user agent + await page.setUserAgent(CUSTOM_USER_AGENT) + + # Set reasonable timeout + page.setDefaultNavigationTimeout(30000) + + # Enable request interception to block unnecessary resources + await page.setRequestInterception(True) + + async def intercept(request): + # Block unnecessary resource types + if request.resourceType in ['image', 'media', 'font', 'stylesheet']: + await request.abort() + else: + await request.continue_() + + page.on('request', lambda req: asyncio.ensure_future(intercept(req))) + + # Perform the operation + result = await operation_func(page) + + return result except Exception as e: - print(f"Error closing page: {str(e)}") + print(f"Error during browser operation: {str(e)}") + raise + finally: + try: + # Ensure page is properly closed + if page: + await page.close() + except Exception as e: + print(f"Error closing page: {str(e)}") @app.get("/") async def visit_url(url: str, x_api_key: Optional[str] = Header(None)): @@ -682,23 +750,93 @@ async def cache_stats(x_api_key: Optional[str] = Header(None)): except Exception as e: raise HTTPException(status_code=500, detail=str(e)) +@app.get("/status") +async def system_status(x_api_key: Optional[str] = Header(None)): + """Get system status and browser pool information""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + try: + # Get system information + process = psutil.Process() + memory_info = process.memory_info() + + # Get browser pool information + pool_size = browser_pool.qsize() + active_browser_count = len(active_browsers) + + # Calculate browser ages + browser_ages = [] + for browser, creation_time in browser_creation_times.items(): + age = time.time() - creation_time + browser_ages.append(age) + + return { + "system": { + "cpu_percent": process.cpu_percent(), + "memory_mb": memory_info.rss / 1024 / 1024, + "memory_percent": process.memory_percent(), + "open_files": len(process.open_files()), + "connections": len(process.connections()), + "threads": process.num_threads() + }, + "browser_pool": { + "pool_size": pool_size, + "active_browsers": active_browser_count, + "max_browsers": MAX_BROWSERS, + "browser_ttl_seconds": BROWSER_TTL, + "browser_ages_seconds": browser_ages, + "concurrent_operations_limit": MAX_CONCURRENT_OPERATIONS + }, + "timestamp": time.time() + } + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +@app.post("/emergency-cleanup") +async def emergency_cleanup(x_api_key: Optional[str] = Header(None)): + """Force emergency cleanup of all browsers""" + # Validate API key + if not x_api_key or x_api_key != API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + try: + await force_cleanup_all_browsers() + return {"status": "success", "message": "Emergency cleanup completed"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + @asynccontextmanager async def get_browser(): """Get a browser from the pool or create a new one if needed""" browser = None try: - # Try to get a browser from the pool + # Try to get a browser from the pool with timeout try: - browser = await browser_pool.get_nowait() - except asyncio.QueueEmpty: - # If pool is empty, create a new browser if under the limit + browser = await asyncio.wait_for(browser_pool.get(), timeout=30.0) + except (asyncio.QueueEmpty, asyncio.TimeoutError): + # If pool is empty or timeout, create a new browser if under the limit async with browser_lock: - if browser_pool.qsize() + 1 <= MAX_BROWSERS: + current_browser_count = len(active_browsers) + if current_browser_count < MAX_BROWSERS: browser = await create_browser() else: - # If at limit, wait for a browser to become available - browser = await browser_pool.get() + # If at limit, wait for a browser to become available with timeout + try: + browser = await asyncio.wait_for(browser_pool.get(), timeout=60.0) + except asyncio.TimeoutError: + # Emergency cleanup and create new browser + print("Emergency: Timeout waiting for browser, forcing cleanup") + await force_cleanup_all_browsers() + browser = await create_browser() + yield browser + except Exception as e: + print(f"Error in get_browser: {e}") + # Emergency cleanup if we can't get a browser + await force_cleanup_all_browsers() + browser = await create_browser() yield browser finally: # Return browser to pool if it's still viable @@ -708,21 +846,23 @@ async def get_browser(): await browser.pages() # Check if browser is too old if time.time() - browser_creation_times.get(browser, 0) > BROWSER_TTL: - await browser.close() - if browser in browser_creation_times: - del browser_creation_times[browser] + print(f"Recycling old browser in get_browser (age: {time.time() - browser_creation_times.get(browser, 0):.0f}s)") + await cleanup_browser(browser) browser = await create_browser() - await browser_pool.put(browser) - except Exception: + + # Only put back if pool is not full + if not browser_pool.full(): + await browser_pool.put(browser) + else: + # Pool is full, cleanup this browser + await cleanup_browser(browser) + except Exception as e: + print(f"Browser health check failed in get_browser: {e}") # If browser is not usable, close it and create a new one - try: - await browser.close() - except: - pass - if browser in browser_creation_times: - del browser_creation_times[browser] - browser = await create_browser() - await browser_pool.put(browser) + await cleanup_browser(browser) + if not browser_pool.full(): + new_browser = await create_browser() + await browser_pool.put(new_browser) if __name__ == "__main__": import uvicorn diff --git a/Dockers/puppeteer-api/monitor.py b/Dockers/puppeteer-api/monitor.py new file mode 100755 index 0000000..8c5fb6b --- /dev/null +++ b/Dockers/puppeteer-api/monitor.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +""" +Monitoring script for the Puppeteer API server. +This script helps track system resources and browser pool status. +""" + +import asyncio +import aiohttp +import json +import time +import sys +from datetime import datetime + +async def get_status(api_key, base_url="http://localhost:8000"): + """Get system status from the API""" + async with aiohttp.ClientSession() as session: + headers = {"X-API-Key": api_key} + + try: + async with session.get(f"{base_url}/status", headers=headers) as response: + if response.status == 200: + return await response.json() + else: + print(f"Error getting status: {response.status}") + return None + except Exception as e: + print(f"Exception getting status: {e}") + return None + +async def emergency_cleanup(api_key, base_url="http://localhost:8000"): + """Trigger emergency cleanup""" + async with aiohttp.ClientSession() as session: + headers = {"X-API-Key": api_key} + + try: + async with session.post(f"{base_url}/emergency-cleanup", headers=headers) as response: + if response.status == 200: + result = await response.json() + print(f"Emergency cleanup: {result}") + return True + else: + print(f"Error triggering cleanup: {response.status}") + return False + except Exception as e: + print(f"Exception triggering cleanup: {e}") + return False + +def print_status(status_data): + """Print formatted status information""" + if not status_data: + print("No status data available") + return + + print(f"\n{'='*60}") + print(f"Status Report - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'='*60}") + + # System information + system = status_data.get('system', {}) + print(f"\n📊 SYSTEM STATUS:") + print(f" CPU Usage: {system.get('cpu_percent', 'N/A')}%") + print(f" Memory: {system.get('memory_mb', 'N/A'):.1f} MB ({system.get('memory_percent', 'N/A'):.1f}%)") + print(f" Open Files: {system.get('open_files', 'N/A')}") + print(f" Network Connections: {system.get('connections', 'N/A')}") + print(f" Threads: {system.get('threads', 'N/A')}") + + # Browser pool information + pool = status_data.get('browser_pool', {}) + print(f"\n🌐 BROWSER POOL:") + print(f" Pool Size: {pool.get('pool_size', 'N/A')}") + print(f" Active Browsers: {pool.get('active_browsers', 'N/A')}") + print(f" Max Browsers: {pool.get('max_browsers', 'N/A')}") + print(f" Browser TTL: {pool.get('browser_ttl_seconds', 'N/A')}s") + print(f" Concurrent Operations Limit: {pool.get('concurrent_operations_limit', 'N/A')}") + + # Browser ages + ages = pool.get('browser_ages_seconds', []) + if ages: + print(f" Browser Ages: {[f'{age:.0f}s' for age in ages]}") + + # Warnings + warnings = [] + if system.get('cpu_percent', 0) > 80: + warnings.append("⚠️ High CPU usage") + if system.get('memory_percent', 0) > 80: + warnings.append("⚠️ High memory usage") + if system.get('open_files', 0) > 1000: + warnings.append("⚠️ Many open files") + if pool.get('active_browsers', 0) >= pool.get('max_browsers', 0): + warnings.append("⚠️ Browser pool at capacity") + + if warnings: + print(f"\n🚨 WARNINGS:") + for warning in warnings: + print(f" {warning}") + else: + print(f"\n✅ All systems normal") + +async def monitor_loop(api_key, interval=30, base_url="http://localhost:8000"): + """Continuous monitoring loop""" + print(f"Starting monitoring with {interval}s intervals...") + print(f"Press Ctrl+C to stop") + + try: + while True: + status = await get_status(api_key, base_url) + print_status(status) + + # Wait for next check + await asyncio.sleep(interval) + + except KeyboardInterrupt: + print("\nMonitoring stopped by user") + +async def main(): + """Main function""" + if len(sys.argv) < 2: + print("Usage: python monitor.py [interval_seconds] [base_url]") + print("Example: python monitor.py my-api-key 30 http://localhost:8000") + sys.exit(1) + + api_key = sys.argv[1] + interval = int(sys.argv[2]) if len(sys.argv) > 2 else 30 + base_url = sys.argv[3] if len(sys.argv) > 3 else "http://localhost:8000" + + if len(sys.argv) > 4 and sys.argv[4] == "cleanup": + # Emergency cleanup mode + print("Triggering emergency cleanup...") + success = await emergency_cleanup(api_key, base_url) + if success: + print("Emergency cleanup completed successfully") + else: + print("Emergency cleanup failed") + return + + # Start monitoring + await monitor_loop(api_key, interval, base_url) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file