215 lines
8.3 KiB
Python
215 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Healthcheck script for Puppeteer API
|
|
Monitors multiple API endpoints and restarts containers if they fail
|
|
"""
|
|
|
|
import os
|
|
import time
|
|
import logging
|
|
import requests
|
|
import docker
|
|
import json
|
|
from urllib.parse import quote
|
|
from datetime import datetime
|
|
|
|
# Configure logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.StreamHandler(),
|
|
logging.FileHandler('/app/healthcheck.log')
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class PuppeteerHealthcheck:
|
|
def __init__(self):
|
|
self.hosts = self._parse_hosts()
|
|
self.test_url = os.getenv('TEST_URL', 'https://www.google.com')
|
|
self.api_key = os.getenv('API_KEY', 'Q7Sd#hhFkyHy*T')
|
|
self.check_interval = int(os.getenv('CHECK_INTERVAL', '60'))
|
|
self.max_consecutive_failures = int(os.getenv('MAX_CONSECUTIVE_FAILURES', '3'))
|
|
self.timeout = int(os.getenv('TIMEOUT', '20'))
|
|
|
|
# Track consecutive failures for each host
|
|
self.failure_counters = {host: 0 for host in self.hosts}
|
|
|
|
# Initialize Docker client with proper error handling
|
|
self.docker_client = self._initialize_docker_client()
|
|
|
|
logger.info(f"Healthcheck initialized with:")
|
|
logger.info(f" Hosts: {', '.join(self.hosts)}")
|
|
logger.info(f" Test URL: {self.test_url}")
|
|
logger.info(f" Check Interval: {self.check_interval} seconds")
|
|
logger.info(f" Timeout: {self.timeout} seconds")
|
|
|
|
def _parse_hosts(self):
|
|
"""Parse hosts from environment variable or use default"""
|
|
hosts_env = os.getenv('HOSTS', '')
|
|
if hosts_env:
|
|
# Split by comma and strip whitespace
|
|
hosts = [host.strip() for host in hosts_env.split(',') if host.strip()]
|
|
if hosts:
|
|
return hosts
|
|
|
|
# Fallback to legacy BASE_URL for backward compatibility
|
|
base_url = os.getenv('BASE_URL', 'https://puppeteer.workwithkora.com')
|
|
if base_url.startswith('http://'):
|
|
# Extract host from http://host:port format
|
|
host_part = base_url.replace('http://', '').split('/')[0]
|
|
if ':' in host_part:
|
|
host = host_part.split(':')[0]
|
|
return [host]
|
|
|
|
# Default fallback
|
|
return ['puppeteer-api']
|
|
|
|
def _initialize_docker_client(self):
|
|
"""Initialize Docker client with proper error handling"""
|
|
try:
|
|
# Try to initialize Docker client
|
|
client = docker.from_env()
|
|
|
|
# Test the connection by getting Docker info
|
|
client.info()
|
|
logger.info("Docker client initialized successfully")
|
|
return client
|
|
|
|
except docker.errors.DockerException as e:
|
|
logger.error(f"Docker client initialization failed: {e}")
|
|
logger.error("This usually means the Docker socket is not accessible or permissions are incorrect.")
|
|
logger.error("Make sure to:")
|
|
logger.error(" 1. Mount the Docker socket: -v /var/run/docker.sock:/var/run/docker.sock")
|
|
logger.error(" 2. Run with proper permissions or add the container to the docker group")
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error initializing Docker client: {e}")
|
|
raise
|
|
|
|
def perform_health_check(self, host):
|
|
"""Perform the health check for a specific host"""
|
|
try:
|
|
# Construct the URL with the test URL as a parameter
|
|
base_url = f"http://{host}:8000"
|
|
encoded_test_url = quote(self.test_url, safe='')
|
|
url = f"{base_url}/?url={encoded_test_url}&skipCache=true"
|
|
|
|
headers = {
|
|
'x-api-key': self.api_key,
|
|
'User-Agent': 'Puppeteer-Healthcheck/1.0'
|
|
}
|
|
|
|
logger.info(f"Performing health check for {host}: {url}")
|
|
|
|
response = requests.get(
|
|
url,
|
|
headers=headers,
|
|
timeout=self.timeout,
|
|
allow_redirects=True
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
# Check if the response body indicates an error
|
|
try:
|
|
response_data = response.json()
|
|
if response_data.get('status') == 'error':
|
|
logger.warning(f"Health check failed for {host} - API returned status: error")
|
|
logger.warning(f"Response: {response.text[:200]}...")
|
|
return False
|
|
else:
|
|
logger.info(f"Health check passed for {host} - API is responding correctly")
|
|
return True
|
|
except json.JSONDecodeError:
|
|
# If response is not JSON, treat as success (backward compatibility)
|
|
logger.info(f"Health check passed for {host} - API is responding correctly (non-JSON response)")
|
|
return True
|
|
|
|
else:
|
|
logger.warning(f"Health check failed for {host} - Status code: {response.status_code}")
|
|
logger.warning(f"Response: {response.text[:200]}...")
|
|
return False
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
logger.error(f"Health check failed for {host} - Request error: {e}")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"Health check failed for {host} - Unexpected error: {e}")
|
|
return False
|
|
|
|
def restart_container(self, host):
|
|
"""Restart the container for a specific host"""
|
|
try:
|
|
logger.info(f"Attempting to restart container: {host}")
|
|
|
|
# Get the container (container name matches host name)
|
|
container = self.docker_client.containers.get(host)
|
|
|
|
# Restart the container
|
|
container.restart(timeout=30)
|
|
|
|
logger.info(f"Successfully restarted container: {host}")
|
|
return True
|
|
|
|
except docker.errors.NotFound:
|
|
logger.error(f"Container not found: {host}")
|
|
return False
|
|
except docker.errors.APIError as e:
|
|
logger.error(f"Docker API error while restarting container {host}: {e}")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error while restarting container {host}: {e}")
|
|
return False
|
|
|
|
def check_host(self, host):
|
|
"""Check a single host and handle failures"""
|
|
if self.perform_health_check(host):
|
|
if self.failure_counters[host] > 0:
|
|
logger.info(f"Health check passed for {host} - resetting failure counter")
|
|
self.failure_counters[host] = 0
|
|
else:
|
|
self.failure_counters[host] += 1
|
|
logger.warning(f"Health check failed for {host} - consecutive failures: {self.failure_counters[host]}")
|
|
|
|
# Restart container if we've had too many consecutive failures
|
|
if self.failure_counters[host] >= self.max_consecutive_failures:
|
|
logger.error(f"Health check failed {self.failure_counters[host]} times consecutively for {host} - restarting container")
|
|
if self.restart_container(host):
|
|
self.failure_counters[host] = 0
|
|
logger.info(f"Container {host} restarted successfully - resetting failure counter")
|
|
else:
|
|
logger.error(f"Failed to restart container {host}")
|
|
|
|
def run(self):
|
|
"""Main loop for the healthcheck"""
|
|
logger.info("Starting Puppeteer API healthcheck service")
|
|
|
|
while True:
|
|
try:
|
|
# Check each host
|
|
for host in self.hosts:
|
|
self.check_host(host)
|
|
|
|
# Wait before next check
|
|
logger.info(f"Waiting {self.check_interval} seconds before next health check...")
|
|
time.sleep(self.check_interval)
|
|
|
|
except KeyboardInterrupt:
|
|
logger.info("Healthcheck service stopped by user")
|
|
break
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error in main loop: {e}")
|
|
time.sleep(self.check_interval)
|
|
|
|
def main():
|
|
"""Main entry point"""
|
|
try:
|
|
healthcheck = PuppeteerHealthcheck()
|
|
healthcheck.run()
|
|
except Exception as e:
|
|
logger.error(f"Failed to start healthcheck service: {e}")
|
|
exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main() |