|
@@ -64,8 +64,12 @@ app = Flask(__name__)
|
|
|
config = None
|
|
config = None
|
|
|
|
|
|
|
|
# Systemd watchdog support
|
|
# Systemd watchdog support
|
|
|
|
|
+watchdog_interval = None
|
|
|
|
|
+last_watchdog_time = 0
|
|
|
|
|
+
|
|
|
def setup_systemd_watchdog():
|
|
def setup_systemd_watchdog():
|
|
|
"""Setup systemd watchdog notifications"""
|
|
"""Setup systemd watchdog notifications"""
|
|
|
|
|
+ global watchdog_interval
|
|
|
import os
|
|
import os
|
|
|
import shutil
|
|
import shutil
|
|
|
|
|
|
|
@@ -82,33 +86,31 @@ def setup_systemd_watchdog():
|
|
|
|
|
|
|
|
try:
|
|
try:
|
|
|
# Convert microseconds to seconds and send notifications at half the interval
|
|
# Convert microseconds to seconds and send notifications at half the interval
|
|
|
- interval = int(watchdog_usec) / 2000000 # Half interval in seconds
|
|
|
|
|
- logger.info(f"Systemd watchdog enabled, sending notifications every {interval:.1f}s")
|
|
|
|
|
-
|
|
|
|
|
- def watchdog_ping():
|
|
|
|
|
- while True:
|
|
|
|
|
- try:
|
|
|
|
|
- result = subprocess.run(['systemd-notify', 'WATCHDOG=1'],
|
|
|
|
|
- check=False, stdout=subprocess.DEVNULL,
|
|
|
|
|
- stderr=subprocess.DEVNULL, timeout=5)
|
|
|
|
|
- if result.returncode != 0:
|
|
|
|
|
- logger.debug(f"systemd-notify returned {result.returncode}")
|
|
|
|
|
- time.sleep(interval)
|
|
|
|
|
- except subprocess.TimeoutExpired:
|
|
|
|
|
- logger.warning("systemd-notify timeout")
|
|
|
|
|
- time.sleep(interval)
|
|
|
|
|
- except Exception as e:
|
|
|
|
|
- logger.warning(f"Failed to send watchdog notification: {e}")
|
|
|
|
|
- time.sleep(interval)
|
|
|
|
|
-
|
|
|
|
|
- # Start watchdog thread
|
|
|
|
|
- watchdog_thread = threading.Thread(target=watchdog_ping, daemon=True)
|
|
|
|
|
- watchdog_thread.start()
|
|
|
|
|
|
|
+ watchdog_interval = int(watchdog_usec) / 2000000 # Half interval in seconds
|
|
|
|
|
+ logger.info(f"Systemd watchdog enabled, will send notifications every {watchdog_interval:.1f}s")
|
|
|
return True
|
|
return True
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
logger.warning(f"Failed to setup systemd watchdog: {e}")
|
|
logger.warning(f"Failed to setup systemd watchdog: {e}")
|
|
|
return False
|
|
return False
|
|
|
|
|
|
|
|
|
|
+def send_watchdog_if_needed():
|
|
|
|
|
+ """Send watchdog notification if it's time"""
|
|
|
|
|
+ global last_watchdog_time
|
|
|
|
|
+ if watchdog_interval is None:
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+ current_time = time.time()
|
|
|
|
|
+ if current_time - last_watchdog_time >= watchdog_interval:
|
|
|
|
|
+ try:
|
|
|
|
|
+ result = subprocess.run(['systemd-notify', 'WATCHDOG=1'],
|
|
|
|
|
+ check=False, stdout=subprocess.DEVNULL,
|
|
|
|
|
+ stderr=subprocess.DEVNULL, timeout=2)
|
|
|
|
|
+ last_watchdog_time = current_time
|
|
|
|
|
+ if result.returncode != 0:
|
|
|
|
|
+ logger.debug(f"systemd-notify returned {result.returncode}")
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ logger.warning(f"Failed to send watchdog notification: {e}")
|
|
|
|
|
+
|
|
|
def require_auth(key_config=None, is_health=False):
|
|
def require_auth(key_config=None, is_health=False):
|
|
|
"""Decorator for HTTP Basic Authentication"""
|
|
"""Decorator for HTTP Basic Authentication"""
|
|
|
def decorator(f):
|
|
def decorator(f):
|
|
@@ -272,6 +274,9 @@ def create_key_handler(key_config):
|
|
|
|
|
|
|
|
logger.warning(f"EMERGENCY: Key access attempt detected for key '{key_config.key_id}'")
|
|
logger.warning(f"EMERGENCY: Key access attempt detected for key '{key_config.key_id}'")
|
|
|
|
|
|
|
|
|
|
+ # Send watchdog notification
|
|
|
|
|
+ send_watchdog_if_needed()
|
|
|
|
|
+
|
|
|
try:
|
|
try:
|
|
|
# Send notification first - fail-safe approach
|
|
# Send notification first - fail-safe approach
|
|
|
notification_success, successful_backends = send_ntfy_notification(
|
|
notification_success, successful_backends = send_ntfy_notification(
|
|
@@ -323,6 +328,9 @@ def health_check():
|
|
|
"""Health check endpoint that verifies both health monitoring and all key request functionality"""
|
|
"""Health check endpoint that verifies both health monitoring and all key request functionality"""
|
|
|
logger.info("Health check requested")
|
|
logger.info("Health check requested")
|
|
|
|
|
|
|
|
|
|
+ # Send watchdog notification
|
|
|
|
|
+ send_watchdog_if_needed()
|
|
|
|
|
+
|
|
|
if config is None:
|
|
if config is None:
|
|
|
logger.error("Configuration not loaded during health check")
|
|
logger.error("Configuration not loaded during health check")
|
|
|
return jsonify({'status': 'error', 'message': 'Configuration not loaded'}), 500
|
|
return jsonify({'status': 'error', 'message': 'Configuration not loaded'}), 500
|
|
@@ -555,6 +563,11 @@ def main():
|
|
|
# Add health check route
|
|
# Add health check route
|
|
|
app.add_url_rule(config.health_route, 'health_check', health_check, methods=['GET'])
|
|
app.add_url_rule(config.health_route, 'health_check', health_check, methods=['GET'])
|
|
|
|
|
|
|
|
|
|
+ # Add Flask before_request handler for watchdog
|
|
|
|
|
+ @app.before_request
|
|
|
|
|
+ def before_request():
|
|
|
|
|
+ send_watchdog_if_needed()
|
|
|
|
|
+
|
|
|
# Setup systemd watchdog
|
|
# Setup systemd watchdog
|
|
|
watchdog_enabled = setup_systemd_watchdog()
|
|
watchdog_enabled = setup_systemd_watchdog()
|
|
|
|
|
|
|
@@ -581,6 +594,9 @@ def main():
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
logger.warning(f"Failed to notify systemd ready: {e}")
|
|
logger.warning(f"Failed to notify systemd ready: {e}")
|
|
|
|
|
|
|
|
|
|
+ # Send initial watchdog notification
|
|
|
|
|
+ send_watchdog_if_needed()
|
|
|
|
|
+
|
|
|
# Use production-ready server with better error handling
|
|
# Use production-ready server with better error handling
|
|
|
try:
|
|
try:
|
|
|
from waitress import serve
|
|
from waitress import serve
|