main.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. #!/usr/bin/env python3
  2. import os
  3. import sys
  4. import logging
  5. import time
  6. import tempfile
  7. import threading
  8. from flask import Flask, jsonify, request
  9. from werkzeug.security import check_password_hash
  10. from werkzeug.serving import WSGIRequestHandler
  11. from functools import wraps
  12. from config import Config
  13. from typing import List, Tuple
  14. import base64
  15. import signal
  16. # Configure logging with custom handler
  17. class NtfyLogHandler(logging.Handler):
  18. """Custom logging handler that sends logs to ntfy health backends"""
  19. def __init__(self, config_obj):
  20. super().__init__()
  21. self.config = config_obj
  22. def emit(self, record):
  23. """Send log record to health backends"""
  24. if hasattr(self.config, 'ntfy_backends_health') and self.config.send_all_logs:
  25. try:
  26. log_message = self.format(record)
  27. # Get configured log level or default to WARNING
  28. min_level = getattr(logging, self.config.log_level.upper(), logging.WARNING)
  29. if record.levelno >= min_level:
  30. # Format message with appropriate emoji based on log level
  31. emoji = "🚨" if record.levelno >= logging.ERROR else "⚠️" if record.levelno >= logging.WARNING else "ℹ️"
  32. title = f"Emergency Access {record.levelname}"
  33. message = f"{emoji} {record.name}: {record.getMessage()}"
  34. send_ntfy_notification(
  35. self.config.ntfy_backends_health,
  36. message,
  37. title
  38. )
  39. except Exception:
  40. # Don't fail the application if logging notification fails
  41. pass
  42. # Configure logging
  43. logging.basicConfig(
  44. level=logging.INFO,
  45. format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
  46. handlers=[
  47. logging.FileHandler('/var/log/emergency-access.log'),
  48. logging.StreamHandler()
  49. ]
  50. )
  51. logger = logging.getLogger(__name__)
  52. app = Flask(__name__)
  53. # Global config instance - initialized during main execution
  54. config = None
  55. # Systemd watchdog support
  56. def setup_systemd_watchdog():
  57. """Setup systemd watchdog notifications"""
  58. try:
  59. import systemd.daemon
  60. watchdog_usec = systemd.daemon.watchdog_enabled()
  61. if watchdog_usec:
  62. # Convert microseconds to seconds and send notifications at half the interval
  63. interval = (watchdog_usec / 1000000) / 2
  64. logger.info(f"Systemd watchdog enabled, sending notifications every {interval:.1f}s")
  65. def watchdog_ping():
  66. while True:
  67. try:
  68. systemd.daemon.notify("WATCHDOG=1")
  69. time.sleep(interval)
  70. except Exception as e:
  71. logger.warning(f"Failed to send watchdog notification: {e}")
  72. time.sleep(interval)
  73. # Start watchdog thread
  74. watchdog_thread = threading.Thread(target=watchdog_ping, daemon=True)
  75. watchdog_thread.start()
  76. return True
  77. except ImportError:
  78. logger.warning("systemd python package not available, watchdog disabled")
  79. except Exception as e:
  80. logger.warning(f"Failed to setup systemd watchdog: {e}")
  81. return False
  82. def require_auth(key_config=None, is_health=False):
  83. """Decorator for HTTP Basic Authentication"""
  84. def decorator(f):
  85. @wraps(f)
  86. def decorated_function(*args, **kwargs):
  87. if config is None:
  88. logger.error("Configuration not loaded")
  89. return jsonify({'error': 'Server configuration error'}), 500
  90. auth = request.authorization
  91. if not auth:
  92. return jsonify({'error': 'Authentication required'}), 401, {
  93. 'WWW-Authenticate': 'Basic realm="Emergency Access"'
  94. }
  95. # Determine expected credentials
  96. if is_health:
  97. expected_username = config.health_username
  98. expected_password_hash = config.health_password_hash
  99. auth_type = "health check"
  100. else:
  101. if key_config is None:
  102. logger.error("Key configuration not provided for authentication")
  103. return jsonify({'error': 'Server configuration error'}), 500
  104. expected_username = key_config.username
  105. expected_password_hash = key_config.password_hash
  106. auth_type = f"key '{key_config.key_id}'"
  107. # Verify credentials
  108. if (auth.username != expected_username or
  109. not Config.verify_password(auth.password, expected_password_hash)):
  110. logger.warning(f"Authentication failed for {auth_type}: invalid credentials from {request.remote_addr}")
  111. return jsonify({'error': 'Invalid credentials'}), 401, {
  112. 'WWW-Authenticate': 'Basic realm="Emergency Access"'
  113. }
  114. logger.info(f"Authentication successful for {auth_type}")
  115. return f(*args, **kwargs)
  116. return decorated_function
  117. return decorator
  118. def send_ntfy_notification(backends: List[str], message: str, title: str = None) -> Tuple[bool, List[str]]:
  119. """
  120. Send notification using dschep/ntfy with dedicated config file
  121. Returns: (success, successful_backends)
  122. """
  123. successful_backends = []
  124. max_retries = 3
  125. retry_delay = 1
  126. for backend in backends:
  127. success = False
  128. last_error = None
  129. for attempt in range(max_retries):
  130. try:
  131. # Import ntfy here to avoid import issues during startup
  132. try:
  133. import ntfy
  134. from ntfy.config import load_config
  135. except ImportError:
  136. raise Exception("ntfy package not available. Please install with: pip install ntfy")
  137. if config is None:
  138. raise Exception("Configuration not loaded")
  139. # Load the ntfy config file
  140. ntfy_config = load_config(config.ntfy_config_path)
  141. ntfy_config["backends"] = [backend]
  142. # Add timeout to prevent hanging using threading
  143. result = [None]
  144. exception = [None]
  145. def notify_with_timeout():
  146. try:
  147. # Send notification using the backend name from our config file
  148. if title:
  149. result[0] = ntfy.notify(message, title=title, config=ntfy_config)
  150. else:
  151. result[0] = ntfy.notify(message, config=ntfy_config, title="Note")
  152. except Exception as e:
  153. exception[0] = e
  154. thread = threading.Thread(target=notify_with_timeout)
  155. thread.daemon = True
  156. thread.start()
  157. thread.join(timeout=15) # 15 second timeout
  158. if thread.is_alive():
  159. raise Exception("Notification timeout")
  160. if exception[0]:
  161. raise exception[0]
  162. ret = result[0]
  163. if ret == 0:
  164. successful_backends.append(backend)
  165. logger.info(f"Notification sent successfully via {backend}")
  166. success = True
  167. break
  168. else:
  169. raise Exception(f"ntfy returned error code {ret}")
  170. except ImportError as e:
  171. logger.error(f"ntfy package not available for backend {backend}: {e}")
  172. last_error = e
  173. break # Don't retry import errors
  174. except Exception as e:
  175. last_error = e
  176. if attempt < max_retries - 1:
  177. logger.warning(f"Notification attempt {attempt + 1} failed for {backend}: {str(e)}, retrying...")
  178. time.sleep(retry_delay * (attempt + 1)) # Exponential backoff
  179. else:
  180. logger.error(f"All notification attempts failed for {backend}: {str(e)}")
  181. if not success and last_error:
  182. # Don't raise exception - just log failure and continue with other backends
  183. logger.error(f"Failed to send notification to {backend} after {max_retries} attempts: {str(last_error)}")
  184. success = len(successful_backends) > 0
  185. return success, successful_backends
  186. def read_file_safely(file_path: str) -> Tuple[bool, str]:
  187. """
  188. Safely read file content
  189. Returns: (success, content)
  190. """
  191. try:
  192. if not os.path.exists(file_path):
  193. logger.error(f"File not found: {file_path}")
  194. return False, f"File not found: {file_path}"
  195. with open(file_path, 'r') as f:
  196. content = f.read().strip()
  197. if not content:
  198. logger.error(f"File is empty: {file_path}")
  199. return False, f"File is empty: {file_path}"
  200. return True, content
  201. except PermissionError:
  202. logger.error(f"Permission denied reading file: {file_path}")
  203. return False, f"Permission denied: {file_path}"
  204. except Exception as e:
  205. logger.error(f"Failed to read file {file_path}: {str(e)}")
  206. return False, f"Failed to read file: {str(e)}"
  207. def create_key_handler(key_config):
  208. """Create a key access handler for a specific key configuration"""
  209. @require_auth(key_config=key_config)
  210. def get_key_part():
  211. """Emergency key access endpoint"""
  212. if key_config is None:
  213. logger.error("Key configuration is None in get_key_part")
  214. return jsonify({'error': 'Server configuration error'}), 500
  215. logger.warning(f"EMERGENCY: Key access attempt detected for key '{key_config.key_id}'")
  216. try:
  217. # Send notification first - fail-safe approach
  218. notification_success, successful_backends = send_ntfy_notification(
  219. key_config.backends,
  220. key_config.message,
  221. "EMERGENCY ACCESS ALERT"
  222. )
  223. if not notification_success:
  224. logger.error(f"CRITICAL: Failed to send notifications to any backend for key '{key_config.key_id}'")
  225. return jsonify({
  226. 'error': 'Notification system failure',
  227. 'message': 'Access denied for security reasons'
  228. }), 500
  229. logger.info(f"Notifications sent successfully to: {successful_backends} for key '{key_config.key_id}'")
  230. # Read key file
  231. file_success, content = read_file_safely(key_config.file_path)
  232. if not file_success:
  233. logger.error(f"CRITICAL: Failed to read key file for '{key_config.key_id}': {content}")
  234. return jsonify({
  235. 'error': 'File access failure',
  236. 'message': 'Unable to retrieve key part'
  237. }), 500
  238. logger.warning(f"EMERGENCY: Key part successfully retrieved and sent for key '{key_config.key_id}'")
  239. return jsonify({
  240. 'success': True,
  241. 'key_id': key_config.key_id,
  242. 'key_part': content,
  243. 'timestamp': time.time(),
  244. 'notified_backends': successful_backends
  245. })
  246. except Exception as e:
  247. logger.error(f"CRITICAL: Unexpected error in key access for '{key_config.key_id}': {str(e)}")
  248. return jsonify({
  249. 'error': 'System error',
  250. 'message': 'Internal server error'
  251. }), 500
  252. return get_key_part
  253. @require_auth(is_health=True)
  254. def health_check():
  255. """Health check endpoint that verifies both health monitoring and all key request functionality"""
  256. logger.info("Health check requested")
  257. if config is None:
  258. logger.error("Configuration not loaded during health check")
  259. return jsonify({'status': 'error', 'message': 'Configuration not loaded'}), 500
  260. try:
  261. # Test health notification system
  262. health_notification_success, health_backends = send_ntfy_notification(
  263. config.ntfy_backends_health,
  264. config.ntfy_health_message,
  265. "Health Check"
  266. )
  267. # Test dummy file access
  268. dummy_file_success, dummy_content = read_file_safely(config.dummy_file_path)
  269. # Test all key files access (without exposing content)
  270. key_files_status = {}
  271. all_key_files_ok = True
  272. for key_id, key_config in config.keys.items():
  273. key_file_success, key_content = read_file_safely(key_config.file_path)
  274. key_files_status[key_id] = key_file_success
  275. if not key_file_success:
  276. all_key_files_ok = False
  277. # Test all key notification backends
  278. key_backends_status = {}
  279. all_key_backends_ok = True
  280. for key_id, key_config in config.keys.items():
  281. try:
  282. # Test notification without actually sending
  283. backend_test_success = len(key_config.backends) > 0
  284. key_backends_status[key_id] = {
  285. 'backends': key_config.backends,
  286. 'success': backend_test_success
  287. }
  288. if not backend_test_success:
  289. all_key_backends_ok = False
  290. except Exception as e:
  291. key_backends_status[key_id] = {
  292. 'backends': key_config.backends,
  293. 'success': False,
  294. 'error': str(e)
  295. }
  296. all_key_backends_ok = False
  297. # Determine overall health status
  298. all_systems_ok = (health_notification_success and
  299. dummy_file_success and
  300. all_key_files_ok and
  301. all_key_backends_ok)
  302. if not all_systems_ok:
  303. error_details = []
  304. if not health_notification_success:
  305. error_details.append("health notifications failed")
  306. if not dummy_file_success:
  307. error_details.append(f"dummy file access failed: {dummy_content}")
  308. # Add key-specific errors
  309. for key_id, status in key_files_status.items():
  310. if not status:
  311. error_details.append(f"key file access failed for '{key_id}'")
  312. for key_id, status in key_backends_status.items():
  313. if not status['success']:
  314. error_msg = f"key backends failed for '{key_id}'"
  315. if 'error' in status:
  316. error_msg += f": {status['error']}"
  317. error_details.append(error_msg)
  318. logger.error(f"Health check failed: {', '.join(error_details)}")
  319. return jsonify({
  320. 'status': 'error',
  321. 'message': 'System components failed',
  322. 'details': error_details,
  323. 'health_notifications': health_notification_success,
  324. 'dummy_file_access': dummy_file_success,
  325. 'key_files_status': key_files_status,
  326. 'key_backends_status': key_backends_status
  327. }), 500
  328. logger.info("Health check completed successfully - all systems operational")
  329. return jsonify({
  330. 'status': 'ok',
  331. 'timestamp': time.time(),
  332. 'health_backends_notified': health_backends,
  333. 'dummy_content_length': len(dummy_content),
  334. 'keys_accessible': len(key_files_status),
  335. 'key_files_status': key_files_status,
  336. 'key_backends_status': key_backends_status,
  337. 'emergency_system_ready': True
  338. })
  339. except Exception as e:
  340. logger.error(f"Health check error: {str(e)}")
  341. return jsonify({
  342. 'status': 'error',
  343. 'message': 'System error',
  344. 'error': str(e)
  345. }), 500
  346. @app.errorhandler(404)
  347. def not_found(error):
  348. """Handle 404 errors silently for security"""
  349. logger.warning(f"404 attempt: {error}")
  350. return jsonify({'error': 'Not found'}), 404
  351. @app.errorhandler(500)
  352. def internal_error(error):
  353. """Handle internal server errors"""
  354. logger.error(f"Internal server error: {error}")
  355. return jsonify({'error': 'Internal server error'}), 500
  356. def validate_setup():
  357. """Validate system setup before starting"""
  358. logger.info("Validating system setup...")
  359. if config is None:
  360. logger.error("Configuration not loaded")
  361. return False
  362. # Check dummy file exists
  363. if not os.path.exists(config.dummy_file_path):
  364. logger.error(f"Dummy file not found: {config.dummy_file_path}")
  365. return False
  366. # Test dummy file permissions
  367. try:
  368. with open(config.dummy_file_path, 'r') as f:
  369. f.read(1)
  370. except Exception as e:
  371. logger.error(f"Dummy file permission test failed: {str(e)}")
  372. return False
  373. # Validate all key configurations
  374. for key_id, key_config in config.keys.items():
  375. logger.info(f"Validating key '{key_id}'...")
  376. # Check key file exists
  377. if not os.path.exists(key_config.file_path):
  378. logger.error(f"Key file not found for '{key_id}': {key_config.file_path}")
  379. return False
  380. # Test key file permissions
  381. try:
  382. with open(key_config.file_path, 'r') as f:
  383. f.read(1)
  384. except Exception as e:
  385. logger.error(f"Key file permission test failed for '{key_id}': {str(e)}")
  386. return False
  387. # Validate backends are configured
  388. if not key_config.backends:
  389. logger.error(f"No notification backends configured for key '{key_id}'")
  390. return False
  391. # Test notification system
  392. logger.info("Testing notification system...")
  393. try:
  394. # Test health notification system
  395. health_success, _ = send_ntfy_notification(
  396. config.ntfy_backends_health[:1], # Test only first backend
  397. "System startup test - health notifications",
  398. "Emergency Access Startup Test"
  399. )
  400. if not health_success:
  401. logger.error("Health notification system test failed")
  402. return False
  403. # Test each key's notification backends
  404. for key_id, key_config in config.keys.items():
  405. key_success, _ = send_ntfy_notification(
  406. key_config.backends[:1], # Test only first backend
  407. f"System startup test - key '{key_id}' notifications",
  408. "Emergency Access Startup Test"
  409. )
  410. if not key_success:
  411. logger.error(f"Key notification system test failed for '{key_id}'")
  412. return False
  413. except Exception as e:
  414. logger.warning(f"Notification test failed, but continuing: {str(e)}")
  415. logger.info("System validation completed successfully")
  416. return True
  417. class ProductionRequestHandler(WSGIRequestHandler):
  418. """Custom request handler with improved error handling"""
  419. def log_error(self, format, *args):
  420. """Override to use our logger"""
  421. logger.error(f"HTTP Error: {format % args}")
  422. def log_message(self, format, *args):
  423. """Override to use our logger"""
  424. logger.info(f"HTTP: {format % args}")
  425. def main():
  426. global config
  427. try:
  428. # Load configuration
  429. config = Config()
  430. logger.info("Configuration loaded successfully")
  431. # Add ntfy log handler after config is loaded
  432. if config.send_all_logs:
  433. ntfy_handler = NtfyLogHandler(config)
  434. min_level = getattr(logging, config.log_level.upper(), logging.WARNING)
  435. ntfy_handler.setLevel(min_level)
  436. # Add to root logger to catch all application logs
  437. logging.getLogger().addHandler(ntfy_handler)
  438. # Validate system setup
  439. if not validate_setup():
  440. logger.error("System validation failed, exiting")
  441. sys.exit(1)
  442. # Add Flask routes dynamically for each key
  443. for key_id, key_config in config.keys.items():
  444. handler = create_key_handler(key_config)
  445. endpoint_name = f'get_key_part_{key_id}'
  446. app.add_url_rule(key_config.route, endpoint_name, handler, methods=['GET'])
  447. logger.info(f"Registered authenticated key route for '{key_id}': {key_config.route}")
  448. # Add health check route
  449. app.add_url_rule(config.health_route, 'health_check', health_check, methods=['GET'])
  450. # Setup systemd watchdog
  451. watchdog_enabled = setup_systemd_watchdog()
  452. logger.info(f"Starting emergency access server on {config.server_host}:{config.server_port}")
  453. logger.info(f"Health route: {config.health_route}")
  454. logger.info(f"Configured {len(config.keys)} key(s)")
  455. logger.info(f"Systemd watchdog: {'enabled' if watchdog_enabled else 'disabled'}")
  456. # Notify systemd that we're ready
  457. try:
  458. import systemd.daemon
  459. systemd.daemon.notify("READY=1")
  460. logger.info("Sent READY=1 to systemd")
  461. except ImportError:
  462. pass
  463. except Exception as e:
  464. logger.warning(f"Failed to notify systemd ready: {e}")
  465. # Use production-ready server with better error handling
  466. try:
  467. from waitress import serve
  468. logger.info("Using Waitress WSGI server for production")
  469. serve(
  470. app,
  471. host=config.server_host,
  472. port=config.server_port,
  473. threads=6,
  474. connection_limit=100,
  475. cleanup_interval=30,
  476. channel_timeout=120
  477. )
  478. except ImportError:
  479. logger.warning("Waitress not available, falling back to Flask dev server")
  480. app.run(
  481. host=config.server_host,
  482. port=config.server_port,
  483. debug=False,
  484. threaded=True,
  485. request_handler=ProductionRequestHandler
  486. )
  487. except KeyboardInterrupt:
  488. logger.info("Received keyboard interrupt, shutting down gracefully")
  489. except Exception as e:
  490. logger.error(f"Failed to start server: {str(e)}")
  491. if config and hasattr(config, 'ntfy_backends_health'):
  492. try:
  493. send_ntfy_notification(
  494. config.ntfy_backends_health,
  495. f"🚨 Emergency Access server failed to start: {str(e)}",
  496. "Emergency Access CRITICAL"
  497. )
  498. except Exception:
  499. pass
  500. sys.exit(1)
  501. finally:
  502. logger.info("Emergency Access server shutdown complete")
  503. if __name__ == '__main__':
  504. main()