""" health_monitor.py - System health monitoring with alerting. Runs periodic health checks on all critical subsystems: - Disk space, RAM usage - FFmpeg, TTS engines, fonts - API providers (OpenRouter, Groq, Scaleway, SiliconFlow) - YouTube OAuth, state persistence Results are cached for 60s to avoid hammering APIs. Health history is persisted via StateManager. """ import json import os import shutil import subprocess import threading import time from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional class HealthMonitor: """System health monitor for AutoDub HerStory.""" # Cache duration in seconds CACHE_DURATION = 60 def __init__(self, state=None, brain=None): self.state = state self.brain = brain self._lock = threading.Lock() self._cache: Dict[str, Any] = {} self._cache_time: float = 0.0 # ================================================================ # Public API # ================================================================ def check_all(self, force: bool = False) -> Dict[str, Any]: """Run all health checks and return a comprehensive status report.""" # Use cache if fresh if not force and self._is_cache_valid(): return self._cache checks = [ self.check_disk(), self.check_memory(), self.check_ffmpeg(), self.check_apis(), self.check_youtube_auth(), self.check_tts_engines(), self.check_fonts(), self.check_state(), ] overall_status = "ok" for check in checks: if check["status"] == "error": overall_status = "error" break elif check["status"] == "warn" and overall_status != "error": overall_status = "warn" report = { "status": overall_status, "checked_at": datetime.now().isoformat(), "checks": checks, "summary": { "total": len(checks), "ok": sum(1 for c in checks if c["status"] == "ok"), "warn": sum(1 for c in checks if c["status"] == "warn"), "error": sum(1 for c in checks if c["status"] == "error"), }, } with self._lock: self._cache = report self._cache_time = time.time() # Persist health history self._save_health_history(report) return report def get_status(self) -> Dict[str, Any]: """Quick status summary (uses cache if available).""" if self._is_cache_valid(): return self._cache return self.check_all() def is_healthy(self) -> bool: """True if all critical checks pass (no errors).""" report = self.get_status() return report.get("status") != "error" # ================================================================ # Individual Health Checks # ================================================================ def check_disk(self) -> Dict[str, Any]: """Check disk space usage. Warn if >80%, error if >95%.""" try: usage = shutil.disk_usage("/") used_pct = (usage.used / usage.total) * 100 free_gb = usage.free / (1024 ** 3) if used_pct > 95: status = "error" elif used_pct > 80: status = "warn" else: status = "ok" return { "name": "disk", "status": status, "message": f"Disk {used_pct:.1f}% used, {free_gb:.1f} GB free", "details": { "total_gb": round(usage.total / (1024 ** 3), 1), "used_gb": round(usage.used / (1024 ** 3), 1), "free_gb": round(free_gb, 1), "used_pct": round(used_pct, 1), }, } except Exception as e: return {"name": "disk", "status": "error", "message": f"Check failed: {e}", "details": {}} def check_memory(self) -> Dict[str, Any]: """Check RAM usage. Warn if >80%.""" try: # Read from /proc/meminfo (Linux) meminfo = {} with open("/proc/meminfo", "r") as f: for line in f: parts = line.split() if len(parts) >= 2: key = parts[0].rstrip(":") value = int(parts[1]) # in kB meminfo[key] = value total_kb = meminfo.get("MemTotal", 0) available_kb = meminfo.get("MemAvailable", 0) used_kb = total_kb - available_kb if total_kb == 0: return {"name": "memory", "status": "ok", "message": "Cannot determine RAM", "details": {}} used_pct = (used_kb / total_kb) * 100 if used_pct > 90: status = "error" elif used_pct > 80: status = "warn" else: status = "ok" return { "name": "memory", "status": status, "message": f"RAM {used_pct:.1f}% used ({used_kb / 1024 / 1024:.1f} GB / {total_kb / 1024 / 1024:.1f} GB)", "details": { "total_gb": round(total_kb / 1024 / 1024, 1), "used_gb": round(used_kb / 1024 / 1024, 1), "available_gb": round(available_kb / 1024 / 1024, 1), "used_pct": round(used_pct, 1), }, } except Exception as e: # /proc/meminfo not available (non-Linux) try: import psutil mem = psutil.virtual_memory() used_pct = mem.percent if used_pct > 90: status = "error" elif used_pct > 80: status = "warn" else: status = "ok" return { "name": "memory", "status": status, "message": f"RAM {used_pct:.1f}% used", "details": {"used_pct": round(used_pct, 1)}, } except ImportError: return {"name": "memory", "status": "ok", "message": "Cannot check RAM (no psutil, not Linux)", "details": {}} def check_ffmpeg(self) -> Dict[str, Any]: """Verify FFmpeg is installed and working.""" try: proc = subprocess.run( ["ffmpeg", "-version"], capture_output=True, text=True, timeout=10 ) if proc.returncode == 0: version_line = proc.stdout.split("\n")[0] # Also check ffprobe probe_proc = subprocess.run( ["ffprobe", "-version"], capture_output=True, text=True, timeout=10 ) probe_ok = probe_proc.returncode == 0 return { "name": "ffmpeg", "status": "ok" if probe_ok else "warn", "message": f"FFmpeg OK ({version_line}), ffprobe {'OK' if probe_ok else 'MISSING'}", "details": {"version": version_line, "ffprobe": probe_ok}, } else: return {"name": "ffmpeg", "status": "error", "message": "FFmpeg returned non-zero", "details": {}} except FileNotFoundError: return {"name": "ffmpeg", "status": "error", "message": "FFmpeg not found in PATH", "details": {}} except subprocess.TimeoutExpired: return {"name": "ffmpeg", "status": "error", "message": "FFmpeg check timed out", "details": {}} except Exception as e: return {"name": "ffmpeg", "status": "error", "message": f"Check failed: {e}", "details": {}} def check_apis(self) -> Dict[str, Any]: """Test all API providers with lightweight calls.""" import httpx results = {} # OpenRouter or_keys = [k.strip() for k in os.getenv("OPENROUTER_API_KEYS", "").split(",") if k.strip()] if or_keys: try: with httpx.Client(timeout=10) as client: resp = client.get( "https://openrouter.ai/api/v1/models", headers={"Authorization": f"Bearer {or_keys[0]}"}, ) results["openrouter"] = { "status": "ok" if resp.status_code == 200 else "error", "keys": len(or_keys), "code": resp.status_code, } except Exception as e: results["openrouter"] = {"status": "error", "keys": len(or_keys), "error": str(e)[:100]} else: results["openrouter"] = {"status": "warn", "keys": 0, "message": "No keys configured"} # Groq groq_key = os.getenv("GROQ_API_KEY", "") if groq_key: try: with httpx.Client(timeout=10) as client: resp = client.get( "https://api.groq.com/openai/v1/models", headers={"Authorization": f"Bearer {groq_key}"}, ) results["groq"] = { "status": "ok" if resp.status_code == 200 else "error", "code": resp.status_code, } except Exception as e: results["groq"] = {"status": "error", "error": str(e)[:100]} else: results["groq"] = {"status": "warn", "message": "No key configured"} # Scaleway scaleway_key = os.getenv("SCALEWAY_API_KEY", "") if scaleway_key: try: with httpx.Client(timeout=10) as client: resp = client.get( "https://api.scaleway.ai/llm/v1/models", headers={"Authorization": f"Bearer {scaleway_key}"}, ) results["scaleway"] = { "status": "ok" if resp.status_code == 200 else "error", "code": resp.status_code, } except Exception as e: results["scaleway"] = {"status": "error", "error": str(e)[:100]} else: results["scaleway"] = {"status": "ok", "message": "No key (optional fallback)"} # SiliconFlow sf_key = os.getenv("SILICONFLOW_API_KEY", "") if sf_key: try: with httpx.Client(timeout=10) as client: resp = client.get( "https://api.siliconflow.cn/v1/models", headers={"Authorization": f"Bearer {sf_key}"}, ) results["siliconflow"] = { "status": "ok" if resp.status_code == 200 else "warn", "code": resp.status_code, "message": "Legacy, may return 401", } except Exception as e: results["siliconflow"] = {"status": "warn", "error": str(e)[:100]} else: results["siliconflow"] = {"status": "ok", "message": "No key (optional)"} # Determine overall API status critical_ok = results.get("openrouter", {}).get("status") in ("ok",) groq_ok = results.get("groq", {}).get("status") in ("ok",) if not critical_ok and not groq_ok: overall = "error" elif not critical_ok or not groq_ok: overall = "warn" else: overall = "ok" return { "name": "apis", "status": overall, "message": f"OpenRouter: {results.get('openrouter', {}).get('status')}, Groq: {results.get('groq', {}).get('status')}", "details": results, } def check_youtube_auth(self) -> Dict[str, Any]: """Verify YouTube OAuth tokens are valid.""" if not self.state: return {"name": "youtube_auth", "status": "warn", "message": "No state manager", "details": {}} tokens = self.state.get_youtube_tokens() if not tokens: return { "name": "youtube_auth", "status": "warn", "message": "YouTube not authenticated - no tokens found", "details": {"has_tokens": False}, } # Check if tokens have required fields has_access = bool(tokens.get("access_token") or tokens.get("token")) has_refresh = bool(tokens.get("refresh_token")) if not has_access and not has_refresh: return { "name": "youtube_auth", "status": "error", "message": "YouTube tokens invalid - no access or refresh token", "details": {"has_access": has_access, "has_refresh": has_refresh}, } # Try to verify token with a lightweight API call access_token = tokens.get("access_token") or tokens.get("token", "") if access_token: try: import httpx with httpx.Client(timeout=10) as client: resp = client.get( "https://www.googleapis.com/youtube/v3/channels?part=snippet&mine=true", headers={"Authorization": f"Bearer {access_token}"}, ) if resp.status_code == 200: items = resp.json().get("items", []) channel_title = items[0]["snippet"]["title"] if items else "Unknown" return { "name": "youtube_auth", "status": "ok", "message": f"YouTube authenticated as: {channel_title}", "details": {"channel": channel_title, "has_refresh": has_refresh}, } elif resp.status_code == 401: # Token expired - can be refreshed if we have refresh_token if has_refresh: return { "name": "youtube_auth", "status": "warn", "message": "YouTube access token expired (refresh available)", "details": {"has_refresh": True, "needs_refresh": True}, } return { "name": "youtube_auth", "status": "error", "message": "YouTube token expired, no refresh token", "details": {"has_refresh": False}, } else: return { "name": "youtube_auth", "status": "warn", "message": f"YouTube API returned {resp.status_code}", "details": {"code": resp.status_code}, } except Exception as e: return { "name": "youtube_auth", "status": "warn", "message": f"Cannot verify YouTube auth: {e}", "details": {}, } return { "name": "youtube_auth", "status": "warn", "message": "YouTube has refresh token only", "details": {"has_refresh": has_refresh}, } def check_tts_engines(self) -> Dict[str, Any]: """Verify TTS engines are available.""" details = {} # Supertonic 3 try: from supertonic import TTS as SupertonicTTS details["supertonic"] = "available" supertonic_ok = True except ImportError: details["supertonic"] = "not_installed" supertonic_ok = False # gTTS try: from gtts import gTTS details["gtts"] = "available" gtts_ok = True except ImportError: details["gtts"] = "not_installed" gtts_ok = False # SiliconFlow TTS (API-based, check key) sf_key = os.getenv("SILICONFLOW_API_KEY", "") details["siliconflow_tts"] = "key_set" if sf_key else "no_key" if supertonic_ok: status = "ok" message = "Supertonic 3 available (primary TTS)" elif gtts_ok: status = "warn" message = "Only gTTS available (low quality fallback)" else: status = "error" message = "No TTS engine available!" return { "name": "tts_engines", "status": status, "message": message, "details": details, } def check_fonts(self) -> Dict[str, Any]: """Verify Montserrat fonts exist.""" font_dirs = [ "/usr/share/fonts/truetype/montserrat", "/usr/share/fonts/truetype/chinese", "/app/fonts", "/home/user/fonts", "fonts", ] found_fonts = {} for fd in font_dirs: p = Path(fd) if p.exists(): for font_file in p.glob("*.ttf"): found_fonts[font_file.name] = str(font_file) # Check for required Montserrat fonts has_black = any("Montserrat-Black" in name or "MontserratBlack" in name for name in found_fonts) has_bold = any("Montserrat-Bold" in name or "MontserratBold" in name for name in found_fonts) if has_black and has_bold: status = "ok" elif has_black or has_bold: status = "warn" else: status = "error" return { "name": "fonts", "status": status, "message": f"Montserrat Black: {'OK' if has_black else 'MISSING'}, Bold: {'OK' if has_bold else 'MISSING'} ({len(found_fonts)} fonts found)", "details": { "montserrat_black": has_black, "montserrat_bold": has_bold, "total_fonts": len(found_fonts), "font_files": list(found_fonts.keys())[:10], }, } def check_state(self) -> Dict[str, Any]: """Verify state persistence is working.""" if not self.state: return {"name": "state", "status": "warn", "message": "No state manager", "details": {}} try: state_data = self.state._state if hasattr(self.state, "_state") else {} processed = len(state_data.get("processed_videos", {})) failed = len(state_data.get("failed_videos", {})) queue = len(state_data.get("queue", [])) return { "name": "state", "status": "ok", "message": f"State OK: {processed} processed, {failed} failed, {queue} queued", "details": { "processed": processed, "failed": failed, "queued": queue, "has_state": True, }, } except Exception as e: return {"name": "state", "status": "error", "message": f"State check failed: {e}", "details": {}} # ================================================================ # Internal Methods # ================================================================ def _is_cache_valid(self) -> bool: """Check if cached results are still fresh.""" with self._lock: return (time.time() - self._cache_time) < self.CACHE_DURATION and bool(self._cache) def _save_health_history(self, report: Dict[str, Any]): """Save health check result to state history.""" if not self.state or not hasattr(self.state, "_state"): return try: history = self.state._state.get("health_history", []) entry = { "timestamp": report["checked_at"], "status": report["status"], "summary": report["summary"], } history.append(entry) # Keep last 100 entries if len(history) > 100: history = history[-100:] self.state._state["health_history"] = history # Don't save to HF on every health check (too frequent) # Will be persisted on next state.save() call except Exception as e: print(f"[HEALTH] Failed to save health history: {e}")