""" voice_manager.py - Advanced TTS voice management with multi-provider support. Manages multiple TTS providers with priority fallback: 1. Supertonic 3 (local ONNX, FREE, 31 languages) 2. Fish Speech 1.5 (SiliconFlow API, #2 TTS Arena, premium) 3. CosyVoice 2 (SiliconFlow API, cheaper) 4. IndexTTS 2 (SiliconFlow API, cheaper) 5. gTTS (Google, FREE, fallback) DROP-IN replacement for pipeline.py's _generate_voice_async(). Same signature and return format: {"path": str, "segments": list} Critical: Uses the same sequential concatenation approach as pipeline.py to prevent audio overlap/desync bugs. """ import asyncio import json import os import re import shutil import subprocess import threading import time from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple # ================================================================ # Supertonic 3 - Lazy Load # ================================================================ # Supertonic can be used in two modes: # 1. REMOTE: Call a separate HF Space that runs Supertonic (saves RAM on main Space) # 2. LOCAL: Load Supertonic model directly (fallback if remote is down) # Multi-Space Supertonic: Try multiple HF Spaces with failover # Comma-separated URLs are tried in order until one responds. # This ensures TTS is always available even when one Space is busy/paused. SUPERTONIC_REMOTE_URLS = [ url.strip() for url in os.getenv("SUPERTONIC_REMOTE_URL", "").split(",") if url.strip() ] # Also check individual Space env vars (supertonic-tts, supertonic-tts-2, supertonic-tts-3) _base = os.getenv("SUPERTONIC_REMOTE_URL", "").strip() if not SUPERTONIC_REMOTE_URLS: SUPERTONIC_REMOTE_URLS = [ "https://tomatitotoho-supertonic-tts.hf.space", "https://tomatitotoho-supertonic-tts-2.hf.space", "https://tomatitotoho-supertonic-tts-3.hf.space", ] SUPERTONIC_REMOTE_SECRET = os.getenv("SUPERTONIC_REMOTE_SECRET", "") # Optional auth # Track which Space was last successful (prefer it next time) _last_working_supertonic = [0] # Index into SUPERTONIC_REMOTE_URLS # Local fallback (only loaded if remote is not configured or fails) try: from supertonic import TTS as SupertonicTTS SUPERTONIC_AVAILABLE = True except ImportError: SUPERTONIC_AVAILABLE = False print("[VOICE-MGR] Supertonic 3 not available locally") SUPERTONIC_MODEL = None # Lazy-loaded singleton (local fallback only) def _get_supertonic_model(): """Get local Supertonic model (fallback when remote is not available).""" global SUPERTONIC_MODEL if SUPERTONIC_MODEL is None and SUPERTONIC_AVAILABLE: try: SUPERTONIC_MODEL = SupertonicTTS(auto_download=True) print("[VOICE-MGR] Supertonic 3 model loaded (local fallback)") except Exception as e: print(f"[VOICE-MGR] Supertonic 3 load failed: {e}") return SUPERTONIC_MODEL def _is_remote_supertonic_available() -> bool: """Check if any remote Supertonic Space is configured and reachable.""" if not SUPERTONIC_REMOTE_URLS: return False for url in SUPERTONIC_REMOTE_URLS: try: with httpx.Client(timeout=10) as client: resp = client.get(f"{url}/") if resp.status_code == 200: return True except Exception: continue return False # ================================================================ # Provider Configuration # ================================================================ TTS_PROVIDERS = { "supertonic": { "name": "Supertonic 3", "type": "local", "quality": 3, # 1-5 "speed": "fast", "cost": "free", "languages": 31, "voices": { "M1": {"gender": "male", "lang": "es", "description": "Male Spanish, dramatic"}, "F1": {"gender": "female", "lang": "es", "description": "Female Spanish, clear"}, }, }, "fish-speech": { "name": "Fish Speech 1.5", "type": "api", "api_base": "https://api.siliconflow.cn/v1", "model_id": "fishaudio/fish-speech-1.5", "quality": 5, "speed": "medium", "cost": "$15/M bytes", "voices": { "claire": {"gender": "female", "lang": "es", "description": "Female Spanish, premium"}, "alex": {"gender": "male", "lang": "es", "description": "Male Spanish, premium"}, }, }, "cosyvoice": { "name": "CosyVoice 2", "type": "api", "api_base": "https://api.siliconflow.cn/v1", "model_id": "FunAudioLLM/CosyVoice2-0.5B", "quality": 4, "speed": "medium", "cost": "$7.15/M bytes", "voices": { "default_female": {"gender": "female", "lang": "es", "description": "Female Spanish"}, "default_male": {"gender": "male", "lang": "es", "description": "Male Spanish"}, }, }, "indextts": { "name": "IndexTTS 2", "type": "api", "api_base": "https://api.siliconflow.cn/v1", "model_id": "IndexTeam/IndexTTS-2", "quality": 4, "speed": "medium", "cost": "$7.15/M bytes", "voices": { "default": {"gender": "neutral", "lang": "es", "description": "Spanish"}, }, }, "gtts": { "name": "gTTS (Google)", "type": "api", "quality": 2, "speed": "slow", "cost": "free", "voices": { "es": {"gender": "neutral", "lang": "es", "description": "Spanish, robotic"}, }, }, } # Default voice profiles DEFAULT_VOICE_PROFILES = { "default": {"provider": "supertonic", "voice": "M1", "lang": "es"}, "female": {"provider": "supertonic", "voice": "F1", "lang": "es"}, "premium_female": {"provider": "fish-speech", "model": "fish-speech", "voice": "claire", "lang": "es"}, "premium_male": {"provider": "fish-speech", "model": "fish-speech", "voice": "alex", "lang": "es"}, } class VoiceManager: """Advanced TTS voice management with multi-provider support.""" def __init__(self, state=None, brain=None): self.state = state self.brain = brain self._lock = threading.Lock() self._provider_status_cache = {} self._provider_status_time = 0.0 # ================================================================ # Main Entry Points (drop-in replacements for pipeline.py) # ================================================================ async def generate_full(self, segments: list, video_duration: float, voice: Optional[str] = None, provider: Optional[str] = None, work_dir: Optional[Path] = None) -> Optional[Dict[str, Any]]: """Generate complete voice track for all segments. DROP-IN replacement for pipeline._generate_voice_async(). Same return format: {"path": str, "segments": list} Args: segments: List of segment dicts with "text_es", "start", "end" video_duration: Total video duration in seconds voice: Voice name or profile name (default: from state.visual_config) provider: Force specific provider (default: auto-select) work_dir: Working directory for temp files Returns: {"path": str, "segments": list, "provider_used": str} or None """ if not segments: return None if not work_dir: import tempfile work_dir = Path(tempfile.gettempdir()) / "autodub_voice" work_dir.mkdir(parents=True, exist_ok=True) # Get voice profile profile = self._get_voice_profile(voice) provider_name = provider or profile.get("provider", "supertonic") voice_name = profile.get("voice", "M1") print(f"[VOICE-MGR] Generating voice: {len(segments)} segments, provider={provider_name}, voice={voice_name}") # Generate per-segment audio segment_results = [] successful_count = 0 for seg_idx, seg in enumerate(segments): text_es = seg.get("text_es", "").strip() if not text_es: seg["audio_path"] = None seg["audio_duration"] = 0 segment_results.append(seg) continue text_es = self._clean_text_for_tts(text_es) target_duration = seg["end"] - seg["start"] # Generate segment result = await self.generate_segment( text=text_es, target_duration=target_duration, voice=voice_name, provider=provider_name, work_dir=work_dir, seg_idx=seg_idx, ) if result and result.get("audio_path"): seg["audio_path"] = result["audio_path"] seg["audio_duration"] = result.get("duration", 0) successful_count += 1 else: # Fallback: try gTTS for this segment print(f"[VOICE-MGR] Segment {seg_idx}: primary failed, trying gTTS fallback...") fallback_result = await self.generate_segment( text=text_es, target_duration=target_duration, voice="es", provider="gtts", work_dir=work_dir, seg_idx=seg_idx, ) if fallback_result and fallback_result.get("audio_path"): seg["audio_path"] = fallback_result["audio_path"] seg["audio_duration"] = fallback_result.get("duration", 0) successful_count += 1 else: # Generate silence self._generate_silence(seg, target_duration, work_dir, seg_idx) segment_results.append(seg) if successful_count == 0: print("[VOICE-MGR] No segments generated successfully") return None # Concatenate all segments voice_path = await self.concatenate_segments( segment_results, video_duration, work_dir ) if not voice_path: return None # Normalize audio try: voice_path = await asyncio.get_running_loop().run_in_executor( None, self.normalize_audio, voice_path ) except Exception as e: print(f"[VOICE-MGR] Normalization failed (non-critical): {e}") size_kb = Path(voice_path).stat().st_size / 1024 print(f"[VOICE-MGR] Final voice: {size_kb:.0f} KB, provider={provider_name}") return { "path": voice_path, "segments": segment_results, "provider_used": provider_name, } async def generate_segment(self, text: str, target_duration: float, voice: Optional[str] = None, provider: Optional[str] = None, work_dir: Optional[Path] = None, seg_idx: int = 0) -> Optional[Dict[str, Any]]: """Generate TTS for a single segment. Returns: {"audio_path": str, "duration": float, "provider": str, "voice": str} """ if not text or not work_dir: return None provider = provider or self.select_provider(len(text), target_duration) voice = voice or "M1" seg_path = str(work_dir / f"seg_{seg_idx}_{provider}.wav") generated = False if provider == "supertonic": generated = await self._gen_supertonic(text, voice, seg_path, seg_idx) elif provider in ("fish-speech", "cosyvoice", "indextts"): generated = await self._gen_siliconflow(text, voice, provider, seg_path, seg_idx) elif provider == "gtts": generated = await self._gen_gtts(text, seg_path, seg_idx) if not generated: return None # Speed adjustment duration = self._get_duration(seg_path) if duration > 0 and target_duration > 0: adjusted_path = await asyncio.get_running_loop().run_in_executor( None, self.adjust_speed, seg_path, target_duration, work_dir, seg_idx ) if adjusted_path: seg_path = adjusted_path duration = self._get_duration(seg_path) return { "audio_path": seg_path, "duration": duration, "provider": provider, "voice": voice, } # ================================================================ # TTS Providers # ================================================================ async def _gen_supertonic(self, text: str, voice: str, output_path: str, seg_idx: int) -> bool: """Generate TTS using Supertonic 3 - tries multiple REMOTE Spaces with failover, then LOCAL fallback.""" # Try all remote Supertonic Spaces with failover if SUPERTONIC_REMOTE_URLS: # Start from the last working Space (or first if none worked yet) start_idx = _last_working_supertonic[0] % len(SUPERTONIC_REMOTE_URLS) for i in range(len(SUPERTONIC_REMOTE_URLS)): idx = (start_idx + i) % len(SUPERTONIC_REMOTE_URLS) url = SUPERTONIC_REMOTE_URLS[idx] try: result = await self._gen_supertonic_remote_url(url, text, voice, output_path, seg_idx) if result: _last_working_supertonic[0] = idx # Remember this Space worked return True print(f"[VOICE-MGR] Segment {seg_idx}: Supertonic Space {idx+1}/{len(SUPERTONIC_REMOTE_URLS)} failed, trying next...") except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: Supertonic Space {idx+1}/{len(SUPERTONIC_REMOTE_URLS)} error: {e}, trying next...") print(f"[VOICE-MGR] Segment {seg_idx}: All {len(SUPERTONIC_REMOTE_URLS)} remote Supertonic Spaces failed, trying local fallback...") # Local fallback model = _get_supertonic_model() if model is None: return False try: style = model.get_voice_style(voice_name=voice) loop = asyncio.get_running_loop() wav, duration = await loop.run_in_executor( None, lambda: model.synthesize(text, voice_style=style, lang="es") ) model.save_audio(wav, output_path) if Path(output_path).exists() and Path(output_path).stat().st_size > 500: print(f"[VOICE-MGR] Segment {seg_idx}: Supertonic LOCAL OK ({duration:.1f}s)") return True return False except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: Supertonic local error: {e}") return False async def _gen_supertonic_remote_url(self, base_url: str, text: str, voice: str, output_path: str, seg_idx: int) -> bool: """Generate TTS using a specific remote Supertonic Space URL.""" try: headers = {"Content-Type": "application/json"} if SUPERTONIC_REMOTE_SECRET: headers["Authorization"] = f"Bearer {SUPERTONIC_REMOTE_SECRET}" payload = { "text": text[:5000], "voice": voice, "lang": "es", "speed": 1.0, } loop = asyncio.get_running_loop() response = await loop.run_in_executor( None, lambda: self._supertonic_remote_call_url(base_url, headers, payload) ) if response and len(response) > 500: # Save raw WAV bytes Path(output_path).parent.mkdir(parents=True, exist_ok=True) with open(output_path, "wb") as f: f.write(response) if Path(output_path).exists() and Path(output_path).stat().st_size > 500: duration = self._get_duration(output_path) print(f"[VOICE-MGR] Segment {seg_idx}: Supertonic REMOTE ({base_url.split('//')[1][:30]}) OK ({duration:.1f}s)") return True return False except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: Remote Supertonic ({base_url[:40]}) error: {e}") return False # Keep backward compatibility async def _gen_supertonic_remote(self, text: str, voice: str, output_path: str, seg_idx: int) -> bool: """Generate TTS using remote Supertonic Space API (uses first configured URL).""" if not SUPERTONIC_REMOTE_URLS: return False return await self._gen_supertonic_remote_url( SUPERTONIC_REMOTE_URLS[0], text, voice, output_path, seg_idx ) def _supertonic_remote_call_url(self, base_url: str, headers: dict, payload: dict) -> Optional[bytes]: """Make remote Supertonic API call to a specific URL (sync).""" try: with httpx.Client(timeout=60) as client: resp = client.post( f"{base_url}/tts/raw", headers=headers, json=payload, ) resp.raise_for_status() return resp.content except Exception as e: print(f"[VOICE-MGR] Remote Supertonic API error ({base_url[:40]}): {e}") return None def _supertonic_remote_call(self, headers: dict, payload: dict) -> Optional[bytes]: """Make remote Supertonic API call (sync, uses first configured URL).""" if not SUPERTONIC_REMOTE_URLS: return None return self._supertonic_remote_call_url(SUPERTONIC_REMOTE_URLS[0], headers, payload) async def _gen_siliconflow(self, text: str, voice: str, provider: str, output_path: str, seg_idx: int) -> bool: """Generate TTS using SiliconFlow API (Fish Speech, CosyVoice, IndexTTS).""" sf_key = os.getenv("SILICONFLOW_API_KEY", "") if not sf_key: # Try OpenRouter for SiliconFlow routing print(f"[VOICE-MGR] Segment {seg_idx}: No SiliconFlow key") return False provider_config = TTS_PROVIDERS.get(provider, {}) model_id = provider_config.get("model_id", "fishaudio/fish-speech-1.5") api_base = provider_config.get("api_base", "https://api.siliconflow.cn/v1") # Build full voice identifier full_voice = f"{model_id}:{voice}" if ":" not in voice else voice try: import httpx headers = { "Authorization": f"Bearer {sf_key}", "Content-Type": "application/json", } payload = { "model": model_id, "input": text[:5000], "voice": full_voice, "response_format": "mp3", } loop = asyncio.get_running_loop() response = await loop.run_in_executor( None, lambda: self._siliconflow_api_call(api_base, headers, payload) ) if response and len(response) > 500: # Save as MP3 first, then convert to WAV mp3_path = output_path.replace(".wav", ".mp3") Path(output_path).parent.mkdir(parents=True, exist_ok=True) with open(mp3_path, "wb") as f: f.write(response) # Convert to WAV proc = subprocess.run([ "ffmpeg", "-y", "-i", mp3_path, "-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le", output_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(output_path).exists(): print(f"[VOICE-MGR] Segment {seg_idx}: {provider} OK") return True else: # WAV conversion failed — try re-encoding with explicit WAV format # BUG FIX: Never copy MP3 data to a .wav path (corrupts downstream concat) if Path(mp3_path).exists(): reenc_path = output_path.replace('.wav', '_reenc.wav') proc_re = subprocess.run([ "ffmpeg", "-y", "-i", mp3_path, "-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le", reenc_path, ], capture_output=True, timeout=15) if proc_re.returncode == 0 and Path(reenc_path).exists() and Path(reenc_path).stat().st_size > 500: shutil.move(reenc_path, output_path) print(f"[VOICE-MGR] Segment {seg_idx}: {provider} re-encoded OK") return True # Last resort: save as proper MP3, let concat handle format conversion mp3_output = output_path.replace('.wav', '.mp3') shutil.copy2(mp3_path, mp3_output) seg["audio_path"] = mp3_output # Update segment path print(f"[VOICE-MGR] Segment {seg_idx}: {provider} saved as MP3 fallback") return True return False except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: {provider} error: {e}") return False def _siliconflow_api_call(self, api_base: str, headers: dict, payload: dict) -> Optional[bytes]: """Make SiliconFlow TTS API call (sync).""" import httpx try: with httpx.Client(timeout=60) as client: resp = client.post( f"{api_base}/audio/speech", headers=headers, json=payload, ) resp.raise_for_status() return resp.content except Exception as e: print(f"[VOICE-MGR] SiliconFlow API error: {e}") return None async def _gen_gtts(self, text: str, output_path: str, seg_idx: int) -> bool: """Generate TTS using gTTS (Google Translate).""" try: from gtts import gTTS loop = asyncio.get_running_loop() gtts_path = output_path.replace(".wav", "_gtts.mp3") await loop.run_in_executor( None, lambda: gTTS(text=text[:5000], lang='es', slow=False).save(gtts_path) ) if Path(gtts_path).exists() and Path(gtts_path).stat().st_size > 500: # Convert to WAV proc = subprocess.run([ "ffmpeg", "-y", "-i", gtts_path, "-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le", output_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(output_path).exists(): print(f"[VOICE-MGR] Segment {seg_idx}: gTTS OK") return True # BUG FIX: Never copy MP3 data to a .wav path (corrupts downstream concat) # Try re-encoding with explicit WAV format reenc_path = output_path.replace('.wav', '_reenc.wav') proc_re = subprocess.run([ "ffmpeg", "-y", "-i", gtts_path, "-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le", reenc_path, ], capture_output=True, timeout=15) if proc_re.returncode == 0 and Path(reenc_path).exists() and Path(reenc_path).stat().st_size > 500: shutil.move(reenc_path, output_path) print(f"[VOICE-MGR] Segment {seg_idx}: gTTS re-encoded OK") return True # Last resort: save as proper MP3 mp3_output = output_path.replace('.wav', '.mp3') shutil.copy2(gtts_path, mp3_output) print(f"[VOICE-MGR] Segment {seg_idx}: gTTS saved as MP3 fallback") return True return False except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: gTTS error: {e}") return False # ================================================================ # Audio Processing # ================================================================ def adjust_speed(self, audio_path: str, target_duration: float, work_dir: Path, seg_idx: int = 0) -> Optional[str]: """Speed adjustment with atempo filter chain. Bidirectional: adjusts both faster and slower. Handles extreme ratios by chaining atempo filters (range 0.5-2.0 each). Truncates if still too long after adjustment. Returns: path to adjusted audio file (may be same as input). """ gen_duration = self._get_duration(audio_path) if gen_duration <= 0 or target_duration <= 0: return audio_path ratio = gen_duration / target_duration # Only adjust if > 0.5% off (same threshold as pipeline.py fix) if abs(ratio - 1.0) <= 0.005: return audio_path tempo = target_duration / gen_duration # Build atempo filter chain (each atempo has range 0.5-2.0) atempo_filters = [] remaining_tempo = tempo while remaining_tempo < 0.5: atempo_filters.append("atempo=0.5") remaining_tempo = remaining_tempo / 0.5 while remaining_tempo > 2.0: atempo_filters.append("atempo=2.0") remaining_tempo = remaining_tempo / 2.0 atempo_filters.append(f"atempo={remaining_tempo:.4f}") atempo_filter = ",".join(atempo_filters) try: tempo_path = str(work_dir / f"seg_{seg_idx}_tempo.wav") proc = subprocess.run([ "ffmpeg", "-i", audio_path, "-filter:a", atempo_filter, "-y", tempo_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(tempo_path).exists(): shutil.move(tempo_path, audio_path) gen_duration = self._get_duration(audio_path) except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: speed adjust error: {e}") # Truncation safety net (10ms tolerance, same as pipeline.py fix) gen_duration = self._get_duration(audio_path) if gen_duration > target_duration + 0.01: try: trunc_path = str(work_dir / f"seg_{seg_idx}_trunc.wav") proc = subprocess.run([ "ffmpeg", "-y", "-i", audio_path, "-t", str(target_duration), "-c:a", "pcm_s16le", trunc_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(trunc_path).exists(): shutil.move(trunc_path, audio_path) gen_duration = self._get_duration(audio_path) except Exception as e: print(f"[VOICE-MGR] Segment {seg_idx}: truncation error: {e}") return audio_path def normalize_audio(self, audio_path: str, target_lufs: float = -14.0) -> str: """Loudness normalization using FFmpeg loudnorm filter. Target: -14 LUFS (YouTube standard). Returns: path to normalized audio (overwrites input). """ if not audio_path or not Path(audio_path).exists(): return audio_path try: is_wav = audio_path.lower().endswith(".wav") norm_path = audio_path.replace(".mp3", "_norm.mp3").replace(".wav", "_norm.wav") # BUG FIX: Use pcm_s16le codec for .wav files, libmp3lame for .mp3 files # Previously always used libmp3lame which produced MP3 data in .wav files if is_wav: norm_codec = ["-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le"] else: norm_codec = ["-c:a", "libmp3lame", "-b:a", "128k"] # Two-pass loudnorm for precise normalization # Pass 1: Analyze proc1 = subprocess.run([ "ffmpeg", "-i", audio_path, "-af", f"loudnorm=I={target_lufs}:TP=-1:LRA=11:print_format=json", "-f", "null", "-", ], capture_output=True, text=True, timeout=30) # Parse measured values json_match = re.search(r'\{[^}]*"target_offset"[^}]*\}', proc1.stderr, re.DOTALL) if json_match: measured = json.loads(json_match.group()) # Pass 2: Apply with measured values proc2 = subprocess.run([ "ffmpeg", "-y", "-i", audio_path, "-af", (f"loudnorm=I={target_lufs}:TP=-1:LRA=11" f":measured_I={measured.get('input_i', target_lufs)}" f":measured_TP={measured.get('input_tp', -1)}" f":measured_LRA={measured.get('input_lra', 11)}" f":measured_thresh={measured.get('input_thresh', -70)}" f":offset={measured.get('target_offset', 0)}" f":linear=true"), *norm_codec, norm_path, ], capture_output=True, timeout=30) if proc2.returncode == 0 and Path(norm_path).exists(): shutil.move(norm_path, audio_path) print(f"[VOICE-MGR] Audio normalized to {target_lufs} LUFS") return audio_path # Fallback: single-pass normalization proc3 = subprocess.run([ "ffmpeg", "-y", "-i", audio_path, "-af", f"loudnorm=I={target_lufs}:TP=-1:LRA=11", *norm_codec, norm_path, ], capture_output=True, timeout=30) if proc3.returncode == 0 and Path(norm_path).exists(): shutil.move(norm_path, audio_path) return audio_path except Exception as e: print(f"[VOICE-MGR] Normalization error: {e}") return audio_path async def concatenate_segments(self, segments: list, video_duration: float, work_dir: Path) -> Optional[str]: """Sequential concatenation of segment audio files. CRITICAL: Uses the same approach as pipeline.py's _generate_supertonic_async: - Track current_time using ACTUAL audio duration (NOT Whisper timestamps) - Insert silence gaps between segments (>10ms threshold) - Handle overlap warnings - Pad final track to video_duration - Use FFmpeg concat demuxer with file list Returns: path to final voice track MP3. """ valid_segments = [ (i, seg) for i, seg in enumerate(segments) if seg.get("audio_path") and Path(seg["audio_path"]).exists() ] if not valid_segments: print("[VOICE-MGR] No valid segment audio files") return None voice_path = str(work_dir / "voice_es.mp3") if len(valid_segments) == 1: idx, seg = valid_segments[0] # BUG FIX: Always convert to consistent format (44100Hz mono) for single segment # Previously just copied the raw file which could have wrong sample rate/channels if seg["start"] > 0.05: delay_ms = round(seg["start"] * 1000) try: delayed_path = str(work_dir / f"seg_{idx}_delayed.wav") proc = subprocess.run([ "ffmpeg", "-y", "-i", seg["audio_path"], "-ar", "44100", "-ac", "1", "-af", f"adelay={delay_ms}|{delay_ms}", delayed_path, ], capture_output=True, timeout=15) if proc.returncode == 0: proc2 = subprocess.run([ "ffmpeg", "-y", "-i", delayed_path, "-af", f"apad=whole_dur={video_duration}", "-c:a", "libmp3lame", "-b:a", "128k", voice_path, ], capture_output=True, timeout=30) if proc2.returncode == 0 and Path(voice_path).exists(): return voice_path except Exception: pass # Convert to consistent format before copy try: proc_conv = subprocess.run([ "ffmpeg", "-y", "-i", seg["audio_path"], "-ar", "44100", "-ac", "1", "-c:a", "libmp3lame", "-b:a", "128k", voice_path, ], capture_output=True, timeout=15) if proc_conv.returncode == 0 and Path(voice_path).exists(): return voice_path except Exception: pass shutil.copy2(seg["audio_path"], voice_path) return voice_path if Path(voice_path).exists() else None # Multiple segments: sequential concat try: concat_parts = [] current_time = 0.0 # Track ACTUAL position (NOT Whisper timestamps) for idx, seg in valid_segments: seg_start = seg["start"] gap = seg_start - current_time # Insert silence gap if needed (10ms threshold) if gap > 0.01: silence_path = str(work_dir / f"gap_{idx}.wav") gap_duration = min(gap, video_duration - current_time) if gap_duration > 0: proc = subprocess.run([ "ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", "-t", str(gap_duration), "-c:a", "pcm_s16le", silence_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(silence_path).exists(): concat_parts.append(silence_path) current_time += gap_duration elif gap < 0: print(f"[VOICE-MGR] WARNING: Segment {idx} overlap by {-gap:.3f}s") # Convert segment to consistent format seg_wav_path = str(work_dir / f"seg_{idx}_aligned.wav") proc = subprocess.run([ "ffmpeg", "-y", "-i", seg["audio_path"], "-ar", "44100", "-ac", "1", "-c:a", "pcm_s16le", seg_wav_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(seg_wav_path).exists(): concat_parts.append(seg_wav_path) # Track ACTUAL audio duration (critical for desync fix) actual_dur = self._get_duration(seg_wav_path) current_time += actual_dur if not concat_parts: return None # Concatenate using FFmpeg concat demuxer list_file = str(work_dir / "concat_list.txt") with open(list_file, "w") as f: for part_path in concat_parts: f.write(f"file '{part_path}'\n") proc = subprocess.run([ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", list_file, "-af", f"apad=whole_dur={video_duration}", "-c:a", "libmp3lame", "-b:a", "128k", voice_path, ], capture_output=True, text=True, timeout=60) if proc.returncode != 0: # Fallback: concat without padding proc2 = subprocess.run([ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", list_file, "-c:a", "libmp3lame", "-b:a", "128k", voice_path, ], capture_output=True, timeout=30) if proc2.returncode != 0: print(f"[VOICE-MGR] Concat failed: {proc2.stderr[-300:]}") return None except Exception as e: print(f"[VOICE-MGR] Concat error: {e}") return None if not Path(voice_path).exists() or Path(voice_path).stat().st_size <= 1000: return None return voice_path # ================================================================ # Voice Selection # ================================================================ def select_provider(self, text_length: int = 0, target_duration: float = 0, quality_tier: str = "balanced") -> str: """Smart provider selection based on quality tier and availability. quality_tier: - "best": Fish Speech 1.5 (highest quality, costs money) - "balanced": Supertonic 3 (good quality, free, local) - "fast": gTTS (fastest, lowest quality) """ if quality_tier == "best": # Check if SiliconFlow is available if os.getenv("SILICONFLOW_API_KEY"): return "fish-speech" return "supertonic" elif quality_tier == "fast": return "gtts" else: # balanced if SUPERTONIC_AVAILABLE: return "supertonic" return "gtts" def select_voice(self, content_type: str = "storytelling") -> Dict[str, str]: """Select voice based on content type. Returns: {"provider": str, "voice": str} """ voice_map = { "storytelling": {"provider": "supertonic", "voice": "M1"}, # Male, dramatic "dramatic": {"provider": "supertonic", "voice": "M1"}, "casual": {"provider": "supertonic", "voice": "F1"}, # Female, conversational "educational": {"provider": "supertonic", "voice": "F1"}, # Female, clear "premium_storytelling": {"provider": "fish-speech", "voice": "alex"}, "premium_casual": {"provider": "fish-speech", "voice": "claire"}, } return voice_map.get(content_type, voice_map["storytelling"]) # ================================================================ # Provider Status # ================================================================ def get_available_providers(self) -> List[Dict[str, Any]]: """Check which TTS providers are available.""" providers = [] # Supertonic (local) if SUPERTONIC_AVAILABLE: model = _get_supertonic_model() providers.append({ "provider": "supertonic", "name": "Supertonic 3", "available": model is not None, "quality": 3, "cost": "free", "type": "local", }) # SiliconFlow providers (API) sf_key = os.getenv("SILICONFLOW_API_KEY", "") for name in ["fish-speech", "cosyvoice", "indextts"]: config = TTS_PROVIDERS.get(name, {}) providers.append({ "provider": name, "name": config.get("name", name), "available": bool(sf_key), "quality": config.get("quality", 3), "cost": config.get("cost", "unknown"), "type": "api", }) # gTTS try: from gtts import gTTS providers.append({ "provider": "gtts", "name": "gTTS (Google)", "available": True, "quality": 2, "cost": "free", "type": "api", }) except ImportError: providers.append({ "provider": "gtts", "name": "gTTS (Google)", "available": False, "quality": 2, "cost": "free", "type": "api", }) return providers def get_provider_status(self) -> Dict[str, Any]: """Detailed provider status with quality/speed/cost info.""" return { "providers": self.get_available_providers(), "voice_profiles": self._get_all_voice_profiles(), "supertonic_model_loaded": SUPERTONIC_MODEL is not None, } async def test_provider(self, provider: str, text: str = "Hola, esto es una prueba.") -> Dict[str, Any]: """Test a specific TTS provider. Returns: {"success": bool, "provider": str, "duration": float, "error": str} """ import tempfile work_dir = Path(tempfile.gettempdir()) / "voice_test" work_dir.mkdir(parents=True, exist_ok=True) try: result = await self.generate_segment( text=text, target_duration=5.0, # 5 second target provider=provider, work_dir=work_dir, seg_idx=0, ) if result and result.get("audio_path"): duration = self._get_duration(result["audio_path"]) return { "success": True, "provider": provider, "duration": duration, "path": result["audio_path"], } else: return {"success": False, "provider": provider, "error": "No audio generated"} except Exception as e: return {"success": False, "provider": provider, "error": str(e)} # ================================================================ # Internal Methods # ================================================================ def _get_voice_profile(self, profile_name: Optional[str] = None) -> Dict[str, str]: """Get voice profile from state or defaults.""" # Check state.visual_config.voice_profile if self.state and hasattr(self.state, "_state"): visual_config = self.state._state.get("visual_config", {}) profile_name = profile_name or visual_config.get("voice_profile", "default") # Check custom profiles in state if self.state and hasattr(self.state, "_state"): profiles = self.state._state.get("voice_profiles", {}) if profile_name in profiles: return profiles[profile_name] # Use default profiles return DEFAULT_VOICE_PROFILES.get(profile_name or "default", DEFAULT_VOICE_PROFILES["default"]) def _get_all_voice_profiles(self) -> Dict[str, Dict]: """Get all voice profiles (defaults + custom).""" profiles = dict(DEFAULT_VOICE_PROFILES) if self.state and hasattr(self.state, "_state"): custom = self.state._state.get("voice_profiles", {}) profiles.update(custom) return profiles def _clean_text_for_tts(self, text: str) -> str: """Clean text for TTS: remove special characters, normalize whitespace.""" # Remove emojis text = re.sub(r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF\U00002702-\U000027B0\U000024C2-\U0001F251]', '', text) # Remove URLs text = re.sub(r'https?://\S+', '', text) # Remove hashtags text = re.sub(r'#\w+', '', text) # Remove markdown text = re.sub(r'[*_~`]', '', text) # Normalize whitespace text = re.sub(r'\s+', ' ', text).strip() # Remove leading/trailing punctuation that sounds bad text = text.strip('.,;:') return text def _generate_silence(self, seg: Dict, target_duration: float, work_dir: Path, seg_idx: int): """Generate silence for a segment that couldn't be voiced.""" seg_audio_path = str(work_dir / f"seg_{seg_idx}_silence.wav") try: proc = subprocess.run([ "ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", "-t", str(max(0.1, target_duration)), seg_audio_path, ], capture_output=True, timeout=15) if proc.returncode == 0 and Path(seg_audio_path).exists(): seg["audio_path"] = seg_audio_path seg["audio_duration"] = target_duration return except Exception: pass seg["audio_path"] = None seg["audio_duration"] = 0 def _get_duration(self, path: str) -> float: """Get audio file duration using ffprobe.""" try: proc = subprocess.run([ "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path, ], capture_output=True, text=True, timeout=10) if proc.returncode == 0 and proc.stdout.strip(): return float(proc.stdout.strip()) except Exception: pass return 0.0