import time
import asyncio
import threading
from socket import socket, AF_INET, SOCK_STREAM
from json import dumps, loads
from collections import deque
from typing import Optional
import discord
from discord.ext import voice_recv
from discord.ext import commands
import numpy as np
import logging
import pyaudio
logging.getLogger("discord.ext.voice_recv.reader").setLevel(logging.ERROR)
local_audio_module = None
try:
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import local_audio_capture
local_audio_module = local_audio_capture
print("[OK] Local audio capture module loaded")
except ImportError as e:
print(f"[WARN] Local audio capture module not available: {e}")
try:
import pyaudio
print("[OK] PyAudio is available")
except Exception as pyaudio_error:
print(f"[ERROR] PyAudio error: {pyaudio_error}")
voice_recv_available = True
voice_recv_imported = voice_recv
connecting = False
CORE_HOST = os.getenv("CORE_HOST", "127.0.0.1")
CORE_PORT = int(os.getenv("CORE_PORT", "8765"))
DISCORD_TOKEN = os.getenv("DISCORD_TOKEN")
if not DISCORD_TOKEN:
raise RuntimeError("DISCORD_TOKEN environment variable not set")
MONITORED_USER_IDS = {
int(x.strip())
for x in os.getenv("MONITORED_USER_IDS", "").split(",")
if x.strip().isdigit()
}
LOCAL_AUDIO_ENABLED = os.getenv("LOCAL_AUDIO_ENABLED", "false").lower() == "true" # Disabled by default to avoid conflicts
LOCAL_AUDIO_DEVICE_INDEX = os.getenv("LOCAL_AUDIO_DEVICE_INDEX")
if LOCAL_AUDIO_DEVICE_INDEX is not None:
LOCAL_AUDIO_DEVICE_INDEX = int(LOCAL_AUDIO_DEVICE_INDEX)
else:
LOCAL_AUDIO_DEVICE_INDEX = 1
librosa = None
def get_librosa():
global librosa
if librosa is None:
import librosa as _librosa
librosa = _librosa
return librosa
vc_buffer = deque()
buffer_lock = threading.Lock()
last_audio_time = 10
SILENCE_TIMEOUT = 1.5
tts_playing_flag = False
tts_cooldown_time = 10 # Time when TTS cooldown ends
TTS_COOLDOWN_DURATION = 1.5 # Seconds to ignore audio after TTS finishes (echo protection)
current_voice_client: Optional[discord.VoiceClient] = None
current_sink = None
bot_loop: Optional[asyncio.AbstractEventLoop] = None
local_audio_pa = None # PyAudio instance for local audio
local_audio_stream = None # Stream for local audio
local_audio_thread = None # Thread for local audio processing
voice_connect_lock = asyncio.Lock()
voice_action_lock = asyncio.Lock()
import discord.opus
_real_decode = discord.opus.Decoder.decode
def safe_decode(self, *args, **kwargs):
try:
return _real_decode(self, *args, **kwargs)
except discord.opus.OpusError:
try:
return _real_decode(self, None, fec=False)
except Exception:
return b""
discord.opus.Decoder.decode = safe_decode
def is_allowed_member(member: discord.Member) -> bool:
if member is None:
return False
if member.bot:
return False
if MONITORED_USER_IDS:
return member.id in MONITORED_USER_IDS
return True
_raw_pcm_counter = 0
_debug_counter = 0
_vc_debug_wav_dir = "debug_audio"
os.makedirs(_vc_debug_wav_dir, exist_ok=True)
def save_debug_wav(audio, sample_rate: int, filename: str):
"""Save audio to debug_audio/ as a WAV file for STT debugging."""
global _debug_counter
_debug_counter += 1
audio = np.asarray(audio, dtype=np.float32)
audio = np.clip(audio, -1.0, 1.0)
path = os.path.join(_vc_debug_wav_dir, filename)
try:
import soundfile as sf
sf.write(path, audio, sample_rate)
except Exception as e:
print(f"[DEBUG] Save failed ({filename}): {e}")
def discord_pcm_to_whisper(pcm_bytes: bytes) -> np.ndarray:
"""Convert Discord PCM (48kHz, 2ch, s16le, interleaved) → 16kHz mono float32."""
global _raw_pcm_counter
_raw_pcm_counter += 1
raw = np.frombuffer(pcm_bytes, dtype=np.int16)
n = raw.size
if _raw_pcm_counter <= 5 or _raw_pcm_counter % 50 == 0:
clipped = np.sum((raw == 32767) | (raw == -32768))
if clipped > n * 0.05:
print(f"[CLIP] #{_raw_pcm_counter}: {clipped}/{n} samples at int16 extremes — source audio too loud!")
if _raw_pcm_counter < 5 or _raw_pcm_counter % 50 == 0:
print(f"[PCM] #{_raw_pcm_counter}: {len(pcm_bytes)}B, {n} int16, first 10: {raw[:10].tolist()}")
if _raw_pcm_counter <= 5:
try:
import soundfile as sf
sf.write(
f"{_vc_debug_wav_dir}/raw{_raw_pcm_counter}_stereo.wav",
raw.reshape(-1, 2),
48000,
subtype='PCM_16',
)
except Exception as e:
print(f"[PCM] Save err: {e}")
if n >= 2 and n % 2 == 0:
mono_48k = raw.astype(np.float32).reshape(-1, 2).mean(axis=1)
else:
mono_48k = raw.astype(np.float32)
mono_48k /= 32768.0
mono_48k = np.clip(mono_48k, -1.0, 1.0)
if _raw_pcm_counter <= 5:
save_debug_wav(mono_48k, 48000, f"stage2_{_raw_pcm_counter}_48k_mono.wav")
try:
from scipy.signal import resample_poly
mono_16k = resample_poly(mono_48k * 0.9, up=1, down=3)
except ImportError:
librosa_local = get_librosa()
mono_16k = librosa_local.resample(mono_48k * 0.9, orig_sr=48000, target_sr=16000)
mono_16k = np.clip(mono_16k, -1.0, 1.0)
if _raw_pcm_counter <= 5:
save_debug_wav(mono_16k, 16000, f"stage3_{_raw_pcm_counter}_16k_mono.wav")
return mono_16k.astype(np.float32)
def send_audio_to_core(audio_np: np.ndarray) -> Optional[str]:
MAX_SECONDS = 8
max_samples = 16000 * MAX_SECONDS
if len(audio_np) > max_samples:
audio_np = audio_np[-max_samples:]
for attempt in range(3):
try:
print(f"🔌 Connecting to vtuber_core at {CORE_HOST}:{CORE_PORT} (attempt {attempt+1}/3)...")
sock = socket(AF_INET, SOCK_STREAM)
sock.settimeout(30.0)
sock.connect((CORE_HOST, CORE_PORT))
print("✅ Connected to vtuber_core")
payload = {"samples": audio_np.tolist()}
data = dumps(payload).encode("utf-8")
sock.sendall(len(data).to_bytes(4, "big") + data)
response = b""
while True:
chunk = sock.recv(4096)
if not chunk:
break
response += chunk
sock.close()
if not response:
print("⚠️ No response from vtuber_core")
return None
result = loads(response.decode("utf-8"))
wav_path = result.get("wav_path")
if wav_path and os.path.exists(wav_path):
return wav_path
print(f"⚠️ TTS file not found: {wav_path}")
return None
except (ConnectionRefusedError, TimeoutError, OSError) as e:
print(f"❌ Core not ready (attempt {attempt+1}/3): {e}")
time.sleep(5)
except Exception as e:
print(f"❌ send_audio_to_core error: {e}")
import traceback
traceback.print_exc()
return None
return None
def play_wav_in_vc(vc: discord.VoiceClient, wav_path: str):
"""Play WAV file in Discord VC."""
global tts_playing_flag
if not vc or vc.channel is None or not os.path.exists(wav_path):
return
try:
if hasattr(vc, "is_playing") and vc.is_playing():
vc.stop()
except Exception:
pass
tts_playing_flag = True
def after_play(err):
global tts_playing_flag, tts_cooldown_time
tts_playing_flag = False
tts_cooldown_time = time.time() + TTS_COOLDOWN_DURATION # Set cooldown
try:
if os.path.exists(wav_path):
os.unlink(wav_path)
print(f"🗑️ Cleaned up: {wav_path}")
except Exception as e:
print(f"⚠️ Failed to clean up {wav_path}: {e}")
if err:
print(f"❌ Playback error: {err}")
try:
vc.play(
discord.FFmpegPCMAudio(
wav_path,
executable="ffmpeg"
),
after=after_play
)
print(f"▶️ Playing: {wav_path}")
except Exception as e:
print(f"❌ Error playing audio: {e}")
tts_playing_flag = False
print(f"❌ Error playing audio: {e}")
tts_playing_flag = False
def local_audio_player(wav_path: str):
"""Player function for local audio capture responses."""
global current_voice_client
if current_voice_client and os.path.exists(wav_path):
if bot_loop and bot_loop.is_running():
bot_loop.call_soon_threadsafe(lambda: play_wav_in_vc(current_voice_client, wav_path))
else:
play_wav_in_vc(current_voice_client, wav_path)
def stop_voice_listener(vc: Optional["discord.VoiceProtocol"] = None):
"""Stop any existing voice receive sink safely."""
if vc is None or not isinstance(vc, discord.VoiceClient):
return
for name in ("stop_listening", "stop"):
fn = getattr(vc, name, None)
if callable(fn):
try:
fn()
except Exception:
pass
break
def stop_local_audio_capture():
"""Stop local audio capture if running."""
global local_audio_pa, local_audio_stream, local_audio_thread
if local_audio_module and local_audio_pa:
try:
local_audio_module.stop_local_audio_capture(local_audio_pa, local_audio_stream)
local_audio_pa = None
local_audio_stream = None
local_audio_thread = None
print("🛑 Local audio capture stopped")
except Exception as e:
print(f"❌ Error stopping local audio capture: {e}")
async def connect_or_move_to_channel(channel: discord.VoiceChannel):
"""Single safe path for connecting/moving the voice client."""
global current_voice_client, current_sink, local_audio_pa, local_audio_stream, local_audio_thread
async with voice_connect_lock:
guild = channel.guild
vc = guild.voice_client
if vc:
if vc.channel is None:
try:
await vc.disconnect(force=True)
except Exception:
pass
vc = None
elif isinstance(vc, discord.VoiceClient) and vc.channel != channel:
await vc.move_to(channel)
print(f"🔊 Moved to {channel.name}")
current_voice_client = vc
return vc
print(f"🔌 Connecting to {channel.name}...")
if not voice_recv_available:
raise RuntimeError("voice_recv not available")
vc = await channel.connect(
cls=voice_recv_imported.VoiceRecvClient,
timeout=30.0,
self_deaf=False,
self_mute=False
)
print("✅ Connected with VoiceRecvClient")
stop_voice_listener(vc)
sink = DiscordAudioSink()
vc.listen(sink)
current_sink = sink
current_voice_client = vc
print(f"🎧 VC connected: {channel.name} (voice_recv sink attached)")
if LOCAL_AUDIO_ENABLED and local_audio_module:
try:
print("🎤 Starting local audio capture...")
local_audio_pa, local_audio_stream, local_audio_thread = local_audio_module.start_local_audio_capture(
device_index=LOCAL_AUDIO_DEVICE_INDEX,
discord_vc_player=local_audio_player
)
print("✅ Local audio capture started")
except Exception as e:
print(f"❌ Failed to start local audio capture: {e}")
try:
print("🔄 Retrying with default device...")
local_audio_pa, local_audio_stream, local_audio_thread = local_audio_module.start_local_audio_capture(
discord_vc_player=local_audio_player
)
print("✅ Local audio capture started with default device")
except Exception as e2:
print(f"❌ Failed to start local audio capture with default device: {e2}")
elif LOCAL_AUDIO_ENABLED:
print("⚠️ Local audio capture enabled but module not available")
return vc
class DiscordAudioSink(voice_recv_imported.AudioSink):
"""Proper voice_recv.AudioSink for PCM capture -> Whisper"""
def __init__(self):
super().__init__()
self.rolling = []
self.rolling_max = 50 # ~1 second (50 chunks × ~320 samples @16kHz)
self.speech_active = False
self.post_speech_seen = 0
self.hold_chunks = int(12000 / 320) # ~0.75s end-of-turn silence hold
def wants_opus(self) -> bool:
return False
def write(self, user, data):
global last_audio_time, vc_buffer, buffer_lock, tts_cooldown_time
if user is None or not data.pcm:
return
if not is_allowed_member(user):
return
if tts_playing_flag or time.time() < tts_cooldown_time:
return
try:
audio_np = discord_pcm_to_whisper(data.pcm)
if audio_np.size < 80:
return
rms = np.sqrt(np.mean(audio_np ** 2))
amp = np.max(np.abs(audio_np))
is_speech = amp >= 0.003 and rms >= 0.0006
self.rolling.append(audio_np)
if len(self.rolling) > self.rolling_max:
self.rolling.pop(0)
if not is_speech and not self.speech_active:
return
with buffer_lock:
if not self.speech_active and is_speech:
print(f"🔊 Speech onset ({amp:.5f} rms={rms:.5f}) prepending {len(self.rolling)} rolling chunks")
for chunk in self.rolling[:-1]:
vc_buffer.append(chunk)
self.rolling.clear()
self.speech_active = True
self.post_speech_seen = 0
if is_speech:
self.post_speech_seen = 0
else:
self.post_speech_seen += 1
vc_buffer.append(audio_np)
last_audio_time = time.time()
if self.post_speech_seen >= self.hold_chunks:
self.speech_active = False
except Exception as e:
print(f"⚠️ Audio sink write error: {e}")
def cleanup(self):
print("🧹 Audio sink cleanup")
def vc_speech_processor():
global last_audio_time, tts_playing_flag, current_voice_client
print("[REFRESH] Speech processor thread started")
print("💡 Make sure vtuber_core.py is running")
while True:
time.sleep(0.1)
try:
with buffer_lock:
if not vc_buffer:
continue
time_since_audio = time.time() - last_audio_time
total_samples = sum(len(chunk) for chunk in vc_buffer)
buffer_full = total_samples >= 16000 * 2 # ~2s utterance
silence_long = time_since_audio >= 1.0 and total_samples >= 14000 # ~0.9s minimum length
max_wait = time_since_audio >= 5.0 and total_samples >= 14000
if not buffer_full and not silence_long and not max_wait:
continue
chunks = list(vc_buffer)
vc_buffer.clear()
if not chunks:
continue
audio_np = np.concatenate(chunks)
if audio_np.size < 8000:
duration_short = audio_np.size / 16000.0
print(f"⚠️ Audio too short ({audio_np.size} samples = {duration_short:.2f}s), skipping")
continue
current_time = time.time()
if current_time < tts_cooldown_time:
continue
if tts_playing_flag:
continue
duration = audio_np.size / 16000.0
print(f"📊 Audio buffer: {len(chunks)} chunks, {len(audio_np)} total samples ({duration:.2f}s)")
print(f"📤 Sending {duration:.2f}s audio to core...")
save_debug_wav(audio_np, 16000, f"stt_input_{_debug_counter}.wav")
wav_path = send_audio_to_core(audio_np)
if not wav_path:
continue
if current_voice_client and isinstance(current_voice_client, discord.VoiceClient) and wav_path:
print(f"🔊 AI reply: {wav_path}")
if bot_loop and bot_loop.is_running():
bot_loop.call_soon_threadsafe(lambda: play_wav_in_vc(current_voice_client, wav_path))
else:
play_wav_in_vc(current_voice_client, wav_path)
with buffer_lock:
last_audio_time = 0.0
except Exception as e:
print(f"❌ Speech processor error: {e}")
threading.Thread(target=vc_speech_processor, daemon=True).start()
intents = discord.Intents.default()
intents.voice_states = True
intents.message_content = True
intents.members = True
bot = commands.Bot(command_prefix="!", intents=intents)
@bot.event
async def on_ready():
global bot_loop
bot_loop = asyncio.get_running_loop()
print(f"[OK] Bot logged in as {bot.user}")
print("[BOT] Voice bot ready!")
@bot.event
async def on_voice_state_update(member, before, after):
"""Single clean voice handler"""
global connecting
if connecting or member.bot or not is_allowed_member(member) or after.channel is None or before.channel == after.channel:
return
connecting = True
try:
await asyncio.sleep(2)
await connect_or_move_to_channel(after.channel)
except Exception as e:
print(f"❌ Voice error: {e}")
finally:
connecting = False
@bot.command()
async def join(ctx):
"""Join voice channel"""
if not ctx.author.voice:
await ctx.send("❌ Join voice first!")
return
await connect_or_move_to_channel(ctx.author.voice.channel)
await ctx.send("✅ Joined!")
@bot.command()
async def leave(ctx):
"""Leave voice"""
vc = ctx.guild.voice_client
if vc:
stop_voice_listener(vc)
stop_local_audio_capture() # Stop local audio capture
await vc.disconnect()
await ctx.send("👋 Left")
bot.run(DISCORD_TOKEN)
other file:
import socket
import json
import numpy as np
import tempfile
import soundfile as sf
import os
import sys
import argparse
import threading
from rich.console import Console
import nltk
import re
nltk.download('punkt', quiet=True)
EXCLUDED_PHRASES = set()
import numpy as np
from scipy.signal import resample_poly
def pcm_s16le_48k_stereo_to_16k_mono_float32(pcm_bytes: bytes) -> np.ndarray:
audio_i16 = np.frombuffer(pcm_bytes, dtype=np.int16)
if audio_i16.size == 0:
return np.zeros(0, dtype=np.float32)
usable = (audio_i16.size // 2) * 2
audio_i16 = audio_i16[:usable]
stereo = audio_i16.reshape(-1, 2)
mono = stereo.astype(np.float32).mean(axis=1)
mono = mono / 32768.0
mono = np.clip(mono, -1.0, 1.0)
mono_16k = resample_poly(mono, up=1, down=3)
return mono_16k.astype(np.float32)
if "HF_HOME" in os.environ and "TEST OMNIVOICE" in os.environ.get("HF_HOME", ""):
del os.environ["HF_HOME"]
if "HF_HUB_CACHE" not in os.environ:
os.environ["HF_HUB_CACHE"] = os.path.join(os.path.expanduser("~"), ".cache", "huggingface", "hub")
parser = argparse.ArgumentParser(description="Standalone VTuber Core Server")
parser.add_argument("--voice", type=str, default="meuro-enhanced-v2.wav")
parser.add_argument("--cfg-weight", type=float, default=0.5)
args = parser.parse_args()
console = Console()
from improved_local_stt import process_audio_chunk
def is_valid_text(text):
text = text.strip()
if not text:
return False
words = text.lower().split()
if len(words) > 3:
unique_words = set(words)
repetition_ratio = len(words) / len(unique_words) if len(unique_words) > 0 else 0
if repetition_ratio > 3.0: # High repetition likely means feedback
print(f"🚫 Likely feedback detected (repetition ratio: {repetition_ratio:.2f}): '{text}'")
return False
if len(words) > 3: # Only check repetition if we have more than 3 words
if len(set(words)) <= 1: # Only reject if all words are identical
print(f"🚫 All words identical: '{text}'")
return False
return True
HALLUCINATED_PHRASES = {
"thanks for watching", "subscribe", "thank you", "you",
"thank you for watching", "please subscribe", "like and subscribe",
"thanks", "bye", "goodbye", "see you", "thank",
"thanks for listening", "thank you for listening",
"the", "a", "and", "i", "we", "you", "it",
}
def is_valid_transcription(text):
"""Check if transcription is valid (not garbage/hallucination)"""
if not text or not text.strip():
return False
cleaned = re.sub(r'[.!?,]', '', text.lower()).strip()
if cleaned in HALLUCINATED_PHRASES:
print(f"🚫 Hallucination filtered: '{text}'")
return False
if cleaned in EXCLUDED_PHRASES:
return False
words = text.split()
if len(words) < 2:
print(f"🚫 Too short: '{text}'")
return False
unique_words = len(set(words))
if len(words) > 3 and unique_words / len(words) < 0.3:
print(f"🚫 Repetitive: '{text}'")
return False
return True
AUDIO_DEBUG_DIR = "debug_audio"
os.makedirs(AUDIO_DEBUG_DIR, exist_ok=True)
_audio_counter = 0
def save_debug_wav(audio, sample_rate: int, filename: str):
"""Save audio to debug_audio/ as a WAV file for STT debugging."""
audio = np.asarray(audio, dtype=np.float32)
audio = np.clip(audio, -1.0, 1.0)
path = os.path.join(AUDIO_DEBUG_DIR, filename)
try:
sf.write(path, audio, sample_rate)
except Exception as e:
print(f"[DEBUG] Save failed ({filename}): {e}")
def safe_transcribe(audio_np_or_pcm_bytes, *, input_sr: int = 48000, input_channels: int = 2):
"""Accept either:
- float32 numpy audio at 16kHz mono (current behavior)
- raw PCM bytes: S16LE stereo at 48kHz (will convert to 16k mono float32)
"""
global _audio_counter
if isinstance(audio_np_or_pcm_bytes, (bytes, bytearray)):
if input_sr != 48000 or input_channels != 2:
print(f"[WARN] PCM conversion assumes 48kHz stereo S16LE; got sr={input_sr}, ch={input_channels}")
audio_np = pcm_s16le_48k_stereo_to_16k_mono_float32(bytes(audio_np_or_pcm_bytes))
debug_in_sr = 16000
print(f"[PCM] Converted PCM bytes -> float32 mono {audio_np.size} samples @16k")
else:
audio_np = audio_np_or_pcm_bytes
debug_in_sr = 16000 # bot-side discord_pcm_to_whisper already produced 16kHz mono
if audio_np is None or getattr(audio_np, "size", 0) == 0:
return ""
print(f"[SEARCH] Audio analysis - size: {audio_np.size}, max_amplitude: {np.max(np.abs(audio_np)):.6f}")
max_samples = 48000
if len(audio_np) > max_samples:
audio_np = audio_np[-max_samples:]
print(f"[TRIM] Trimmed to last {max_samples} samples")
max_amp = np.max(np.abs(audio_np))
print(f"[MIC] Incoming audio | amp={max_amp:.6f} | samples={len(audio_np)}")
print(f"[PROCESS] Processing audio: {audio_np.size} samples, {max_amp:.6f} max amplitude")
_audio_counter += 1
save_debug_wav(audio_np, debug_in_sr, f"debug_{_audio_counter}.wav")
print(f"[DEBUG] Saved: debug_{_audio_counter}.wav")
duration_s = len(audio_np) / 16000.0
if duration_s < 0.5:
print(f"🚫 Rejected audio: {duration_s:.2f}s too short")
return ""
if max_amp < 0.001:
print(f"🚫 Rejected audio: amp={max_amp:.6f}")
return ""
if max_amp > 1.05:
print(f"🛑 Clipped audio: amp={max_amp:.6f} (should be ≤1.0)")
try:
print("[STT] Transcribing with improved local STT (fixed)...")
text = process_audio_chunk("discord_user", audio_np) or ""
print(f"[RAW] RAW STT: '{text}'")
print(f"[TEXT] Transcribed text: '{text}'")
return text # Always str
except Exception as stt_error:
print(f"[STT-ERROR] STT failed: {stt_error}")
return ""
import requests
SYSTEM_PROMPT = "You are a helpful AI."
def simple_llm(text):
try:
prompt = f"{SYSTEM_PROMPT}\n\nUser: {text}\nAI:"
print(f"📤 Sending to Ollama: '{text[:50]}...'")
response = requests.post(
"http://127.0.0.1:11434/api/generate",
headers={"Content-Type": "application/json"},
json={
"model": "drivedenpadev/deepseek-v3.2",
"prompt": prompt,
"stream": False
},
timeout=90
)
print(f"📥 Ollama response status: {response.status_code}")
if response.status_code != 200:
print(f"⚠️ Ollama returned status {response.status_code}: {response.text}")
return "Sorry, something went wrong."
try:
data = response.json()
except Exception:
print("❌ Failed to parse JSON:", response.text)
return "Invalid response from LLM."
print(f"📄 Ollama response data: {data}")
result = data.get("response", "").strip()
if not result:
print("⚠️ Empty response from model")
return "No response from AI."
print(f"💬 Extracted response: '{result}'")
return result
except requests.exceptions.ConnectionError:
print("❌ Cannot connect to Ollama - is it running?")
return "Sorry, LLM service unavailable."
except requests.exceptions.Timeout:
print("❌ Ollama request timed out")
return "Sorry, taking too long to think."
except Exception as e:
print(f"❌ Ollama error: {e}")
import traceback
traceback.print_exc()
return "Sorry, something went wrong."
SYSTEM_PROMPT = """
You are a funny VTuber. Keep replies under 14 words. No emojis. Act human
"""
sys.path.insert(0, os.path.dirname(_file_))
from tts import TextToSpeechService, sanitize_tts_text
try:
print("[ROCKET] Initializing TTS service...")
tts = TextToSpeechService()
print(f"[OK] TTS service initialized (device: {tts.device})")
if args.voice and os.path.exists(args.voice):
print(f"[OK] Voice model found: {args.voice}")
elif args.voice:
print(f"[WARN] Voice model not found: {args.voice}")
else:
print("[WARN] No voice model specified")
except Exception as e:
print(f"[ERROR] Failed to initialize TTS service: {e}")
import traceback
traceback.print_exc()
tts = None
HOST = os.getenv("CORE_HOST", "127.0.0.1")
PORT = int(os.getenv("CORE_PORT", "8765"))
def handle_audio(audio_np_or_pcm_bytes):
try:
print("📝 Transcribing...")
text = safe_transcribe(audio_np_or_pcm_bytes)
print(f"📝 Transcribed text: '{text}'")
print(f"👤 You: {text}")
if not is_valid_transcription(text):
print(f"⏭️ Skipping Ollama (invalid transcript)")
return None, text
response = simple_llm(text)
print(f"🤖 AI: {response}")
if len(response) > 500:
response = response[:500] + "..."
print(f"✂️ Trimmed long response: {len(response)} chars")
safe_text = sanitize_tts_text(response)
print(f"🔤 Sanitized text: '{safe_text}'")
try:
if tts is None:
print("❌ TTS service is not initialized")
return None, text
print("🎵 Generating audio...")
sr, audio = tts.synthesize(
safe_text,
audio_prompt_path=args.voice,
cfg_weight=args.cfg_weight
)
print(f"🎵 TTS result: sr={sr}, audio_shape={getattr(audio, 'shape', 'N/A')}")
if audio is None or len(audio) == 0:
print("🔇 TTS generated empty audio, skipping")
return None, text
tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
tmp.close()
sf.write(tmp.name, audio, sr)
print(f"✅ TTS generated successfully: {tmp.name}")
return tmp.name, text
except Exception as tts_error:
print(f"❌ TTS generation failed: {tts_error}")
import traceback
traceback.print_exc()
return None, text
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
return None
def server_loop():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((HOST, PORT))
sock.listen(1)
print(f"[SERVER] Standalone Core on {HOST}:{PORT}")
while True:
conn, addr = sock.accept()
print(f"[CONNECT] {addr}")
size_bytes = conn.recv(4)
size = int.from_bytes(size_bytes, "big")
data = b""
while len(data) < size:
data += conn.recv(4096)
payload = json.loads(data)
if "pcm_s16le" in payload:
pcm_val = payload["pcm_s16le"]
if isinstance(pcm_val, str):
import base64
audio_input = base64.b64decode(pcm_val)
else:
audio_input = bytes(pcm_val)
else:
audio_list = payload.get("samples", [])
audio_input = np.array(audio_list, dtype=np.float32)
wav_path, transcription = handle_audio(audio_input)
response = json.dumps({"wav_path": wav_path or "", "transcription": transcription}).encode()
conn.sendall(response)
conn.close()
if _name_ == "_main_":
server_loop()
what am i doing wrong help i’ve tried everything and nothing works to fix stt on discord i’m trying to do it locally because i am losing my mind after months of trying to make this work
source & further reading
discuss.huggingface.co — original article
Rakarrack-0.6.1 port making progress! ( AI assisted )
Cloud Storage Poll
Welcome to Haiku basic(Haiku Docs, Haiku slide and Haiku sheets)