497 lines
19 KiB
Python
497 lines
19 KiB
Python
"""
|
||
Wake word detection module using Porcupine.
|
||
Listens for the configured wake word.
|
||
"""
|
||
|
||
# Этот модуль отвечает за "уши" ассистента в режиме ожидания.
|
||
# Он использует библиотеку Porcupine для эффективного (мало CPU) обнаружения ключевой фразы.
|
||
|
||
import pvporcupine
|
||
import pyaudio
|
||
import struct
|
||
import io
|
||
import wave
|
||
import time
|
||
import numpy as np
|
||
import httpx
|
||
from collections import deque
|
||
from deepgram import DeepgramClient
|
||
from deepgram.clients.listen.v1.rest.options import PrerecordedOptions
|
||
from ..core.config import (
|
||
DEEPGRAM_API_KEY,
|
||
PORCUPINE_ACCESS_KEY,
|
||
PORCUPINE_KEYWORD_PATH,
|
||
PORCUPINE_SENSITIVITY,
|
||
WAKEWORD_HIT_COOLDOWN_SECONDS,
|
||
WAKEWORD_ENABLE_FALLBACK_STT,
|
||
WAKEWORD_MIN_RMS,
|
||
WAKEWORD_REOPEN_GRACE_SECONDS,
|
||
WAKEWORD_RMS_MULTIPLIER,
|
||
WAKE_WORD,
|
||
WAKE_WORD_ALIASES,
|
||
)
|
||
from ..core.audio_manager import get_audio_manager
|
||
|
||
|
||
class WakeWordDetector:
|
||
"""Класс для обнаружения wake word с использованием Porcupine."""
|
||
|
||
def __init__(self):
|
||
self.porcupine = None
|
||
self.audio_stream = None
|
||
self.pa = None
|
||
self._audio_manager = None
|
||
self._input_device_index = None
|
||
self._capture_sample_rate = None
|
||
self._capture_frame_length = None
|
||
self._resampled_pcm_buffer = np.array([], dtype=np.int16)
|
||
self._stream_closed = True # Флаг состояния потока (закрыт/открыт)
|
||
self._last_hit_ts = 0.0
|
||
self._fallback_dg_client = None
|
||
self._fallback_pre_roll = deque(maxlen=4)
|
||
self._fallback_frames = []
|
||
self._fallback_active = False
|
||
self._fallback_silence_frames = 0
|
||
self._fallback_last_attempt_ts = 0.0
|
||
self._fallback_last_error_ts = 0.0
|
||
self._stream_opened_ts = 0.0
|
||
self._rms_history = deque(maxlen=220)
|
||
self._wakeword_aliases_compact = {
|
||
self._compact_text(WAKE_WORD),
|
||
*(self._compact_text(alias) for alias in WAKE_WORD_ALIASES),
|
||
}
|
||
|
||
def initialize(self):
|
||
"""Инициализация Porcupine и PyAudio."""
|
||
# Создаем экземпляр Porcupine с нашим ключом доступа и файлом модели (.ppn)
|
||
self.porcupine = pvporcupine.create(
|
||
access_key=PORCUPINE_ACCESS_KEY,
|
||
keyword_paths=[str(PORCUPINE_KEYWORD_PATH)],
|
||
sensitivities=[PORCUPINE_SENSITIVITY],
|
||
)
|
||
|
||
# Используем общий экземпляр PyAudio
|
||
self._audio_manager = get_audio_manager()
|
||
self.pa = self._audio_manager.get_pyaudio()
|
||
self._open_stream()
|
||
print(
|
||
f"🎤 Ожидание wake word '{WAKE_WORD}' "
|
||
f"(sens={PORCUPINE_SENSITIVITY:.2f}, mic_rate={self._capture_sample_rate})..."
|
||
)
|
||
|
||
def _open_stream(self):
|
||
"""Открытие аудиопотока с микрофона."""
|
||
if self.audio_stream and not self._stream_closed:
|
||
return # Уже открыт
|
||
|
||
# Если был открыт старый поток, пробуем закрыть
|
||
if self.audio_stream:
|
||
try:
|
||
self.audio_stream.close()
|
||
except Exception:
|
||
pass
|
||
|
||
target_rate = int(self.porcupine.sample_rate)
|
||
fallback_rates = [48000, 44100, 32000, 22050, 16000]
|
||
self.audio_stream, self._input_device_index, actual_rate = self._audio_manager.open_input_stream(
|
||
rate=target_rate,
|
||
channels=1,
|
||
format=pyaudio.paInt16,
|
||
frames_per_buffer=self.porcupine.frame_length,
|
||
preferred_index=self._input_device_index,
|
||
fallback_rates=fallback_rates,
|
||
)
|
||
self._capture_sample_rate = int(actual_rate)
|
||
self._capture_frame_length = max(
|
||
64,
|
||
int(
|
||
round(
|
||
self.porcupine.frame_length
|
||
* self._capture_sample_rate
|
||
/ target_rate
|
||
)
|
||
),
|
||
)
|
||
self._resampled_pcm_buffer = np.array([], dtype=np.int16)
|
||
self._stream_closed = False
|
||
self._stream_opened_ts = time.time()
|
||
self._reset_fallback_state()
|
||
|
||
@staticmethod
|
||
def _compute_rms(pcm: np.ndarray) -> float:
|
||
if pcm.size == 0:
|
||
return 0.0
|
||
as_float = pcm.astype(np.float32)
|
||
return float(np.sqrt(np.mean(as_float * as_float)))
|
||
|
||
@staticmethod
|
||
def _compact_text(text: str) -> str:
|
||
text = str(text or "").lower().replace("ё", "е")
|
||
return "".join(ch for ch in text if ch.isalnum())
|
||
|
||
def _remember_rms(self, rms: float):
|
||
if rms <= 0:
|
||
return
|
||
self._rms_history.append(float(rms))
|
||
|
||
def _noise_floor_rms(self) -> float:
|
||
if not self._rms_history:
|
||
return 0.0
|
||
# Низкий процентиль устойчив к редким всплескам/голосу.
|
||
return float(np.percentile(np.asarray(self._rms_history, dtype=np.float32), 20))
|
||
|
||
def _wakeword_rms_threshold(self) -> float:
|
||
floor = self._noise_floor_rms()
|
||
dynamic = floor * float(WAKEWORD_RMS_MULTIPLIER)
|
||
# Защитный максимум, чтобы в очень шумном окружении не "убить" детект полностью.
|
||
dynamic = min(dynamic, float(WAKEWORD_MIN_RMS) * 4.0)
|
||
return max(float(WAKEWORD_MIN_RMS), dynamic)
|
||
|
||
def _is_hit_in_guard_window(
|
||
self, now_ts: float, *, ignore_hit_cooldown: bool = False
|
||
) -> bool:
|
||
if (
|
||
not ignore_hit_cooldown
|
||
and now_ts - self._last_hit_ts < float(WAKEWORD_HIT_COOLDOWN_SECONDS)
|
||
):
|
||
return True
|
||
if (
|
||
self._stream_opened_ts > 0
|
||
and now_ts - self._stream_opened_ts < float(WAKEWORD_REOPEN_GRACE_SECONDS)
|
||
):
|
||
return True
|
||
return False
|
||
|
||
def _accept_porcupine_hit(
|
||
self,
|
||
pcm: np.ndarray,
|
||
now_ts: float,
|
||
*,
|
||
ignore_hit_cooldown: bool = False,
|
||
during_tts: bool = False,
|
||
) -> bool:
|
||
if self._is_hit_in_guard_window(
|
||
now_ts, ignore_hit_cooldown=ignore_hit_cooldown
|
||
):
|
||
return False
|
||
rms = self._compute_rms(pcm)
|
||
# Для "чистого" Porcupine оставляем мягкий амплитудный фильтр:
|
||
# он отсеивает тишину/щелчки и ложные фаны от фонового шума.
|
||
# Во время TTS делаем фильтр строже, чтобы собственная колонка
|
||
# не "будила" ассистента.
|
||
factor = 0.95 if during_tts else 0.75
|
||
threshold = max(80.0, self._wakeword_rms_threshold() * factor)
|
||
if rms < threshold:
|
||
return False
|
||
self._last_hit_ts = now_ts
|
||
return True
|
||
|
||
def _reset_fallback_state(self):
|
||
self._fallback_pre_roll.clear()
|
||
self._fallback_frames = []
|
||
self._fallback_active = False
|
||
self._fallback_silence_frames = 0
|
||
|
||
def _get_fallback_client(self):
|
||
if not WAKEWORD_ENABLE_FALLBACK_STT:
|
||
return None
|
||
if not DEEPGRAM_API_KEY:
|
||
return None
|
||
if self._fallback_dg_client is None:
|
||
self._fallback_dg_client = DeepgramClient(DEEPGRAM_API_KEY)
|
||
return self._fallback_dg_client
|
||
|
||
def _pcm_to_wav_bytes(self, pcm: np.ndarray) -> bytes:
|
||
buffer = io.BytesIO()
|
||
with wave.open(buffer, "wb") as wav_file:
|
||
wav_file.setnchannels(1)
|
||
wav_file.setsampwidth(2)
|
||
wav_file.setframerate(int(self.porcupine.sample_rate))
|
||
wav_file.writeframes(np.asarray(pcm, dtype=np.int16).tobytes())
|
||
return buffer.getvalue()
|
||
|
||
def _transcribe_wakeword_candidate(self, pcm: np.ndarray) -> bool:
|
||
client = self._get_fallback_client()
|
||
if client is None or pcm.size == 0:
|
||
return False
|
||
|
||
try:
|
||
response = client.listen.rest.v("1").transcribe_file(
|
||
{"buffer": self._pcm_to_wav_bytes(pcm)},
|
||
PrerecordedOptions(
|
||
model="nova-2",
|
||
language="ru",
|
||
smart_format=False,
|
||
punctuate=False,
|
||
utterances=False,
|
||
numerals=False,
|
||
),
|
||
timeout=httpx.Timeout(2.2, connect=2.2, read=2.2, write=2.2),
|
||
)
|
||
except Exception as exc:
|
||
now = time.time()
|
||
if now - self._fallback_last_error_ts >= 30.0:
|
||
print(f"⚠️ Wake word fallback STT failed: {exc}")
|
||
self._fallback_last_error_ts = now
|
||
return False
|
||
|
||
transcript = ""
|
||
confidence = None
|
||
try:
|
||
channels = response.results.channels or []
|
||
if channels and channels[0].alternatives:
|
||
first_alt = channels[0].alternatives[0]
|
||
transcript = str(first_alt.transcript or "").strip()
|
||
try:
|
||
confidence = float(first_alt.confidence)
|
||
except Exception:
|
||
confidence = None
|
||
except Exception:
|
||
transcript = ""
|
||
confidence = None
|
||
|
||
compact = self._compact_text(transcript)
|
||
if confidence is not None and confidence < 0.62:
|
||
return False
|
||
if compact in self._wakeword_aliases_compact:
|
||
print(f"✅ Wake word обнаружен fallback STT: {transcript}")
|
||
return True
|
||
return False
|
||
|
||
def _check_fallback_wakeword(
|
||
self,
|
||
pcm: np.ndarray,
|
||
*,
|
||
during_tts: bool = False,
|
||
ignore_hit_cooldown: bool = False,
|
||
) -> bool:
|
||
if not WAKEWORD_ENABLE_FALLBACK_STT:
|
||
return False
|
||
if self.porcupine is None:
|
||
return False
|
||
|
||
rms = self._compute_rms(pcm)
|
||
base_threshold = self._wakeword_rms_threshold()
|
||
speech_factor = 1.1 if during_tts else 0.85
|
||
speech_threshold = max(170.0, base_threshold * speech_factor)
|
||
silence_threshold = max(95.0, speech_threshold * 0.55)
|
||
silence_frames_to_finalize = 10 if during_tts else 8
|
||
min_frames = 10 if during_tts else 7
|
||
max_frames = 40
|
||
min_attempt_interval = 2.5 if during_tts else 1.0
|
||
|
||
if rms >= speech_threshold:
|
||
if not self._fallback_active:
|
||
self._fallback_active = True
|
||
self._fallback_frames = list(self._fallback_pre_roll)
|
||
self._fallback_silence_frames = 0
|
||
self._fallback_frames.append(np.asarray(pcm, dtype=np.int16))
|
||
elif self._fallback_active:
|
||
self._fallback_frames.append(np.asarray(pcm, dtype=np.int16))
|
||
if rms <= silence_threshold:
|
||
self._fallback_silence_frames += 1
|
||
else:
|
||
self._fallback_silence_frames = 0
|
||
|
||
if len(self._fallback_frames) > max_frames:
|
||
self._reset_fallback_state()
|
||
elif self._fallback_silence_frames >= silence_frames_to_finalize:
|
||
candidate = np.concatenate(self._fallback_frames) if self._fallback_frames else np.asarray([], dtype=np.int16)
|
||
self._reset_fallback_state()
|
||
if len(candidate) >= min_frames * int(self.porcupine.frame_length):
|
||
now = time.time()
|
||
candidate_rms = self._compute_rms(candidate)
|
||
candidate_threshold = self._wakeword_rms_threshold() * (
|
||
0.95 if during_tts else 0.75
|
||
)
|
||
candidate_threshold = max(float(WAKEWORD_MIN_RMS), candidate_threshold)
|
||
if (
|
||
now - self._fallback_last_attempt_ts >= min_attempt_interval
|
||
and not self._is_hit_in_guard_window(
|
||
now, ignore_hit_cooldown=ignore_hit_cooldown
|
||
)
|
||
and candidate_rms >= candidate_threshold
|
||
):
|
||
self._fallback_last_attempt_ts = now
|
||
if self._transcribe_wakeword_candidate(candidate):
|
||
self._last_hit_ts = now
|
||
return True
|
||
|
||
self._fallback_pre_roll.append(np.asarray(pcm, dtype=np.int16))
|
||
return False
|
||
|
||
def stop_monitoring(self):
|
||
"""Явная остановка и закрытие потока (чтобы освободить микрофон для других задач)."""
|
||
if self.audio_stream and not self._stream_closed:
|
||
try:
|
||
self.audio_stream.stop_stream()
|
||
self.audio_stream.close()
|
||
except Exception:
|
||
pass
|
||
self._stream_closed = True
|
||
self._stream_opened_ts = 0.0
|
||
self._reset_fallback_state()
|
||
|
||
def _resample_to_target_rate(self, pcm: np.ndarray) -> np.ndarray:
|
||
target_rate = int(self.porcupine.sample_rate)
|
||
source_rate = int(self._capture_sample_rate or target_rate)
|
||
if source_rate == target_rate:
|
||
return pcm
|
||
if pcm.size == 0:
|
||
return np.array([], dtype=np.int16)
|
||
target_length = max(1, int(round(pcm.size * target_rate / source_rate)))
|
||
x_old = np.arange(pcm.size, dtype=np.float32)
|
||
x_new = np.linspace(0.0, float(max(0, pcm.size - 1)), target_length)
|
||
resampled = np.interp(x_new, x_old, pcm.astype(np.float32))
|
||
return np.asarray(resampled, dtype=np.int16)
|
||
|
||
def _read_porcupine_frame(self):
|
||
target_length = int(self.porcupine.frame_length)
|
||
if self._capture_sample_rate == self.porcupine.sample_rate:
|
||
pcm = self.audio_stream.read(target_length, exception_on_overflow=False)
|
||
return np.asarray(struct.unpack_from("h" * target_length, pcm), dtype=np.int16)
|
||
|
||
while self._resampled_pcm_buffer.size < target_length:
|
||
raw = self.audio_stream.read(
|
||
self._capture_frame_length, exception_on_overflow=False
|
||
)
|
||
captured = np.frombuffer(raw, dtype=np.int16)
|
||
converted = self._resample_to_target_rate(captured)
|
||
if converted.size:
|
||
self._resampled_pcm_buffer = np.concatenate(
|
||
(self._resampled_pcm_buffer, converted)
|
||
)
|
||
|
||
frame = self._resampled_pcm_buffer[:target_length]
|
||
self._resampled_pcm_buffer = self._resampled_pcm_buffer[target_length:]
|
||
return frame
|
||
|
||
def wait_for_wakeword(self, timeout: float = None) -> bool:
|
||
"""
|
||
Блокирующая функция: ждет, пока не будет услышана wake word
|
||
или пока не истечет timeout.
|
||
|
||
Args:
|
||
timeout: Максимальное время ожидания в секундах. None = ждать бесконечно.
|
||
|
||
Returns:
|
||
True, если фраза обнаружена. False, если вышел таймаут.
|
||
"""
|
||
import time
|
||
|
||
if not self.porcupine:
|
||
self.initialize()
|
||
|
||
# Убеждаемся, что поток открыт
|
||
self._open_stream()
|
||
|
||
start_time = time.time()
|
||
|
||
while True:
|
||
# Проверка таймаута
|
||
if timeout and (time.time() - start_time > timeout):
|
||
return False
|
||
|
||
# Читаем небольшой кусочек аудио (frame)
|
||
pcm = self._read_porcupine_frame()
|
||
self._remember_rms(self._compute_rms(pcm))
|
||
|
||
# Обрабатываем фрейм через Porcupine
|
||
keyword_index = self.porcupine.process(pcm.tolist())
|
||
|
||
# Если keyword_index >= 0, значит ключевое слово обнаружено
|
||
if keyword_index >= 0:
|
||
now = time.time()
|
||
if self._accept_porcupine_hit(pcm, now, during_tts=False):
|
||
print("✅ Wake word обнаружен!")
|
||
# Важно: закрываем поток, чтобы освободить микрофон для STT (Deepgram)
|
||
self.stop_monitoring()
|
||
return True
|
||
if self._check_fallback_wakeword(pcm):
|
||
self.stop_monitoring()
|
||
return True
|
||
|
||
def check_wakeword_once(self) -> bool:
|
||
"""
|
||
Неблокирующая проверка (один кадр).
|
||
Используется во время того, как ассистент говорит (TTS),
|
||
чтобы проверить, не пытается ли пользователь его перебить.
|
||
|
||
Returns:
|
||
True, если фраза обнаружена прямо сейчас.
|
||
"""
|
||
import time
|
||
|
||
if not self.porcupine:
|
||
self.initialize()
|
||
|
||
try:
|
||
self._open_stream()
|
||
|
||
pcm = self._read_porcupine_frame()
|
||
self._remember_rms(self._compute_rms(pcm))
|
||
|
||
keyword_index = self.porcupine.process(pcm.tolist())
|
||
if keyword_index >= 0:
|
||
now = time.time()
|
||
if not self._accept_porcupine_hit(
|
||
pcm,
|
||
now,
|
||
ignore_hit_cooldown=True,
|
||
during_tts=True,
|
||
):
|
||
return False
|
||
print("🛑 Wake word обнаружен во время ответа!")
|
||
return True
|
||
if self._check_fallback_wakeword(
|
||
pcm, during_tts=True, ignore_hit_cooldown=True
|
||
):
|
||
print("🛑 Wake word обнаружен fallback STT во время ответа!")
|
||
return True
|
||
return False
|
||
except Exception:
|
||
return False
|
||
|
||
def cleanup(self):
|
||
"""Освобождение ресурсов при выходе."""
|
||
self.stop_monitoring()
|
||
# self.pa.terminate() - Не делаем этого, так как PyAudio общий
|
||
if self.porcupine:
|
||
self.porcupine.delete()
|
||
|
||
|
||
# Глобальный экземпляр детектора (Singleton)
|
||
_detector = None
|
||
|
||
|
||
def get_detector() -> WakeWordDetector:
|
||
"""Получить или создать глобальный экземпляр детектора."""
|
||
global _detector
|
||
if _detector is None:
|
||
_detector = WakeWordDetector()
|
||
return _detector
|
||
|
||
|
||
def wait_for_wakeword(timeout: float = None) -> bool:
|
||
"""Внешняя функция для ожидания wake word."""
|
||
return get_detector().wait_for_wakeword(timeout)
|
||
|
||
|
||
def stop_monitoring():
|
||
"""Внешняя функция для остановки мониторинга."""
|
||
if _detector:
|
||
_detector.stop_monitoring()
|
||
|
||
|
||
def cleanup():
|
||
"""Внешняя функция очистки ресурсов."""
|
||
global _detector
|
||
if _detector:
|
||
_detector.cleanup()
|
||
_detector = None
|
||
|
||
|
||
def check_wakeword_once() -> bool:
|
||
"""Внешняя функция для быстрой проверки."""
|
||
return get_detector().check_wakeword_once()
|