audio_recorder.py 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707
  1. """
  2. The AudioToTextRecorder class in the provided code facilitates
  3. fast speech-to-text transcription.
  4. The class employs the faster_whisper library to transcribe the recorded audio
  5. into text using machine learning models, which can be run either on a GPU or
  6. CPU. Voice activity detection (VAD) is built in, meaning the software can
  7. automatically start or stop recording based on the presence or absence of
  8. speech. It integrates wake word detection through the pvporcupine library,
  9. allowing the software to initiate recording when a specific word or phrase
  10. is spoken. The system provides real-time feedback and can be further
  11. customized.
  12. Features:
  13. - Voice Activity Detection: Automatically starts/stops recording when speech
  14. is detected or when speech ends.
  15. - Wake Word Detection: Starts recording when a specified wake word (or words)
  16. is detected.
  17. - Event Callbacks: Customizable callbacks for when recording starts
  18. or finishes.
  19. - Fast Transcription: Returns the transcribed text from the audio as fast
  20. as possible.
  21. Author: Kolja Beigel
  22. """
  23. from typing import Iterable, List, Optional, Union
  24. import torch.multiprocessing as mp
  25. import torch
  26. from typing import List, Union
  27. from ctypes import c_bool
  28. from scipy.signal import resample
  29. from scipy import signal
  30. import faster_whisper
  31. import collections
  32. import numpy as np
  33. import pvporcupine
  34. import traceback
  35. import threading
  36. import webrtcvad
  37. import itertools
  38. import platform
  39. import pyaudio
  40. import logging
  41. import struct
  42. import halo
  43. import time
  44. import copy
  45. import os
  46. import re
  47. import gc
  48. # Set OpenMP runtime duplicate library handling to OK (Use only for development!)
  49. os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'
  50. INIT_MODEL_TRANSCRIPTION = "tiny"
  51. INIT_MODEL_TRANSCRIPTION_REALTIME = "tiny"
  52. INIT_REALTIME_PROCESSING_PAUSE = 0.2
  53. INIT_SILERO_SENSITIVITY = 0.4
  54. INIT_WEBRTC_SENSITIVITY = 3
  55. INIT_POST_SPEECH_SILENCE_DURATION = 0.6
  56. INIT_MIN_LENGTH_OF_RECORDING = 0.5
  57. INIT_MIN_GAP_BETWEEN_RECORDINGS = 0
  58. INIT_WAKE_WORDS_SENSITIVITY = 0.6
  59. INIT_PRE_RECORDING_BUFFER_DURATION = 1.0
  60. INIT_WAKE_WORD_ACTIVATION_DELAY = 0.0
  61. INIT_WAKE_WORD_TIMEOUT = 5.0
  62. ALLOWED_LATENCY_LIMIT = 10
  63. TIME_SLEEP = 0.02
  64. SAMPLE_RATE = 16000
  65. BUFFER_SIZE = 512
  66. INT16_MAX_ABS_VALUE = 32768.0
  67. INIT_HANDLE_BUFFER_OVERFLOW = False
  68. if platform.system() != 'Darwin':
  69. INIT_HANDLE_BUFFER_OVERFLOW = True
  70. class AudioToTextRecorder:
  71. """
  72. A class responsible for capturing audio from the microphone, detecting
  73. voice activity, and then transcribing the captured audio using the
  74. `faster_whisper` model.
  75. """
  76. def __init__(self,
  77. model: str = INIT_MODEL_TRANSCRIPTION,
  78. language: str = "",
  79. compute_type: str = "default",
  80. input_device_index: int = None,
  81. gpu_device_index: Union[int, List[int]] = 0,
  82. device: str = "cuda",
  83. on_recording_start=None,
  84. on_recording_stop=None,
  85. on_transcription_start=None,
  86. ensure_sentence_starting_uppercase=True,
  87. ensure_sentence_ends_with_period=True,
  88. use_microphone=True,
  89. spinner=True,
  90. level=logging.WARNING,
  91. # Realtime transcription parameters
  92. enable_realtime_transcription=False,
  93. realtime_model_type=INIT_MODEL_TRANSCRIPTION_REALTIME,
  94. realtime_processing_pause=INIT_REALTIME_PROCESSING_PAUSE,
  95. on_realtime_transcription_update=None,
  96. on_realtime_transcription_stabilized=None,
  97. # Voice activation parameters
  98. silero_sensitivity: float = INIT_SILERO_SENSITIVITY,
  99. silero_use_onnx: bool = False,
  100. webrtc_sensitivity: int = INIT_WEBRTC_SENSITIVITY,
  101. post_speech_silence_duration: float = (
  102. INIT_POST_SPEECH_SILENCE_DURATION
  103. ),
  104. min_length_of_recording: float = (
  105. INIT_MIN_LENGTH_OF_RECORDING
  106. ),
  107. min_gap_between_recordings: float = (
  108. INIT_MIN_GAP_BETWEEN_RECORDINGS
  109. ),
  110. pre_recording_buffer_duration: float = (
  111. INIT_PRE_RECORDING_BUFFER_DURATION
  112. ),
  113. on_vad_detect_start=None,
  114. on_vad_detect_stop=None,
  115. # Wake word parameters
  116. wake_words: str = "",
  117. wake_words_sensitivity: float = INIT_WAKE_WORDS_SENSITIVITY,
  118. wake_word_activation_delay: float = (
  119. INIT_WAKE_WORD_ACTIVATION_DELAY
  120. ),
  121. wake_word_timeout: float = INIT_WAKE_WORD_TIMEOUT,
  122. on_wakeword_detected=None,
  123. on_wakeword_timeout=None,
  124. on_wakeword_detection_start=None,
  125. on_wakeword_detection_end=None,
  126. on_recorded_chunk=None,
  127. debug_mode=False,
  128. handle_buffer_overflow: bool = INIT_HANDLE_BUFFER_OVERFLOW,
  129. beam_size: int = 5,
  130. beam_size_realtime: int = 3,
  131. buffer_size: int = BUFFER_SIZE,
  132. sample_rate: int = SAMPLE_RATE,
  133. initial_prompt: Optional[Union[str, Iterable[int]]] = None,
  134. suppress_tokens: Optional[List[int]] = [-1],
  135. ):
  136. """
  137. Initializes an audio recorder and transcription
  138. and wake word detection.
  139. Args:
  140. - model (str, default="tiny"): Specifies the size of the transcription
  141. model to use or the path to a converted model directory.
  142. Valid options are 'tiny', 'tiny.en', 'base', 'base.en',
  143. 'small', 'small.en', 'medium', 'medium.en', 'large-v1',
  144. 'large-v2'.
  145. If a specific size is provided, the model is downloaded
  146. from the Hugging Face Hub.
  147. - language (str, default=""): Language code for speech-to-text engine.
  148. If not specified, the model will attempt to detect the language
  149. automatically.
  150. - compute_type (str, default="default"): Specifies the type of
  151. computation to be used for transcription.
  152. See https://opennmt.net/CTranslate2/quantization.html.
  153. - input_device_index (int, default=0): The index of the audio input
  154. device to use.
  155. - gpu_device_index (int, default=0): Device ID to use.
  156. The model can also be loaded on multiple GPUs by passing a list of
  157. IDs (e.g. [0, 1, 2, 3]). In that case, multiple transcriptions can
  158. run in parallel when transcribe() is called from multiple Python
  159. threads
  160. - device (str, default="cuda"): Device for model to use. Can either be
  161. "cuda" or "cpu".
  162. - on_recording_start (callable, default=None): Callback function to be
  163. called when recording of audio to be transcripted starts.
  164. - on_recording_stop (callable, default=None): Callback function to be
  165. called when recording of audio to be transcripted stops.
  166. - on_transcription_start (callable, default=None): Callback function
  167. to be called when transcription of audio to text starts.
  168. - ensure_sentence_starting_uppercase (bool, default=True): Ensures
  169. that every sentence detected by the algorithm starts with an
  170. uppercase letter.
  171. - ensure_sentence_ends_with_period (bool, default=True): Ensures that
  172. every sentence that doesn't end with punctuation such as "?", "!"
  173. ends with a period
  174. - use_microphone (bool, default=True): Specifies whether to use the
  175. microphone as the audio input source. If set to False, the
  176. audio input source will be the audio data sent through the
  177. feed_audio() method.
  178. - spinner (bool, default=True): Show spinner animation with current
  179. state.
  180. - level (int, default=logging.WARNING): Logging level.
  181. - enable_realtime_transcription (bool, default=False): Enables or
  182. disables real-time transcription of audio. When set to True, the
  183. audio will be transcribed continuously as it is being recorded.
  184. - realtime_model_type (str, default="tiny"): Specifies the machine
  185. learning model to be used for real-time transcription. Valid
  186. options include 'tiny', 'tiny.en', 'base', 'base.en', 'small',
  187. 'small.en', 'medium', 'medium.en', 'large-v1', 'large-v2'.
  188. - realtime_processing_pause (float, default=0.1): Specifies the time
  189. interval in seconds after a chunk of audio gets transcribed. Lower
  190. values will result in more "real-time" (frequent) transcription
  191. updates but may increase computational load.
  192. - on_realtime_transcription_update = A callback function that is
  193. triggered whenever there's an update in the real-time
  194. transcription. The function is called with the newly transcribed
  195. text as its argument.
  196. - on_realtime_transcription_stabilized = A callback function that is
  197. triggered when the transcribed text stabilizes in quality. The
  198. stabilized text is generally more accurate but may arrive with a
  199. slight delay compared to the regular real-time updates.
  200. - silero_sensitivity (float, default=SILERO_SENSITIVITY): Sensitivity
  201. for the Silero Voice Activity Detection model ranging from 0
  202. (least sensitive) to 1 (most sensitive). Default is 0.5.
  203. - silero_use_onnx (bool, default=False): Enables usage of the
  204. pre-trained model from Silero in the ONNX (Open Neural Network
  205. Exchange) format instead of the PyTorch format. This is
  206. recommended for faster performance.
  207. - webrtc_sensitivity (int, default=WEBRTC_SENSITIVITY): Sensitivity
  208. for the WebRTC Voice Activity Detection engine ranging from 0
  209. (least aggressive / most sensitive) to 3 (most aggressive,
  210. least sensitive). Default is 3.
  211. - post_speech_silence_duration (float, default=0.2): Duration in
  212. seconds of silence that must follow speech before the recording
  213. is considered to be completed. This ensures that any brief
  214. pauses during speech don't prematurely end the recording.
  215. - min_gap_between_recordings (float, default=1.0): Specifies the
  216. minimum time interval in seconds that should exist between the
  217. end of one recording session and the beginning of another to
  218. prevent rapid consecutive recordings.
  219. - min_length_of_recording (float, default=1.0): Specifies the minimum
  220. duration in seconds that a recording session should last to ensure
  221. meaningful audio capture, preventing excessively short or
  222. fragmented recordings.
  223. - pre_recording_buffer_duration (float, default=0.2): Duration in
  224. seconds for the audio buffer to maintain pre-roll audio
  225. (compensates speech activity detection latency)
  226. - on_vad_detect_start (callable, default=None): Callback function to
  227. be called when the system listens for voice activity.
  228. - on_vad_detect_stop (callable, default=None): Callback function to be
  229. called when the system stops listening for voice activity.
  230. - wake_words (str, default=""): Comma-separated string of wake words to
  231. initiate recording. Supported wake words include:
  232. 'alexa', 'americano', 'blueberry', 'bumblebee', 'computer',
  233. 'grapefruits', 'grasshopper', 'hey google', 'hey siri', 'jarvis',
  234. 'ok google', 'picovoice', 'porcupine', 'terminator'.
  235. - wake_words_sensitivity (float, default=0.5): Sensitivity for wake
  236. word detection, ranging from 0 (least sensitive) to 1 (most
  237. sensitive). Default is 0.5.
  238. - wake_word_activation_delay (float, default=0): Duration in seconds
  239. after the start of monitoring before the system switches to wake
  240. word activation if no voice is initially detected. If set to
  241. zero, the system uses wake word activation immediately.
  242. - wake_word_timeout (float, default=5): Duration in seconds after a
  243. wake word is recognized. If no subsequent voice activity is
  244. detected within this window, the system transitions back to an
  245. inactive state, awaiting the next wake word or voice activation.
  246. - on_wakeword_detected (callable, default=None): Callback function to
  247. be called when a wake word is detected.
  248. - on_wakeword_timeout (callable, default=None): Callback function to
  249. be called when the system goes back to an inactive state after when
  250. no speech was detected after wake word activation
  251. - on_wakeword_detection_start (callable, default=None): Callback
  252. function to be called when the system starts to listen for wake
  253. words
  254. - on_wakeword_detection_end (callable, default=None): Callback
  255. function to be called when the system stops to listen for
  256. wake words (e.g. because of timeout or wake word detected)
  257. - on_recorded_chunk (callable, default=None): Callback function to be
  258. called when a chunk of audio is recorded. The function is called
  259. with the recorded audio chunk as its argument.
  260. - debug_mode (bool, default=False): If set to True, the system will
  261. print additional debug information to the console.
  262. - handle_buffer_overflow (bool, default=True): If set to True, the system
  263. will log a warning when an input overflow occurs during recording and
  264. remove the data from the buffer.
  265. - beam_size (int, default=5): The beam size to use for beam search
  266. decoding.
  267. - beam_size_realtime (int, default=3): The beam size to use for beam
  268. search decoding in the real-time transcription model.
  269. - buffer_size (int, default=512): The buffer size to use for audio
  270. recording. Changing this may break functionality.
  271. - sample_rate (int, default=16000): The sample rate to use for audio
  272. recording. Changing this will very probably functionality (as the
  273. WebRTC VAD model is very sensitive towards the sample rate).
  274. - initial_prompt (str or iterable of int, default=None): Initial
  275. prompt to be fed to the transcription models.
  276. - suppress_tokens (list of int, default=[-1]): Tokens to be suppressed
  277. from the transcription output.
  278. Raises:
  279. Exception: Errors related to initializing transcription
  280. model, wake word detection, or audio recording.
  281. """
  282. self.language = language
  283. self.compute_type = compute_type
  284. self.input_device_index = input_device_index
  285. self.gpu_device_index = gpu_device_index
  286. self.device = device
  287. self.wake_words = wake_words
  288. self.wake_word_activation_delay = wake_word_activation_delay
  289. self.wake_word_timeout = wake_word_timeout
  290. self.ensure_sentence_starting_uppercase = (
  291. ensure_sentence_starting_uppercase
  292. )
  293. self.ensure_sentence_ends_with_period = (
  294. ensure_sentence_ends_with_period
  295. )
  296. self.use_microphone = mp.Value(c_bool, use_microphone)
  297. self.min_gap_between_recordings = min_gap_between_recordings
  298. self.min_length_of_recording = min_length_of_recording
  299. self.pre_recording_buffer_duration = pre_recording_buffer_duration
  300. self.post_speech_silence_duration = post_speech_silence_duration
  301. self.on_recording_start = on_recording_start
  302. self.on_recording_stop = on_recording_stop
  303. self.on_wakeword_detected = on_wakeword_detected
  304. self.on_wakeword_timeout = on_wakeword_timeout
  305. self.on_vad_detect_start = on_vad_detect_start
  306. self.on_vad_detect_stop = on_vad_detect_stop
  307. self.on_wakeword_detection_start = on_wakeword_detection_start
  308. self.on_wakeword_detection_end = on_wakeword_detection_end
  309. self.on_recorded_chunk = on_recorded_chunk
  310. self.on_transcription_start = on_transcription_start
  311. self.enable_realtime_transcription = enable_realtime_transcription
  312. self.realtime_model_type = realtime_model_type
  313. self.realtime_processing_pause = realtime_processing_pause
  314. self.on_realtime_transcription_update = (
  315. on_realtime_transcription_update
  316. )
  317. self.on_realtime_transcription_stabilized = (
  318. on_realtime_transcription_stabilized
  319. )
  320. self.debug_mode = debug_mode
  321. self.handle_buffer_overflow = handle_buffer_overflow
  322. self.beam_size = beam_size
  323. self.beam_size_realtime = beam_size_realtime
  324. self.allowed_latency_limit = ALLOWED_LATENCY_LIMIT
  325. self.level = level
  326. self.audio_queue = mp.Queue()
  327. self.buffer_size = buffer_size
  328. self.sample_rate = sample_rate
  329. self.recording_start_time = 0
  330. self.recording_stop_time = 0
  331. self.wake_word_detect_time = 0
  332. self.silero_check_time = 0
  333. self.silero_working = False
  334. self.speech_end_silence_start = 0
  335. self.silero_sensitivity = silero_sensitivity
  336. self.listen_start = 0
  337. self.spinner = spinner
  338. self.halo = None
  339. self.state = "inactive"
  340. self.wakeword_detected = False
  341. self.text_storage = []
  342. self.realtime_stabilized_text = ""
  343. self.realtime_stabilized_safetext = ""
  344. self.is_webrtc_speech_active = False
  345. self.is_silero_speech_active = False
  346. self.recording_thread = None
  347. self.realtime_thread = None
  348. self.audio_interface = None
  349. self.audio = None
  350. self.stream = None
  351. self.start_recording_event = threading.Event()
  352. self.stop_recording_event = threading.Event()
  353. self.last_transcription_bytes = None
  354. self.initial_prompt = initial_prompt
  355. self.suppress_tokens = suppress_tokens
  356. # Initialize the logging configuration with the specified level
  357. log_format = 'RealTimeSTT: %(name)s - %(levelname)s - %(message)s'
  358. # Create a logger
  359. logger = logging.getLogger()
  360. logger.setLevel(level) # Set the root logger's level
  361. # Create a file handler and set its level
  362. file_handler = logging.FileHandler('realtimesst.log')
  363. file_handler.setLevel(logging.DEBUG)
  364. file_handler.setFormatter(logging.Formatter(log_format))
  365. # Create a console handler and set its level
  366. console_handler = logging.StreamHandler()
  367. console_handler.setLevel(level)
  368. console_handler.setFormatter(logging.Formatter(log_format))
  369. # Add the handlers to the logger
  370. logger.addHandler(file_handler)
  371. logger.addHandler(console_handler)
  372. self.is_shut_down = False
  373. self.shutdown_event = mp.Event()
  374. try:
  375. logging.debug("Explicitly setting the multiprocessing start method to 'spawn'")
  376. mp.set_start_method('spawn')
  377. except RuntimeError as e:
  378. logging.debug("Start method has already been set. Details:", e)
  379. logging.info("Starting RealTimeSTT")
  380. self.interrupt_stop_event = mp.Event()
  381. self.was_interrupted = mp.Event()
  382. self.main_transcription_ready_event = mp.Event()
  383. self.parent_transcription_pipe, child_transcription_pipe = mp.Pipe()
  384. # Set device for model
  385. self.device = "cuda" if self.device == "cuda" and torch.cuda.is_available() else "cpu"
  386. self.transcript_process = mp.Process(
  387. target=AudioToTextRecorder._transcription_worker,
  388. args=(
  389. child_transcription_pipe,
  390. model,
  391. self.compute_type,
  392. self.gpu_device_index,
  393. self.device,
  394. self.main_transcription_ready_event,
  395. self.shutdown_event,
  396. self.interrupt_stop_event,
  397. self.beam_size,
  398. self.initial_prompt,
  399. self.suppress_tokens
  400. )
  401. )
  402. self.transcript_process.start()
  403. # Start audio data reading process
  404. if self.use_microphone.value:
  405. logging.info("Initializing audio recording"
  406. " (creating pyAudio input stream,"
  407. f" sample rate: {self.sample_rate}"
  408. f" buffer size: {self.buffer_size}"
  409. )
  410. self.reader_process = mp.Process(
  411. target=AudioToTextRecorder._audio_data_worker,
  412. args=(
  413. self.audio_queue,
  414. self.sample_rate,
  415. self.buffer_size,
  416. self.input_device_index,
  417. self.shutdown_event,
  418. self.interrupt_stop_event,
  419. self.use_microphone
  420. )
  421. )
  422. self.reader_process.start()
  423. # Initialize the realtime transcription model
  424. if self.enable_realtime_transcription:
  425. try:
  426. logging.info("Initializing faster_whisper realtime "
  427. f"transcription model {self.realtime_model_type}"
  428. )
  429. self.realtime_model_type = faster_whisper.WhisperModel(
  430. model_size_or_path=self.realtime_model_type,
  431. device=self.device,
  432. compute_type=self.compute_type,
  433. device_index=self.gpu_device_index
  434. )
  435. except Exception as e:
  436. logging.exception("Error initializing faster_whisper "
  437. f"realtime transcription model: {e}"
  438. )
  439. raise
  440. logging.debug("Faster_whisper realtime speech to text "
  441. "transcription model initialized successfully")
  442. # Setup wake word detection
  443. if wake_words:
  444. self.wake_words_list = [
  445. word.strip() for word in wake_words.lower().split(',')
  446. ]
  447. sensitivity_list = [
  448. float(wake_words_sensitivity)
  449. for _ in range(len(self.wake_words_list))
  450. ]
  451. try:
  452. self.porcupine = pvporcupine.create(
  453. keywords=self.wake_words_list,
  454. sensitivities=sensitivity_list
  455. )
  456. self.buffer_size = self.porcupine.frame_length
  457. self.sample_rate = self.porcupine.sample_rate
  458. except Exception as e:
  459. logging.exception("Error initializing porcupine "
  460. f"wake word detection engine: {e}"
  461. )
  462. raise
  463. logging.debug("Porcupine wake word detection "
  464. "engine initialized successfully"
  465. )
  466. # Setup voice activity detection model WebRTC
  467. try:
  468. logging.info("Initializing WebRTC voice with "
  469. f"Sensitivity {webrtc_sensitivity}"
  470. )
  471. self.webrtc_vad_model = webrtcvad.Vad()
  472. self.webrtc_vad_model.set_mode(webrtc_sensitivity)
  473. except Exception as e:
  474. logging.exception("Error initializing WebRTC voice "
  475. f"activity detection engine: {e}"
  476. )
  477. raise
  478. logging.debug("WebRTC VAD voice activity detection "
  479. "engine initialized successfully"
  480. )
  481. # Setup voice activity detection model Silero VAD
  482. try:
  483. self.silero_vad_model, _ = torch.hub.load(
  484. repo_or_dir="snakers4/silero-vad",
  485. model="silero_vad",
  486. verbose=False,
  487. onnx=silero_use_onnx
  488. )
  489. except Exception as e:
  490. logging.exception(f"Error initializing Silero VAD "
  491. f"voice activity detection engine: {e}"
  492. )
  493. raise
  494. logging.debug("Silero VAD voice activity detection "
  495. "engine initialized successfully"
  496. )
  497. self.audio_buffer = collections.deque(
  498. maxlen=int((self.sample_rate // self.buffer_size) *
  499. self.pre_recording_buffer_duration)
  500. )
  501. self.frames = []
  502. # Recording control flags
  503. self.is_recording = False
  504. self.is_running = True
  505. self.start_recording_on_voice_activity = False
  506. self.stop_recording_on_voice_deactivity = False
  507. # Start the recording worker thread
  508. self.recording_thread = threading.Thread(target=self._recording_worker)
  509. self.recording_thread.daemon = True
  510. self.recording_thread.start()
  511. # Start the realtime transcription worker thread
  512. self.realtime_thread = threading.Thread(target=self._realtime_worker)
  513. self.realtime_thread.daemon = True
  514. self.realtime_thread.start()
  515. # Wait for transcription models to start
  516. logging.debug('Waiting for main transcription model to start')
  517. self.main_transcription_ready_event.wait()
  518. logging.debug('Main transcription model ready')
  519. logging.debug('RealtimeSTT initialization completed successfully')
  520. @staticmethod
  521. def _transcription_worker(conn,
  522. model_path,
  523. compute_type,
  524. gpu_device_index,
  525. device,
  526. ready_event,
  527. shutdown_event,
  528. interrupt_stop_event,
  529. beam_size,
  530. initial_prompt,
  531. suppress_tokens
  532. ):
  533. """
  534. Worker method that handles the continuous
  535. process of transcribing audio data.
  536. This method runs in a separate process and is responsible for:
  537. - Initializing the `faster_whisper` model used for transcription.
  538. - Receiving audio data sent through a pipe and using the model
  539. to transcribe it.
  540. - Sending transcription results back through the pipe.
  541. - Continuously checking for a shutdown event to gracefully
  542. terminate the transcription process.
  543. Args:
  544. conn (multiprocessing.Connection): The connection endpoint used
  545. for receiving audio data and sending transcription results.
  546. model_path (str): The path to the pre-trained faster_whisper model
  547. for transcription.
  548. compute_type (str): Specifies the type of computation to be used
  549. for transcription.
  550. gpu_device_index (int): Device ID to use.
  551. device (str): Device for model to use.
  552. ready_event (threading.Event): An event that is set when the
  553. transcription model is successfully initialized and ready.
  554. shutdown_event (threading.Event): An event that, when set,
  555. signals this worker method to terminate.
  556. interrupt_stop_event (threading.Event): An event that, when set,
  557. signals this worker method to stop processing audio data.
  558. beam_size (int): The beam size to use for beam search decoding.
  559. initial_prompt (str or iterable of int): Initial prompt to be fed
  560. to the transcription model.
  561. suppress_tokens (list of int): Tokens to be suppressed from the
  562. transcription output.
  563. Raises:
  564. Exception: If there is an error while initializing the
  565. transcription model.
  566. """
  567. logging.info("Initializing faster_whisper "
  568. f"main transcription model {model_path}"
  569. )
  570. try:
  571. model = faster_whisper.WhisperModel(
  572. model_size_or_path=model_path,
  573. device=device,
  574. compute_type=compute_type,
  575. device_index=gpu_device_index,
  576. )
  577. except Exception as e:
  578. logging.exception("Error initializing main "
  579. f"faster_whisper transcription model: {e}"
  580. )
  581. raise
  582. ready_event.set()
  583. logging.debug("Faster_whisper main speech to text "
  584. "transcription model initialized successfully"
  585. )
  586. while not shutdown_event.is_set():
  587. try:
  588. if conn.poll(0.5):
  589. audio, language = conn.recv()
  590. try:
  591. segments = model.transcribe(
  592. audio,
  593. language=language if language else None,
  594. beam_size=beam_size,
  595. initial_prompt=initial_prompt,
  596. suppress_tokens=suppress_tokens
  597. )
  598. segments = segments[0]
  599. transcription = " ".join(seg.text for seg in segments)
  600. transcription = transcription.strip()
  601. conn.send(('success', transcription))
  602. except Exception as e:
  603. logging.error(f"General transcription error: {e}")
  604. conn.send(('error', str(e)))
  605. else:
  606. # If there's no data, sleep / prevent busy waiting
  607. time.sleep(0.02)
  608. except KeyboardInterrupt:
  609. interrupt_stop_event.set()
  610. logging.debug("Transcription worker process "
  611. "finished due to KeyboardInterrupt"
  612. )
  613. break
  614. @staticmethod
  615. def _audio_data_worker(audio_queue,
  616. sample_rate,
  617. buffer_size,
  618. input_device_index,
  619. shutdown_event,
  620. interrupt_stop_event,
  621. use_microphone):
  622. """
  623. Worker method that handles the audio recording process.
  624. This method runs in a separate process and is responsible for:
  625. - Setting up the audio input stream for recording.
  626. - Continuously reading audio data from the input stream
  627. and placing it in a queue.
  628. - Handling errors during the recording process, including
  629. input overflow.
  630. - Gracefully terminating the recording process when a shutdown
  631. event is set.
  632. Args:
  633. audio_queue (queue.Queue): A queue where recorded audio
  634. data is placed.
  635. sample_rate (int): The sample rate of the audio input stream.
  636. buffer_size (int): The size of the buffer used in the audio
  637. input stream.
  638. input_device_index (int): The index of the audio input device
  639. shutdown_event (threading.Event): An event that, when set, signals
  640. this worker method to terminate.
  641. Raises:
  642. Exception: If there is an error while initializing the audio
  643. recording.
  644. """
  645. try:
  646. audio_interface = pyaudio.PyAudio()
  647. if input_device_index is None:
  648. default_device = audio_interface.get_default_input_device_info()
  649. input_device_index = default_device['index']
  650. stream = audio_interface.open(
  651. rate=sample_rate,
  652. format=pyaudio.paInt16,
  653. channels=1,
  654. input=True,
  655. frames_per_buffer=buffer_size,
  656. input_device_index=input_device_index,
  657. )
  658. except Exception as e:
  659. logging.exception("Error initializing pyaudio "
  660. f"audio recording: {e}"
  661. )
  662. raise
  663. logging.debug("Audio recording (pyAudio input "
  664. "stream) initialized successfully"
  665. )
  666. try:
  667. while not shutdown_event.is_set():
  668. try:
  669. data = stream.read(buffer_size)
  670. except OSError as e:
  671. if e.errno == pyaudio.paInputOverflowed:
  672. logging.warning("Input overflowed. Frame dropped.")
  673. else:
  674. logging.error(f"Error during recording: {e}")
  675. tb_str = traceback.format_exc()
  676. print(f"Traceback: {tb_str}")
  677. print(f"Error: {e}")
  678. continue
  679. except Exception as e:
  680. logging.error(f"Error during recording: {e}")
  681. tb_str = traceback.format_exc()
  682. print(f"Traceback: {tb_str}")
  683. print(f"Error: {e}")
  684. continue
  685. if use_microphone.value:
  686. audio_queue.put(data)
  687. except KeyboardInterrupt:
  688. interrupt_stop_event.set()
  689. logging.debug("Audio data worker process "
  690. "finished due to KeyboardInterrupt"
  691. )
  692. finally:
  693. stream.stop_stream()
  694. stream.close()
  695. audio_interface.terminate()
  696. def wakeup(self):
  697. """
  698. If in wake work modus, wake up as if a wake word was spoken.
  699. """
  700. self.listen_start = time.time()
  701. def abort(self):
  702. self.start_recording_on_voice_activity = False
  703. self.stop_recording_on_voice_deactivity = False
  704. self._set_state("inactive")
  705. self.interrupt_stop_event.set()
  706. self.was_interrupted.wait()
  707. self.was_interrupted.clear()
  708. def wait_audio(self):
  709. """
  710. Waits for the start and completion of the audio recording process.
  711. This method is responsible for:
  712. - Waiting for voice activity to begin recording if not yet started.
  713. - Waiting for voice inactivity to complete the recording.
  714. - Setting the audio buffer from the recorded frames.
  715. - Resetting recording-related attributes.
  716. Side effects:
  717. - Updates the state of the instance.
  718. - Modifies the audio attribute to contain the processed audio data.
  719. """
  720. self.listen_start = time.time()
  721. # If not yet started recording, wait for voice activity to initiate.
  722. if not self.is_recording and not self.frames:
  723. self._set_state("listening")
  724. self.start_recording_on_voice_activity = True
  725. # Wait until recording starts
  726. while not self.interrupt_stop_event.is_set():
  727. if self.start_recording_event.wait(timeout=0.02):
  728. break
  729. # If recording is ongoing, wait for voice inactivity
  730. # to finish recording.
  731. if self.is_recording:
  732. self.stop_recording_on_voice_deactivity = True
  733. # Wait until recording stops
  734. while not self.interrupt_stop_event.is_set():
  735. if (self.stop_recording_event.wait(timeout=0.02)):
  736. break
  737. # Convert recorded frames to the appropriate audio format.
  738. audio_array = np.frombuffer(b''.join(self.frames), dtype=np.int16)
  739. self.audio = audio_array.astype(np.float32) / INT16_MAX_ABS_VALUE
  740. self.frames.clear()
  741. # Reset recording-related timestamps
  742. self.recording_stop_time = 0
  743. self.listen_start = 0
  744. self._set_state("inactive")
  745. def transcribe(self):
  746. """
  747. Transcribes audio captured by this class instance using the
  748. `faster_whisper` model.
  749. Automatically starts recording upon voice activity if not manually
  750. started using `recorder.start()`.
  751. Automatically stops recording upon voice deactivity if not manually
  752. stopped with `recorder.stop()`.
  753. Processes the recorded audio to generate transcription.
  754. Args:
  755. on_transcription_finished (callable, optional): Callback function
  756. to be executed when transcription is ready.
  757. If provided, transcription will be performed asynchronously,
  758. and the callback will receive the transcription as its argument.
  759. If omitted, the transcription will be performed synchronously,
  760. and the result will be returned.
  761. Returns (if no callback is set):
  762. str: The transcription of the recorded audio.
  763. Raises:
  764. Exception: If there is an error during the transcription process.
  765. """
  766. self._set_state("transcribing")
  767. audio_copy = copy.deepcopy(self.audio)
  768. self.parent_transcription_pipe.send((self.audio, self.language))
  769. status, result = self.parent_transcription_pipe.recv()
  770. self._set_state("inactive")
  771. if status == 'success':
  772. self.last_transcription_bytes = audio_copy
  773. return self._preprocess_output(result)
  774. else:
  775. logging.error(result)
  776. raise Exception(result)
  777. def text(self,
  778. on_transcription_finished=None,
  779. ):
  780. """
  781. Transcribes audio captured by this class instance
  782. using the `faster_whisper` model.
  783. - Automatically starts recording upon voice activity if not manually
  784. started using `recorder.start()`.
  785. - Automatically stops recording upon voice deactivity if not manually
  786. stopped with `recorder.stop()`.
  787. - Processes the recorded audio to generate transcription.
  788. Args:
  789. on_transcription_finished (callable, optional): Callback function
  790. to be executed when transcription is ready.
  791. If provided, transcription will be performed asynchronously, and
  792. the callback will receive the transcription as its argument.
  793. If omitted, the transcription will be performed synchronously,
  794. and the result will be returned.
  795. Returns (if not callback is set):
  796. str: The transcription of the recorded audio
  797. """
  798. self.interrupt_stop_event.clear()
  799. self.was_interrupted.clear()
  800. self.wait_audio()
  801. if self.is_shut_down or self.interrupt_stop_event.is_set():
  802. if self.interrupt_stop_event.is_set():
  803. self.was_interrupted.set()
  804. return ""
  805. if on_transcription_finished:
  806. threading.Thread(target=on_transcription_finished,
  807. args=(self.transcribe(),)).start()
  808. else:
  809. return self.transcribe()
  810. def start(self):
  811. """
  812. Starts recording audio directly without waiting for voice activity.
  813. """
  814. # Ensure there's a minimum interval
  815. # between stopping and starting recording
  816. if (time.time() - self.recording_stop_time
  817. < self.min_gap_between_recordings):
  818. logging.info("Attempted to start recording "
  819. "too soon after stopping."
  820. )
  821. return self
  822. logging.info("recording started")
  823. self._set_state("recording")
  824. self.text_storage = []
  825. self.realtime_stabilized_text = ""
  826. self.realtime_stabilized_safetext = ""
  827. self.wakeword_detected = False
  828. self.wake_word_detect_time = 0
  829. self.frames = []
  830. self.is_recording = True
  831. self.recording_start_time = time.time()
  832. self.is_silero_speech_active = False
  833. self.is_webrtc_speech_active = False
  834. self.stop_recording_event.clear()
  835. self.start_recording_event.set()
  836. if self.on_recording_start:
  837. self.on_recording_start()
  838. return self
  839. def stop(self):
  840. """
  841. Stops recording audio.
  842. """
  843. # Ensure there's a minimum interval
  844. # between starting and stopping recording
  845. if (time.time() - self.recording_start_time
  846. < self.min_length_of_recording):
  847. logging.info("Attempted to stop recording "
  848. "too soon after starting."
  849. )
  850. return self
  851. logging.info("recording stopped")
  852. self.is_recording = False
  853. self.recording_stop_time = time.time()
  854. self.is_silero_speech_active = False
  855. self.is_webrtc_speech_active = False
  856. self.silero_check_time = 0
  857. self.start_recording_event.clear()
  858. self.stop_recording_event.set()
  859. if self.on_recording_stop:
  860. self.on_recording_stop()
  861. return self
  862. def feed_audio(self, chunk, original_sample_rate=16000):
  863. """
  864. Feed an audio chunk into the processing pipeline. Chunks are
  865. accumulated until the buffer size is reached, and then the accumulated
  866. data is fed into the audio_queue.
  867. """
  868. # Check if the buffer attribute exists, if not, initialize it
  869. if not hasattr(self, 'buffer'):
  870. self.buffer = bytearray()
  871. # Check if input is a NumPy array
  872. if isinstance(chunk, np.ndarray):
  873. # Handle stereo to mono conversion if necessary
  874. if chunk.ndim == 2:
  875. chunk = np.mean(chunk, axis=1)
  876. # Resample to 16000 Hz if necessary
  877. if original_sample_rate != 16000:
  878. num_samples = int(len(chunk) * 16000 / original_sample_rate)
  879. chunk = resample(chunk, num_samples)
  880. # Ensure data type is int16
  881. chunk = chunk.astype(np.int16)
  882. # Convert the NumPy array to bytes
  883. chunk = chunk.tobytes()
  884. # Append the chunk to the buffer
  885. self.buffer += chunk
  886. buf_size = 2 * self.buffer_size # silero complains if too short
  887. # Check if the buffer has reached or exceeded the buffer_size
  888. while len(self.buffer) >= buf_size:
  889. # Extract self.buffer_size amount of data from the buffer
  890. to_process = self.buffer[:buf_size]
  891. self.buffer = self.buffer[buf_size:]
  892. # Feed the extracted data to the audio_queue
  893. self.audio_queue.put(to_process)
  894. def set_microphone(self, microphone_on=True):
  895. """
  896. Set the microphone on or off.
  897. """
  898. logging.info("Setting microphone to: " + str(microphone_on))
  899. self.use_microphone.value = microphone_on
  900. def shutdown(self):
  901. """
  902. Safely shuts down the audio recording by stopping the
  903. recording worker and closing the audio stream.
  904. """
  905. # Force wait_audio() and text() to exit
  906. self.is_shut_down = True
  907. self.start_recording_event.set()
  908. self.stop_recording_event.set()
  909. self.shutdown_event.set()
  910. self.is_recording = False
  911. self.is_running = False
  912. logging.debug('Finishing recording thread')
  913. if self.recording_thread:
  914. self.recording_thread.join()
  915. logging.debug('Terminating reader process')
  916. # Give it some time to finish the loop and cleanup.
  917. if self.use_microphone:
  918. self.reader_process.join(timeout=10)
  919. if self.reader_process.is_alive():
  920. logging.warning("Reader process did not terminate "
  921. "in time. Terminating forcefully."
  922. )
  923. self.reader_process.terminate()
  924. logging.debug('Terminating transcription process')
  925. self.transcript_process.join(timeout=10)
  926. if self.transcript_process.is_alive():
  927. logging.warning("Transcript process did not terminate "
  928. "in time. Terminating forcefully."
  929. )
  930. self.transcript_process.terminate()
  931. self.parent_transcription_pipe.close()
  932. logging.debug('Finishing realtime thread')
  933. if self.realtime_thread:
  934. self.realtime_thread.join()
  935. if self.enable_realtime_transcription:
  936. if self.realtime_model_type:
  937. del self.realtime_model_type
  938. self.realtime_model_type = None
  939. gc.collect()
  940. def _recording_worker(self):
  941. """
  942. The main worker method which constantly monitors the audio
  943. input for voice activity and accordingly starts/stops the recording.
  944. """
  945. logging.debug('Starting recording worker')
  946. try:
  947. was_recording = False
  948. delay_was_passed = False
  949. # Continuously monitor audio for voice activity
  950. while self.is_running:
  951. try:
  952. data = self.audio_queue.get()
  953. if self.on_recorded_chunk:
  954. self.on_recorded_chunk(data)
  955. if self.handle_buffer_overflow:
  956. # Handle queue overflow
  957. if (self.audio_queue.qsize() >
  958. self.allowed_latency_limit):
  959. logging.warning("Audio queue size exceeds "
  960. "latency limit. Current size: "
  961. f"{self.audio_queue.qsize()}. "
  962. "Discarding old audio chunks."
  963. )
  964. while (self.audio_queue.qsize() >
  965. self.allowed_latency_limit):
  966. data = self.audio_queue.get()
  967. except BrokenPipeError:
  968. print("BrokenPipeError _recording_worker")
  969. self.is_running = False
  970. break
  971. if not self.is_recording:
  972. # Handle not recording state
  973. time_since_listen_start = (time.time() - self.listen_start
  974. if self.listen_start else 0)
  975. wake_word_activation_delay_passed = (
  976. time_since_listen_start >
  977. self.wake_word_activation_delay
  978. )
  979. # Handle wake-word timeout callback
  980. if wake_word_activation_delay_passed \
  981. and not delay_was_passed:
  982. if self.wake_words and self.wake_word_activation_delay:
  983. if self.on_wakeword_timeout:
  984. self.on_wakeword_timeout()
  985. delay_was_passed = wake_word_activation_delay_passed
  986. # Set state and spinner text
  987. if not self.recording_stop_time:
  988. if self.wake_words \
  989. and wake_word_activation_delay_passed \
  990. and not self.wakeword_detected:
  991. self._set_state("wakeword")
  992. else:
  993. if self.listen_start:
  994. self._set_state("listening")
  995. else:
  996. self._set_state("inactive")
  997. # Detect wake words if applicable
  998. if self.wake_words and wake_word_activation_delay_passed:
  999. try:
  1000. pcm = struct.unpack_from(
  1001. "h" * self.buffer_size,
  1002. data
  1003. )
  1004. wakeword_index = self.porcupine.process(pcm)
  1005. except struct.error:
  1006. logging.error("Error unpacking audio data "
  1007. "for wake word processing.")
  1008. continue
  1009. except Exception as e:
  1010. logging.error(f"Wake word processing error: {e}")
  1011. continue
  1012. # If a wake word is detected
  1013. if wakeword_index >= 0:
  1014. # Removing the wake word from the recording
  1015. samples_for_0_1_sec = int(self.sample_rate * 0.1)
  1016. start_index = max(
  1017. 0,
  1018. len(self.audio_buffer) - samples_for_0_1_sec
  1019. )
  1020. temp_samples = collections.deque(
  1021. itertools.islice(
  1022. self.audio_buffer,
  1023. start_index,
  1024. None)
  1025. )
  1026. self.audio_buffer.clear()
  1027. self.audio_buffer.extend(temp_samples)
  1028. self.wake_word_detect_time = time.time()
  1029. self.wakeword_detected = True
  1030. if self.on_wakeword_detected:
  1031. self.on_wakeword_detected()
  1032. # Check for voice activity to
  1033. # trigger the start of recording
  1034. if ((not self.wake_words
  1035. or not wake_word_activation_delay_passed)
  1036. and self.start_recording_on_voice_activity) \
  1037. or self.wakeword_detected:
  1038. if self._is_voice_active():
  1039. logging.info("voice activity detected")
  1040. self.start()
  1041. if self.is_recording:
  1042. self.start_recording_on_voice_activity = False
  1043. # Add the buffered audio
  1044. # to the recording frames
  1045. self.frames.extend(list(self.audio_buffer))
  1046. self.audio_buffer.clear()
  1047. self.silero_vad_model.reset_states()
  1048. else:
  1049. data_copy = data[:]
  1050. self._check_voice_activity(data_copy)
  1051. self.speech_end_silence_start = 0
  1052. else:
  1053. # If we are currently recording
  1054. # Stop the recording if silence is detected after speech
  1055. if self.stop_recording_on_voice_deactivity:
  1056. if not self._is_webrtc_speech(data, True):
  1057. # Voice deactivity was detected, so we start
  1058. # measuring silence time before stopping recording
  1059. if self.speech_end_silence_start == 0:
  1060. self.speech_end_silence_start = time.time()
  1061. else:
  1062. self.speech_end_silence_start = 0
  1063. # Wait for silence to stop recording after speech
  1064. if self.speech_end_silence_start and time.time() - \
  1065. self.speech_end_silence_start > \
  1066. self.post_speech_silence_duration:
  1067. logging.info("voice deactivity detected")
  1068. self.stop()
  1069. if not self.is_recording and was_recording:
  1070. # Reset after stopping recording to ensure clean state
  1071. self.stop_recording_on_voice_deactivity = False
  1072. if time.time() - self.silero_check_time > 0.1:
  1073. self.silero_check_time = 0
  1074. # Handle wake word timeout (waited to long initiating
  1075. # speech after wake word detection)
  1076. if self.wake_word_detect_time and time.time() - \
  1077. self.wake_word_detect_time > self.wake_word_timeout:
  1078. self.wake_word_detect_time = 0
  1079. if self.wakeword_detected and self.on_wakeword_timeout:
  1080. self.on_wakeword_timeout()
  1081. self.wakeword_detected = False
  1082. was_recording = self.is_recording
  1083. if self.is_recording:
  1084. self.frames.append(data)
  1085. if not self.is_recording or self.speech_end_silence_start:
  1086. self.audio_buffer.append(data)
  1087. except Exception as e:
  1088. if not self.interrupt_stop_event.is_set():
  1089. logging.error(f"Unhandled exeption in _recording_worker: {e}")
  1090. raise
  1091. def _realtime_worker(self):
  1092. """
  1093. Performs real-time transcription if the feature is enabled.
  1094. The method is responsible transcribing recorded audio frames
  1095. in real-time based on the specified resolution interval.
  1096. The transcribed text is stored in `self.realtime_transcription_text`
  1097. and a callback
  1098. function is invoked with this text if specified.
  1099. """
  1100. try:
  1101. logging.debug('Starting realtime worker')
  1102. # Return immediately if real-time transcription is not enabled
  1103. if not self.enable_realtime_transcription:
  1104. return
  1105. # Continue running as long as the main process is active
  1106. while self.is_running:
  1107. # Check if the recording is active
  1108. if self.is_recording:
  1109. # Sleep for the duration of the transcription resolution
  1110. time.sleep(self.realtime_processing_pause)
  1111. # Convert the buffer frames to a NumPy array
  1112. audio_array = np.frombuffer(
  1113. b''.join(self.frames),
  1114. dtype=np.int16
  1115. )
  1116. # Normalize the array to a [-1, 1] range
  1117. audio_array = audio_array.astype(np.float32) / \
  1118. INT16_MAX_ABS_VALUE
  1119. # Perform transcription and assemble the text
  1120. segments = self.realtime_model_type.transcribe(
  1121. audio_array,
  1122. language=self.language if self.language else None,
  1123. beam_size=self.beam_size_realtime,
  1124. initial_prompt=self.initial_prompt,
  1125. suppress_tokens=self.suppress_tokens,
  1126. )
  1127. # double check recording state
  1128. # because it could have changed mid-transcription
  1129. if self.is_recording and time.time() - \
  1130. self.recording_start_time > 0.5:
  1131. logging.debug('Starting realtime transcription')
  1132. self.realtime_transcription_text = " ".join(
  1133. seg.text for seg in segments[0]
  1134. )
  1135. self.realtime_transcription_text = \
  1136. self.realtime_transcription_text.strip()
  1137. self.text_storage.append(
  1138. self.realtime_transcription_text
  1139. )
  1140. # Take the last two texts in storage, if they exist
  1141. if len(self.text_storage) >= 2:
  1142. last_two_texts = self.text_storage[-2:]
  1143. # Find the longest common prefix
  1144. # between the two texts
  1145. prefix = os.path.commonprefix(
  1146. [last_two_texts[0], last_two_texts[1]]
  1147. )
  1148. # This prefix is the text that was transcripted
  1149. # two times in the same way
  1150. # Store as "safely detected text"
  1151. if len(prefix) >= \
  1152. len(self.realtime_stabilized_safetext):
  1153. # Only store when longer than the previous
  1154. # as additional security
  1155. self.realtime_stabilized_safetext = prefix
  1156. # Find parts of the stabilized text
  1157. # in the freshly transcripted text
  1158. matching_pos = self._find_tail_match_in_text(
  1159. self.realtime_stabilized_safetext,
  1160. self.realtime_transcription_text
  1161. )
  1162. if matching_pos < 0:
  1163. if self.realtime_stabilized_safetext:
  1164. self._on_realtime_transcription_stabilized(
  1165. self._preprocess_output(
  1166. self.realtime_stabilized_safetext,
  1167. True
  1168. )
  1169. )
  1170. else:
  1171. self._on_realtime_transcription_stabilized(
  1172. self._preprocess_output(
  1173. self.realtime_transcription_text,
  1174. True
  1175. )
  1176. )
  1177. else:
  1178. # We found parts of the stabilized text
  1179. # in the transcripted text
  1180. # We now take the stabilized text
  1181. # and add only the freshly transcripted part to it
  1182. output_text = self.realtime_stabilized_safetext + \
  1183. self.realtime_transcription_text[matching_pos:]
  1184. # This yields us the "left" text part as stabilized
  1185. # AND at the same time delivers fresh detected
  1186. # parts on the first run without the need for
  1187. # two transcriptions
  1188. self._on_realtime_transcription_stabilized(
  1189. self._preprocess_output(output_text, True)
  1190. )
  1191. # Invoke the callback with the transcribed text
  1192. self._on_realtime_transcription_update(
  1193. self._preprocess_output(
  1194. self.realtime_transcription_text,
  1195. True
  1196. )
  1197. )
  1198. # If not recording, sleep briefly before checking again
  1199. else:
  1200. time.sleep(TIME_SLEEP)
  1201. except Exception as e:
  1202. logging.error(f"Unhandled exeption in _realtime_worker: {e}")
  1203. raise
  1204. def _is_silero_speech(self, chunk):
  1205. """
  1206. Returns true if speech is detected in the provided audio data
  1207. Args:
  1208. data (bytes): raw bytes of audio data (1024 raw bytes with
  1209. 16000 sample rate and 16 bits per sample)
  1210. """
  1211. if self.sample_rate != 16000:
  1212. pcm_data = np.frombuffer(chunk, dtype=np.int16)
  1213. data_16000 = signal.resample_poly(
  1214. pcm_data, 16000, self.sample_rate)
  1215. chunk = data_16000.astype(np.int16).tobytes()
  1216. self.silero_working = True
  1217. audio_chunk = np.frombuffer(chunk, dtype=np.int16)
  1218. audio_chunk = audio_chunk.astype(np.float32) / INT16_MAX_ABS_VALUE
  1219. vad_prob = self.silero_vad_model(
  1220. torch.from_numpy(audio_chunk),
  1221. SAMPLE_RATE).item()
  1222. is_silero_speech_active = vad_prob > (1 - self.silero_sensitivity)
  1223. if is_silero_speech_active:
  1224. self.is_silero_speech_active = True
  1225. self.silero_working = False
  1226. return is_silero_speech_active
  1227. def _is_webrtc_speech(self, chunk, all_frames_must_be_true=False):
  1228. """
  1229. Returns true if speech is detected in the provided audio data
  1230. Args:
  1231. data (bytes): raw bytes of audio data (1024 raw bytes with
  1232. 16000 sample rate and 16 bits per sample)
  1233. """
  1234. if self.sample_rate != 16000:
  1235. pcm_data = np.frombuffer(chunk, dtype=np.int16)
  1236. data_16000 = signal.resample_poly(
  1237. pcm_data, 16000, self.sample_rate)
  1238. chunk = data_16000.astype(np.int16).tobytes()
  1239. # Number of audio frames per millisecond
  1240. frame_length = int(16000 * 0.01) # for 10ms frame
  1241. num_frames = int(len(chunk) / (2 * frame_length))
  1242. speech_frames = 0
  1243. for i in range(num_frames):
  1244. start_byte = i * frame_length * 2
  1245. end_byte = start_byte + frame_length * 2
  1246. frame = chunk[start_byte:end_byte]
  1247. if self.webrtc_vad_model.is_speech(frame, 16000):
  1248. speech_frames += 1
  1249. if not all_frames_must_be_true:
  1250. if self.debug_mode:
  1251. print(f"Speech detected in frame {i + 1}"
  1252. f" of {num_frames}")
  1253. return True
  1254. if all_frames_must_be_true:
  1255. if self.debug_mode and speech_frames == num_frames:
  1256. print(f"Speech detected in {speech_frames} of "
  1257. f"{num_frames} frames")
  1258. elif self.debug_mode:
  1259. print(f"Speech not detected in all {num_frames} frames")
  1260. return speech_frames == num_frames
  1261. else:
  1262. if self.debug_mode:
  1263. print(f"Speech not detected in any of {num_frames} frames")
  1264. return False
  1265. def _check_voice_activity(self, data):
  1266. """
  1267. Initiate check if voice is active based on the provided data.
  1268. Args:
  1269. data: The audio data to be checked for voice activity.
  1270. """
  1271. self.is_webrtc_speech_active = self._is_webrtc_speech(data)
  1272. # First quick performing check for voice activity using WebRTC
  1273. if self.is_webrtc_speech_active:
  1274. if not self.silero_working:
  1275. self.silero_working = True
  1276. # Run the intensive check in a separate thread
  1277. threading.Thread(
  1278. target=self._is_silero_speech,
  1279. args=(data,)).start()
  1280. def _is_voice_active(self):
  1281. """
  1282. Determine if voice is active.
  1283. Returns:
  1284. bool: True if voice is active, False otherwise.
  1285. """
  1286. return self.is_webrtc_speech_active and self.is_silero_speech_active
  1287. def _set_state(self, new_state):
  1288. """
  1289. Update the current state of the recorder and execute
  1290. corresponding state-change callbacks.
  1291. Args:
  1292. new_state (str): The new state to set.
  1293. """
  1294. # Check if the state has actually changed
  1295. if new_state == self.state:
  1296. return
  1297. # Store the current state for later comparison
  1298. old_state = self.state
  1299. # Update to the new state
  1300. self.state = new_state
  1301. # Execute callbacks based on transitioning FROM a particular state
  1302. if old_state == "listening":
  1303. if self.on_vad_detect_stop:
  1304. self.on_vad_detect_stop()
  1305. elif old_state == "wakeword":
  1306. if self.on_wakeword_detection_end:
  1307. self.on_wakeword_detection_end()
  1308. # Execute callbacks based on transitioning TO a particular state
  1309. if new_state == "listening":
  1310. if self.on_vad_detect_start:
  1311. self.on_vad_detect_start()
  1312. self._set_spinner("speak now")
  1313. if self.spinner and self.halo:
  1314. self.halo._interval = 250
  1315. elif new_state == "wakeword":
  1316. if self.on_wakeword_detection_start:
  1317. self.on_wakeword_detection_start()
  1318. self._set_spinner(f"say {self.wake_words}")
  1319. if self.spinner and self.halo:
  1320. self.halo._interval = 500
  1321. elif new_state == "transcribing":
  1322. if self.on_transcription_start:
  1323. self.on_transcription_start()
  1324. self._set_spinner("transcribing")
  1325. if self.spinner and self.halo:
  1326. self.halo._interval = 50
  1327. elif new_state == "recording":
  1328. self._set_spinner("recording")
  1329. if self.spinner and self.halo:
  1330. self.halo._interval = 100
  1331. elif new_state == "inactive":
  1332. if self.spinner and self.halo:
  1333. self.halo.stop()
  1334. self.halo = None
  1335. def _set_spinner(self, text):
  1336. """
  1337. Update the spinner's text or create a new
  1338. spinner with the provided text.
  1339. Args:
  1340. text (str): The text to be displayed alongside the spinner.
  1341. """
  1342. if self.spinner:
  1343. # If the Halo spinner doesn't exist, create and start it
  1344. if self.halo is None:
  1345. self.halo = halo.Halo(text=text)
  1346. self.halo.start()
  1347. # If the Halo spinner already exists, just update the text
  1348. else:
  1349. self.halo.text = text
  1350. def _preprocess_output(self, text, preview=False):
  1351. """
  1352. Preprocesses the output text by removing any leading or trailing
  1353. whitespace, converting all whitespace sequences to a single space
  1354. character, and capitalizing the first character of the text.
  1355. Args:
  1356. text (str): The text to be preprocessed.
  1357. Returns:
  1358. str: The preprocessed text.
  1359. """
  1360. text = re.sub(r'\s+', ' ', text.strip())
  1361. if self.ensure_sentence_starting_uppercase:
  1362. if text:
  1363. text = text[0].upper() + text[1:]
  1364. # Ensure the text ends with a proper punctuation
  1365. # if it ends with an alphanumeric character
  1366. if not preview:
  1367. if self.ensure_sentence_ends_with_period:
  1368. if text and text[-1].isalnum():
  1369. text += '.'
  1370. return text
  1371. def _find_tail_match_in_text(self, text1, text2, length_of_match=10):
  1372. """
  1373. Find the position where the last 'n' characters of text1
  1374. match with a substring in text2.
  1375. This method takes two texts, extracts the last 'n' characters from
  1376. text1 (where 'n' is determined by the variable 'length_of_match'), and
  1377. searches for an occurrence of this substring in text2, starting from
  1378. the end of text2 and moving towards the beginning.
  1379. Parameters:
  1380. - text1 (str): The text containing the substring that we want to find
  1381. in text2.
  1382. - text2 (str): The text in which we want to find the matching
  1383. substring.
  1384. - length_of_match(int): The length of the matching string that we are
  1385. looking for
  1386. Returns:
  1387. int: The position (0-based index) in text2 where the matching
  1388. substring starts. If no match is found or either of the texts is
  1389. too short, returns -1.
  1390. """
  1391. # Check if either of the texts is too short
  1392. if len(text1) < length_of_match or len(text2) < length_of_match:
  1393. return -1
  1394. # The end portion of the first text that we want to compare
  1395. target_substring = text1[-length_of_match:]
  1396. # Loop through text2 from right to left
  1397. for i in range(len(text2) - length_of_match + 1):
  1398. # Extract the substring from text2
  1399. # to compare with the target_substring
  1400. current_substring = text2[len(text2) - i - length_of_match:
  1401. len(text2) - i]
  1402. # Compare the current_substring with the target_substring
  1403. if current_substring == target_substring:
  1404. # Position in text2 where the match starts
  1405. return len(text2) - i
  1406. return -1
  1407. def _on_realtime_transcription_stabilized(self, text):
  1408. """
  1409. Callback method invoked when the real-time transcription stabilizes.
  1410. This method is called internally when the transcription text is
  1411. considered "stable" meaning it's less likely to change significantly
  1412. with additional audio input. It notifies any registered external
  1413. listener about the stabilized text if recording is still ongoing.
  1414. This is particularly useful for applications that need to display
  1415. live transcription results to users and want to highlight parts of the
  1416. transcription that are less likely to change.
  1417. Args:
  1418. text (str): The stabilized transcription text.
  1419. """
  1420. if self.on_realtime_transcription_stabilized:
  1421. if self.is_recording:
  1422. self.on_realtime_transcription_stabilized(text)
  1423. def _on_realtime_transcription_update(self, text):
  1424. """
  1425. Callback method invoked when there's an update in the real-time
  1426. transcription.
  1427. This method is called internally whenever there's a change in the
  1428. transcription text, notifying any registered external listener about
  1429. the update if recording is still ongoing. This provides a mechanism
  1430. for applications to receive and possibly display live transcription
  1431. updates, which could be partial and still subject to change.
  1432. Args:
  1433. text (str): The updated transcription text.
  1434. """
  1435. if self.on_realtime_transcription_update:
  1436. if self.is_recording:
  1437. self.on_realtime_transcription_update(text)
  1438. def __enter__(self):
  1439. """
  1440. Method to setup the context manager protocol.
  1441. This enables the instance to be used in a `with` statement, ensuring
  1442. proper resource management. When the `with` block is entered, this
  1443. method is automatically called.
  1444. Returns:
  1445. self: The current instance of the class.
  1446. """
  1447. return self
  1448. def __exit__(self, exc_type, exc_value, traceback):
  1449. """
  1450. Method to define behavior when the context manager protocol exits.
  1451. This is called when exiting the `with` block and ensures that any
  1452. necessary cleanup or resource release processes are executed, such as
  1453. shutting down the system properly.
  1454. Args:
  1455. exc_type (Exception or None): The type of the exception that
  1456. caused the context to be exited, if any.
  1457. exc_value (Exception or None): The exception instance that caused
  1458. the context to be exited, if any.
  1459. traceback (Traceback or None): The traceback corresponding to the
  1460. exception, if any.
  1461. """
  1462. self.shutdown()