From 3cd96367fb91310d7f4068f5ff20ede174b6a5a1 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Thu, 15 Feb 2024 14:59:43 +0530 Subject: [PATCH 1/9] make vad an option --- whisper_live/client.py | 9 +++-- whisper_live/server.py | 86 ++++++++++++++++++++++-------------------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/whisper_live/client.py b/whisper_live/client.py index 179f359..888d238 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -25,7 +25,8 @@ class Client: lang=None, translate=False, model="small", - srt_file_path="output.srt" + srt_file_path="output.srt", + use_vad=True ): """ Initializes a Client instance for audio recording and streaming to a server. @@ -55,6 +56,7 @@ class Client: self.model = model self.server_error = False self.srt_file_path = srt_file_path + self.use_vad = use_vad if translate: self.task = "translate" @@ -201,6 +203,7 @@ class Client: "language": self.language, "task": self.task, "model": self.model, + "use_vad": self.use_vad, } ) ) @@ -497,8 +500,8 @@ class TranscriptionClient: transcription_client() ``` """ - def __init__(self, host, port, lang=None, translate=False, model="small"): - self.client = Client(host, port, lang, translate, model) + def __init__(self, host, port, lang=None, translate=False, model="small", use_vad=True): + self.client = Client(host, port, lang, translate, model, use_vad=use_vad) def __call__(self, audio=None, hls_url=None): """ diff --git a/whisper_live/server.py b/whisper_live/server.py index a85da62..f21d7b3 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -127,6 +127,7 @@ class TranscriptionServer: def __init__(self): self.client_manager = ClientManager() self.no_voice_activity_chunks = 0 + self.use_vad = True def initialize_client( self, websocket, options, faster_whisper_custom_model_path, @@ -165,12 +166,11 @@ class TranscriptionServer: client_uid=options["uid"], model=options["model"], initial_prompt=options.get("initial_prompt"), - vad_parameters=options.get("vad_parameters") + vad_parameters=options.get("vad_parameters"), + use_vad=self.use_vad, ) logging.info("Running faster_whisper backend.") - # self.clients[websocket] = client - # self.clients_start_time[websocket] = time.time() self.client_manager.add_client(websocket, client) def get_audio_from_websocket(self, websocket): @@ -186,22 +186,42 @@ class TranscriptionServer: frame_data = websocket.recv() return np.frombuffer(frame_data, dtype=np.float32) - def handle_new_connection(self, websocket, backend, faster_whisper_custom_model_path, + def handle_new_connection(self, websocket, faster_whisper_custom_model_path, whisper_tensorrt_path, trt_multilingual): - logging.info("New client connected") - options = websocket.recv() - options = json.loads(options) + try: + logging.info("New client connected") + options = websocket.recv() + options = json.loads(options) + self.use_vad = options.get('use_vad') + if self.client_manager.is_server_full(websocket, options): + websocket.close() + return False # Indicates that the connection should not continue - if self.client_manager.is_server_full(websocket, options): - websocket.close() - return + if self.backend == "tensorrt": + self.vad_detector = VoiceActivityDetector(frame_rate=self.RATE) + self.initialize_client(websocket, options, faster_whisper_custom_model_path, + whisper_tensorrt_path, trt_multilingual) + return True + except json.JSONDecodeError: + logging.error("Failed to decode JSON from client") + return False + except Exception as e: + logging.error(f"Error during new connection initialization: {str(e)}") + return False + + def process_audio_frames(self, websocket): + frame_np = self.get_audio_from_websocket(websocket) + client = self.client_manager.get_client(websocket) - self.backend = backend if self.backend == "tensorrt": - self.vad_detector = VoiceActivityDetector(frame_rate=self.RATE) + voice_active = self.voice_activity(websocket, frame_np) + if voice_active: + self.no_voice_activity_chunks = 0 + client.set_eos(False) + if self.use_vad and not voice_active: + return - self.initialize_client( - websocket, options, faster_whisper_custom_model_path, whisper_tensorrt_path, trt_multilingual) + client.add_frames(frame_np) def recv_audio(self, websocket, @@ -233,33 +253,16 @@ class TranscriptionServer: Raises: Exception: If there is an error during the audio frame processing. """ + self.backend = backend + if not self.handle_new_connection(websocket, faster_whisper_custom_model_path, + whisper_tensorrt_path, trt_multilingual): + return + try: - self.handle_new_connection(websocket, backend, faster_whisper_custom_model_path, - whisper_tensorrt_path, trt_multilingual) - while not self.client_manager.is_client_timeout(websocket): - try: - frame_np = self.get_audio_from_websocket(websocket) - client = self.client_manager.get_client(websocket) - - # VAD, for faster_whisper VAD model is already integrated - if self.backend == "tensorrt": - if not self.voice_activity(websocket, frame_np): - continue - self.no_voice_activity_chunks = 0 - client.set_eos(False) - - client.add_frames(frame_np) - - except Exception as e: - logging.error(e) - self.cleanup(websocket) - websocket.close() - break + self.process_audio_frames(websocket) except ConnectionClosed: - logging.info(f"Connection closed by client with path: {websocket.path}") - except json.JSONDecodeError: - logging.error("Failed to decode JSON from client") + logging.info("Connection closed by client") except Exception as e: logging.error(f"Unexpected error: {str(e)}") finally: @@ -660,7 +663,7 @@ class ServeClientTensorRT(ServeClientBase): class ServeClientFasterWhisper(ServeClientBase): def __init__(self, websocket, task="transcribe", device=None, language=None, client_uid=None, model="small.en", - initial_prompt=None, vad_parameters=None): + initial_prompt=None, vad_parameters=None, use_vad=True): """ Initialize a ServeClient instance. The Whisper model is initialized based on the client's language and device availability. @@ -702,6 +705,7 @@ class ServeClientFasterWhisper(ServeClientBase): compute_type="int8" if device == "cpu" else "float16", local_files_only=False, ) + self.use_vad = use_vad # threading self.trans_thread = threading.Thread(target=self.speech_to_text) @@ -776,8 +780,8 @@ class ServeClientFasterWhisper(ServeClientBase): initial_prompt=self.initial_prompt, language=self.language, task=self.task, - vad_filter=True, - vad_parameters=self.vad_parameters) + vad_filter=self.use_vad, + vad_parameters=self.vad_parameters if self.use_vad else None) if self.language is None: self.set_language(info) return result From 9bb92b9bb2f65b19077df4b1f17e5d46387a7649 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Thu, 15 Feb 2024 17:59:12 +0530 Subject: [PATCH 2/9] use_vad option and send end of audio message --- whisper_live/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/whisper_live/client.py b/whisper_live/client.py index 888d238..49c5f7e 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -17,6 +17,7 @@ class Client: Handles audio recording, streaming, and communication with a server using WebSocket. """ INSTANCES = {} + END_OF_AUDIO = "END_OF_AUDIO" def __init__( self, @@ -203,7 +204,7 @@ class Client: "language": self.language, "task": self.task, "model": self.model, - "use_vad": self.use_vad, + "use_vad": self.use_vad } ) ) @@ -274,7 +275,7 @@ class Client: self.stream.write(data) wavfile.close() - + self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8')) # Ensure it's sent as bytes assert self.last_response_recieved while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for: continue From 01dc69e068942ef9ab573e938c48d9c6fcb14914 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Thu, 15 Feb 2024 18:07:19 +0530 Subject: [PATCH 3/9] close when end of audio from client --- whisper_live/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/whisper_live/server.py b/whisper_live/server.py index f21d7b3..353097e 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -184,6 +184,8 @@ class TranscriptionServer: A numpy array containing the audio. """ frame_data = websocket.recv() + if frame_data == b"END_OF_AUDIO": + return False return np.frombuffer(frame_data, dtype=np.float32) def handle_new_connection(self, websocket, faster_whisper_custom_model_path, @@ -212,6 +214,10 @@ class TranscriptionServer: def process_audio_frames(self, websocket): frame_np = self.get_audio_from_websocket(websocket) client = self.client_manager.get_client(websocket) + if frame_np is False: + if self.backend == "tensorrt": + client.set_eos(True) + return False if self.backend == "tensorrt": voice_active = self.voice_activity(websocket, frame_np) @@ -219,9 +225,10 @@ class TranscriptionServer: self.no_voice_activity_chunks = 0 client.set_eos(False) if self.use_vad and not voice_active: - return + return True client.add_frames(frame_np) + return True def recv_audio(self, websocket, @@ -260,7 +267,8 @@ class TranscriptionServer: try: while not self.client_manager.is_client_timeout(websocket): - self.process_audio_frames(websocket) + if not self.process_audio_frames(websocket): + break except ConnectionClosed: logging.info("Connection closed by client") except Exception as e: From 8266099ed089428b8b21e0648c10a2c995cbb448 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Thu, 15 Feb 2024 18:08:02 +0530 Subject: [PATCH 4/9] update tensorrt readme --- TensorRT_whisper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TensorRT_whisper.md b/TensorRT_whisper.md index 6ff2868..1bc303f 100644 --- a/TensorRT_whisper.md +++ b/TensorRT_whisper.md @@ -21,7 +21,7 @@ docker pull ghcr.io/collabora/whisperbot-base:latest ```bash docker run -it --gpus all --shm-size=8g \ --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ - -v /path/to/WhisperLive:/home/WhisperLive \ + -p 9090:9090 -v /path/to/WhisperLive:/home/WhisperLive \ ghcr.io/collabora/whisperbot-base:latest ``` @@ -48,7 +48,7 @@ bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small cd /home/WhisperLive # Install requirements -bash scripts/setup.sh +apt update && bash scripts/setup.sh pip install -r requirements/server.txt # Required to create mel spectogram From e3c7666cf7ce2095bf12796be907e67b40cac5c4 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Thu, 15 Feb 2024 18:13:26 +0530 Subject: [PATCH 5/9] update readme with use_vad --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c4712df..5d4c323 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ client = TranscriptionClient( 9090, lang="en", translate=False, - model="small" + model="small", + use_vad=False, ) client("tests/jfk.wav") @@ -73,11 +74,12 @@ client = TranscriptionClient( 9090, lang="hi", translate=True, - model="small" + model="small", + use_vad=True, ) client() ``` -This command captures audio from the microphone and sends it to the server for transcription. It uses the multilingual model with `hi` as the selected language. We use whisper `small` by default but can be changed to any other option based on the requirements and the hardware running the server. +This command captures audio from the microphone and sends it to the server for transcription. It uses the multilingual model with `hi` as the selected language. We use whisper `small` by default but can be changed to any other option based on the requirements and the hardware running the server. The server also has an option to use `VAD`(voice activity detection) which is set to True by default. - To transcribe from a HLS stream: ```python From b42ced9816d95c3aa5e9032df75fb37dcd6904f3 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Fri, 16 Feb 2024 20:31:38 +0530 Subject: [PATCH 6/9] fix: tests for end of speech message while mocking pyaudio --- tests/test_client.py | 1 + tests/test_server.py | 4 ++-- whisper_live/client.py | 11 ++++++++--- whisper_live/server.py | 3 +++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 468b5a1..56d5dbc 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -46,6 +46,7 @@ class TestClientCallbacks(BaseTestCase): "language": self.client.language, "task": self.client.task, "model": self.client.model, + "use_vad": True }) self.client.on_open(self.mock_ws_app) self.mock_ws_app.send.assert_called_with(expected_message) diff --git a/tests/test_server.py b/tests/test_server.py index cd14bb3..e5d630f 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -69,7 +69,7 @@ class TestServerConnection(unittest.TestCase): class TestServerInferenceAccuracy(unittest.TestCase): @classmethod def setUpClass(cls): - cls.server_process = subprocess.Popen(["python", "run_server.py"]) # Adjust the command as needed + cls.server_process = subprocess.Popen(["python", "run_server.py"]) time.sleep(2) @classmethod @@ -134,4 +134,4 @@ class TestExceptionHandling(unittest.TestCase): for message in log.output: print(message) print() - self.assertTrue(any("Unexpected error: Unexpected error" in message for message in log.output)) + self.assertTrue(any("Unexpected error" in message for message in log.output)) diff --git a/whisper_live/client.py b/whisper_live/client.py index 104824c..6d32257 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -58,6 +58,7 @@ class Client: self.server_error = False self.srt_file_path = srt_file_path self.use_vad = use_vad + self.last_recieved_segment = None if translate: self.task = "translate" @@ -123,6 +124,10 @@ class Client: (not self.transcript or float(seg['start']) >= float(self.transcript[-1]['end']))): self.transcript.append(seg) + # update last received segment and last valild responsne time + if self.last_recieved_segment is None or self.last_recieved_segment != segments[-1]["text"]: + self.last_response_recieved = time.time() + self.last_recieved_segment = segments[-1]["text"] # Truncate to last 3 entries for brevity. text = text[-3:] @@ -142,7 +147,6 @@ class Client: message (str): The received message from the server. """ - self.last_response_recieved = time.time() message = json.loads(message) if self.uid != message.get("uid"): @@ -158,6 +162,7 @@ class Client: self.recording = False if "message" in message.keys() and message["message"] == "SERVER_READY": + self.last_response_recieved = time.time() self.recording = True self.server_backend = message["backend"] print(f"[INFO]: Server Running with backend {self.server_backend}") @@ -275,11 +280,11 @@ class Client: self.stream.write(data) wavfile.close() - self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8')) # Ensure it's sent as bytes + assert self.last_response_recieved while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for: continue - + self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8')) # Ensure it's sent as bytes if self.server_backend == "faster_whisper": self.write_srt_file(self.srt_file_path) self.stream.close() diff --git a/whisper_live/server.py b/whisper_live/server.py index 353097e..7bebe39 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -207,6 +207,9 @@ class TranscriptionServer: except json.JSONDecodeError: logging.error("Failed to decode JSON from client") return False + except ConnectionClosed: + logging.info("Connection closed by client") + return False except Exception as e: logging.error(f"Error during new connection initialization: {str(e)}") return False From dc22b7da9f4d5829b5b57885260497f82549aff7 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 20 Feb 2024 11:53:45 +0530 Subject: [PATCH 7/9] add srt_file_path option --- whisper_live/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper_live/client.py b/whisper_live/client.py index 6d32257..a759eae 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -284,7 +284,7 @@ class Client: assert self.last_response_recieved while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for: continue - self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8')) # Ensure it's sent as bytes + self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8')) if self.server_backend == "faster_whisper": self.write_srt_file(self.srt_file_path) self.stream.close() @@ -507,7 +507,7 @@ class TranscriptionClient: ``` """ def __init__(self, host, port, lang=None, translate=False, model="small", use_vad=True): - self.client = Client(host, port, lang, translate, model, use_vad=use_vad) + self.client = Client(host, port, lang, translate, model, srt_file_path="output.srt", use_vad=use_vad) def __call__(self, audio=None, hls_url=None): """ From 99af50208d109d1f53b3d80c64e16a117ecc0c20 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 20 Feb 2024 13:04:24 +0530 Subject: [PATCH 8/9] add vad option in chrome extension --- Audio-Transcription-Chrome/background.js | 3 ++- Audio-Transcription-Chrome/options.js | 3 ++- Audio-Transcription-Chrome/popup.html | 4 ++++ Audio-Transcription-Chrome/popup.js | 18 ++++++++++++++++-- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Audio-Transcription-Chrome/background.js b/Audio-Transcription-Chrome/background.js index a5028a7..52ce4a7 100644 --- a/Audio-Transcription-Chrome/background.js +++ b/Audio-Transcription-Chrome/background.js @@ -157,7 +157,8 @@ async function startCapture(options) { multilingual: options.useMultilingual, language: options.language, task: options.task, - modelSize: options.modelSize + modelSize: options.modelSize, + useVad: options.useVad, }, }); } else { diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index 435d75b..6c3ef62 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -99,7 +99,8 @@ async function startRecord(option) { uid: uuid, language: option.language, task: option.task, - model: option.modelSize + model: option.modelSize, + use_vad: option.useVad }) ); }; diff --git a/Audio-Transcription-Chrome/popup.html b/Audio-Transcription-Chrome/popup.html index 8d45bcb..1320336 100644 --- a/Audio-Transcription-Chrome/popup.html +++ b/Audio-Transcription-Chrome/popup.html @@ -15,6 +15,10 @@ +
+ + +
+
+ + +
diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js index d71dc0a..61c37a5 100644 --- a/Audio-Transcription-Firefox/popup.js +++ b/Audio-Transcription-Firefox/popup.js @@ -3,6 +3,7 @@ document.addEventListener("DOMContentLoaded", function() { const stopButton = document.getElementById("stopCapture"); const useServerCheckbox = document.getElementById("useServerCheckbox"); + const useVadCheckbox = document.getElementById("useVadCheckbox"); const languageDropdown = document.getElementById('languageDropdown'); const taskDropdown = document.getElementById('taskDropdown'); const modelSizeDropdown = document.getElementById('modelSizeDropdown'); @@ -34,6 +35,12 @@ document.addEventListener("DOMContentLoaded", function() { } }); + browser.storage.local.get("useVadState", ({ useVadState }) => { + if (useVadState !== undefined) { + useVadCheckbox.checked = useVadState; + } + }); + browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { if (storedLanguage !== undefined) { languageDropdown.value = storedLanguage; @@ -76,7 +83,8 @@ document.addEventListener("DOMContentLoaded", function() { port: port, language: selectedLanguage, task: selectedTask, - modelSize: selectedModelSize + modelSize: selectedModelSize, + useVad: useVadCheckbox.checked, } }); toggleCaptureButtons(true); @@ -115,6 +123,7 @@ document.addEventListener("DOMContentLoaded", function() { startButton.disabled = isCapturing; stopButton.disabled = !isCapturing; useServerCheckbox.disabled = isCapturing; + useVadCheckbox.disabled = isCapturing; modelSizeDropdown.disabled = isCapturing; languageDropdown.disabled = isCapturing; taskDropdown.disabled = isCapturing; @@ -128,6 +137,11 @@ document.addEventListener("DOMContentLoaded", function() { browser.storage.local.set({ useServerState }); }); + useVadCheckbox.addEventListener("change", () => { + const useVadState = useVadCheckbox.checked; + browser.storage.local.set({ useVadState }); + }); + languageDropdown.addEventListener('change', function() { if (languageDropdown.value === "") { selectedLanguage = null;