From 025873d2caf9e62303828bb0d50b6c695dd1b13e Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:57:03 +0530 Subject: [PATCH 1/8] Update TensorRT server requirements --- requirements/server.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements/server.txt b/requirements/server.txt index 77e5378..7113229 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -2,4 +2,7 @@ faster-whisper==0.10.0 torch websockets onnxruntime==1.16.0 -numba \ No newline at end of file +numba +openai-whisper +kaldialign +soundfile From d5a0f5859e50c2eb8bae7867f5dc989e464fd8e3 Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:06:56 +0530 Subject: [PATCH 2/8] Update TensorRT_whisper.md ffmpeg is needed for model warmup --- TensorRT_whisper.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TensorRT_whisper.md b/TensorRT_whisper.md index 2935363..8f1e9eb 100644 --- a/TensorRT_whisper.md +++ b/TensorRT_whisper.md @@ -48,6 +48,7 @@ bash build_whisper_tensorrt /root/TensorRT-LLM-examples small cd /home/WhisperLive # Install requirements +bash setup.sh pip install -r requirements/server.txt # Required to create mel spectogram From 4d94c6b38ba1ce371d0131f286b07b84187e7b53 Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:10:18 +0530 Subject: [PATCH 3/8] Update server.txt install ffmpeg-python to load test file for model warmup --- requirements/server.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/server.txt b/requirements/server.txt index 7113229..98752ae 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -6,3 +6,4 @@ numba openai-whisper kaldialign soundfile +ffmpeg-python From 30d2cffb9324a328ae3433471fa968a9bd2356d1 Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:15:39 +0530 Subject: [PATCH 4/8] load audio for warmup --- whisper_live/transcriber_tensorrt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper_live/transcriber_tensorrt.py b/whisper_live/transcriber_tensorrt.py index 8634a8f..a36bb72 100644 --- a/whisper_live/transcriber_tensorrt.py +++ b/whisper_live/transcriber_tensorrt.py @@ -11,7 +11,7 @@ import numpy as np from whisper.tokenizer import get_tokenizer from whisper_live.tensorrt_utils import (mel_filters, store_transcripts, write_error_stats, load_audio_wav_format, - pad_or_trim) + pad_or_trim, load_audio) import tensorrt_llm import tensorrt_llm.logger as logger @@ -337,4 +337,4 @@ def decode_wav_file( if normalizer: prediction = normalizer(prediction) - return prediction.strip() \ No newline at end of file + return prediction.strip() From 703263b375def03c2f4c06f6193fc14f6314155f Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:17:27 +0530 Subject: [PATCH 5/8] wamrup tensorrt engine --- whisper_live/server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/whisper_live/server.py b/whisper_live/server.py index be0e959..3305911 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -397,6 +397,7 @@ class ServeClientTensorRT(ServeClientBase): language=self.language, task=self.task ) + self.warmup() # threading self.trans_thread = threading.Thread(target=self.speech_to_text) @@ -410,6 +411,12 @@ class ServeClientTensorRT(ServeClientBase): } ) ) + + def warmup(self, warmup_steps=10): + logging.info("[INFO:] Warming up TensorRT engine..") + mel, duration = self.transcriber.log_mel_spectrogram("tests/jfk.flac") + for i in range(warmup_steps): + last_segment = self.transcriber.transcribe(mel) def set_eos(self, eos): self.lock.acquire() From 2f3c1cd17228a94795bfce98d49ffe1283b34995 Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:27:51 +0530 Subject: [PATCH 6/8] Update TensorRT_whisper.md fix: typo in bash script name to build tensorrt engine --- TensorRT_whisper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TensorRT_whisper.md b/TensorRT_whisper.md index 8f1e9eb..10bd4e5 100644 --- a/TensorRT_whisper.md +++ b/TensorRT_whisper.md @@ -37,10 +37,10 @@ python -c "import torch; import tensorrt; import tensorrt_llm" - We build `small.en` and `small` multilingual TensorRT engine. The script logs the path of the directory with Whisper TensorRT engine. We need the model_path to run the server. ```bash # convert small.en -bash build_whisper_tensorrt /root/TensorRT-LLM-examples small.en +bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small.en # convert small multilingual model -bash build_whisper_tensorrt /root/TensorRT-LLM-examples small +bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small ``` ## Run WhisperLive Server with TensorRT Backend From d6b51ccd7d6c8aaeedf814dd07a516aff718b89a Mon Sep 17 00:00:00 2001 From: makaveli <39617050+makaveli10@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:33:12 +0530 Subject: [PATCH 7/8] Update TensorRT_whisper.md typo: setup.sh file path --- TensorRT_whisper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TensorRT_whisper.md b/TensorRT_whisper.md index 10bd4e5..6ff2868 100644 --- a/TensorRT_whisper.md +++ b/TensorRT_whisper.md @@ -48,7 +48,7 @@ bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small cd /home/WhisperLive # Install requirements -bash setup.sh +bash scripts/setup.sh pip install -r requirements/server.txt # Required to create mel spectogram From 0e6b1c0632d8d4267db1a244f0bc6c5d2eb9fef7 Mon Sep 17 00:00:00 2001 From: gchust Date: Wed, 31 Jan 2024 11:33:48 +0000 Subject: [PATCH 8/8] fix: keyError: 'model' in server, when using browser extension --- Audio-Transcription-Chrome/options.js | 2 +- Audio-Transcription-Firefox/content.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index 9717856..7e8ae5b 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -103,7 +103,7 @@ async function startRecord(option) { multilingual: option.multilingual, language: option.language, task: option.task, - model_size: option.modelSize + model: option.modelSize }) ); }; diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index 682b142..82b045b 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -78,7 +78,7 @@ function startRecording(data) { multilingual: data.useMultilingual, language: data.language, task: data.task, - model_size: data.modelSize + model: data.modelSize }) ); };