bump version 0.0.10

Add the `large-v3` model

Audio-Transcription-Chrome/README.md

@@ -29,6 +29,7 @@ When using the Audio Transcription extension, you have the following options:
  - **Use Multilingual Model**: Enable this option to utilize the multilingual capabilities of OpenAI-whisper.
  - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper.
  - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English.
+ - **Model Size**: Select the whisper model size to run the server with.
 
 ### Getting Started
 - Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live).

Audio-Transcription-Chrome/background.js

@@ -156,7 +156,8 @@ async function startCapture(options) {
           port: options.port, 
           multilingual: options.useMultilingual,
           language: options.language,
-          task: options.task
+          task: options.task,
+          modelSize: options.modelSize
         },
       });
     } else {
@@ -207,13 +208,3 @@ chrome.runtime.onMessage.addListener(async (message) => {
 });
 
 
-/**
- * Listens for if the tab is reloaded.
- * @param {Object} message - The message received from the runtime.
- */
-chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, tab) => {
-  if (changeInfo.status === 'complete') {
-    await executeScriptInTab(tabId, "content.js");
-    await delayExecution(500);
-  }
-});

Audio-Transcription-Chrome/content.js

@@ -173,13 +173,13 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
     if (type === "STOP") {
         remove_element();
         sendResponse({data: "STOPPED"});
-        return;
+        return true;
     } else if (type === "showWaitPopup"){
         initPopupElement();
 
         showPopup(`Estimated wait time ~ ${Math.round(data)} minutes`);
         sendResponse({data: "popup"});
-        return;
+        return true;
     }
 
     init_element();
@@ -234,4 +234,5 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
     }
 
     sendResponse({});
+    return true;
 });

Audio-Transcription-Chrome/options.js

@@ -102,7 +102,8 @@ async function startRecord(option) {
           uid: uuid,
           multilingual: option.multilingual,
           language: option.language,
-          task: option.task
+          task: option.task,
+          model_size: option.modelSize
         })
       );
     };
@@ -184,16 +185,17 @@ async function startRecord(option) {
  * @param {Object} sender - The sender object containing information about the message sender.
  * @param {Function} sendResponse - The function to send a response back to the message sender.
  */
-chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
+chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
   const { type, data } = request;
 
   switch (type) {
     case "start_capture":
-      await startRecord(data);
+      startRecord(data);
       break;
     default:
       break;
   }
 
   sendResponse({});
+  return true;
 });

Audio-Transcription-Chrome/popup.html

@@ -125,11 +125,23 @@
   </div>
   <div class="dropdown-container">
     <label for="taskDropdown">Select task:</label>
-    <select id="taskDropdown" disabled>
+    <select id="taskDropdown" >
       <option value="">Select Task</option>
       <option value="transcribe" selected>Transcribe</option>
       <option value="translate">Translate</option>
     </select>
   </div>
+  <div class="dropdown-container">
+    <label for="modelSizeDropdown">Select Model Size:</label>
+    <select id="modelSizeDropdown">
+      <option value="">Select Task</option>
+      <option value="tiny">Tiny</option>
+      <option value="base">Base</option>
+      <option value="small" selected>Small</option>
+      <option value="medium">Medium</option>
+      <option value="large-v2">Large-v2</option>
+      <option value="large-v3">Large-v3</option>
+    </select>
+  </div>
 </body>
 </html>

Audio-Transcription-Chrome/popup.js

@@ -7,8 +7,10 @@ document.addEventListener("DOMContentLoaded", function () {
   const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
+  const modelSizeDropdown = document.getElementById('modelSizeDropdown');
   let selectedLanguage = null;
   let selectedTask = taskDropdown.value;
+  let selectedModelSize = modelSizeDropdown.value;
 
   // Add click event listeners to the buttons
   startButton.addEventListener("click", startCapture);
@@ -52,6 +54,13 @@ document.addEventListener("DOMContentLoaded", function () {
     }
   });
 
+  chrome.storage.local.get("selectedModelSize", ({ selectedModelSize: storedModelSize }) => {
+    if (storedModelSize !== undefined) {
+      modelSizeDropdown.value = storedModelSize;
+      selectedModelSize = storedModelSize;
+    }
+  });
+
   // Function to handle the start capture button click event
   async function startCapture() {
     // Ignore click if the button is disabled
@@ -64,7 +73,7 @@ document.addEventListener("DOMContentLoaded", function () {
 
     // Send a message to the background script to start capturing
     let host = "localhost";
-    let port = "9090";
+    let port = "5901";
     const useCollaboraServer = useServerCheckbox.checked;
     if (useCollaboraServer){
       host = "transcription.kurg.org"
@@ -79,7 +88,8 @@ document.addEventListener("DOMContentLoaded", function () {
         port: port,
         useMultilingual: useMultilingualCheckbox.checked,
         language: selectedLanguage,
-        task: selectedTask
+        task: selectedTask,
+        modelSize: selectedModelSize
       }, () => {
         // Update capturing state in storage and toggle the buttons
         chrome.storage.local.set({ capturingState: { isCapturing: true } }, () => {
@@ -120,6 +130,7 @@ document.addEventListener("DOMContentLoaded", function () {
     stopButton.disabled = !isCapturing;
     useServerCheckbox.disabled = isCapturing; 
     useMultilingualCheckbox.disabled = isCapturing;
+    modelSizeDropdown.disabled = isCapturing;
     
     startButton.classList.toggle("disabled", isCapturing);
     stopButton.classList.toggle("disabled", !isCapturing);
@@ -157,6 +168,11 @@ document.addEventListener("DOMContentLoaded", function () {
     chrome.storage.local.set({ selectedTask });
   });
 
+  modelSizeDropdown.addEventListener('change', function() {
+    selectedModelSize = modelSizeDropdown.value;
+    chrome.storage.local.set({ selectedModelSize });
+  });
+
   chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
     if (request.action === "updateSelectedLanguage") {
       const detectedLanguage = request.detectedLanguage;

Audio-Transcription-Firefox/README.md

@@ -27,6 +27,7 @@ When using the Audio Transcription extension, you have the following options:
  - **Use Multilingual Model**: Enable this option to utilize the multilingual capabilities of OpenAI-whisper.
  - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper.
  - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English.
+  - **Model Size**: Select the whisper model size to run the server with.
 
 ### Getting Started
 - Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live).

Audio-Transcription-Firefox/content.js

@@ -77,7 +77,8 @@ function startRecording(data) {
             uid: uuid,
             multilingual: data.useMultilingual,
             language: data.language,
-            task: data.task
+            task: data.task,
+            model_size: data.modelSize
         })
       );
     };

Audio-Transcription-Firefox/popup.html

@@ -133,5 +133,17 @@
       <option value="translate">Translate</option>
     </select>
   </div>
+  <div class="dropdown-container">
+    <label for="modelSizeDropdown">Select Model Size:</label>
+    <select id="modelSizeDropdown">
+      <option value="">Select Task</option>
+      <option value="tiny">Tiny</option>
+      <option value="base">Base</option>
+      <option value="small" selected>Small</option>
+      <option value="medium">Medium</option>
+      <option value="large-v2">Large-v2</option>
+      <option value="large-v3">Large-v3</option>
+    </select>
+  </div>
 </body>
 </html>

Audio-Transcription-Firefox/popup.js

@@ -6,8 +6,11 @@ document.addEventListener("DOMContentLoaded", function() {
   const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
+  const modelSizeDropdown = document.getElementById('modelSizeDropdown');
   let selectedLanguage = null;
   let selectedTask = taskDropdown.value;
+  let selectedModelSize = modelSizeDropdown.value;
+  
 
   browser.storage.local.get("capturingState")
     .then(function(result) {
@@ -54,9 +57,16 @@ document.addEventListener("DOMContentLoaded", function() {
     }
   });
 
+  browser.storage.local.get("selectedModelSize", ({ selectedModelSize: storedModelSize }) => {
+    if (storedModelSize !== undefined) {
+      modelSizeDropdown.value = storedModelSize;
+      selectedModelSize = storedModelSize;
+    }
+  });
+
   startButton.addEventListener("click", function() {
     let host = "localhost";
-    let port = "9090";
+    let port = "5901";
     const useCollaboraServer = useServerCheckbox.checked;
 
     if (useCollaboraServer){
@@ -75,7 +85,8 @@ document.addEventListener("DOMContentLoaded", function() {
               port: port,
               useMultilingual: useMultilingualCheckbox.checked,
               language: selectedLanguage,
-              task: selectedTask
+              task: selectedTask,
+              modelSize: selectedModelSize
             } 
           });
         toggleCaptureButtons(true);
@@ -115,6 +126,7 @@ document.addEventListener("DOMContentLoaded", function() {
     stopButton.disabled = !isCapturing;
     useServerCheckbox.disabled = isCapturing;
     useMultilingualCheckbox.disabled = isCapturing;
+    modelSizeDropdown.disabled = isCapturing;
 
     startButton.classList.toggle("disabled", isCapturing);
     stopButton.classList.toggle("disabled", !isCapturing);
@@ -152,6 +164,11 @@ document.addEventListener("DOMContentLoaded", function() {
     browser.storage.local.set({ selectedTask });
   });
 
+  modelSizeDropdown.addEventListener('change', function() {
+    selectedModelSize = modelSizeDropdown.value;
+    browser.storage.local.set({ selectedModelSize });
+  });
+
   browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
     if (request.action === "updateSelectedLanguage") {
       const detectedLanguage = request.data;

README.md

@@ -28,19 +28,40 @@ Unlike traditional speech recognition systems that rely on continuous audio stre
     - To transcribe an audio file:
     ```python
       from whisper_live.client import TranscriptionClient
-      client = TranscriptionClient("localhost", 9090, is_multilingual=True, lang="hi", translate=True)
-      client(audio_file_path)
+      client = TranscriptionClient(
+        "localhost",
+        9090,
+        is_multilingual=False,
+        lang="en",
+        translate=False,
+        model_size="small"
+      )
+
+      client("tests/jfk.wav")
     ```
-    This command transcribes the specified audio file (audio.wav) using the Whisper model. It connects to the server running on localhost at port 9090. It also enables the multilingual feature, allowing transcription in multiple languages. The language option specifies the target language for transcription, in this case, Hindi ("hi"). The translate option should be set to `True` if we want to translate from the source language to English and `False` if we want to transcribe in the source language.
+    This command transcribes the specified audio file (audio.wav) using the Whisper model. It connects to the server running on localhost at port 9090. It can also enable the multilingual feature, allowing transcription in multiple languages. The language option specifies the target language for transcription, in this case, English ("en"). The translate option should be set to `True` if we want to translate from the source language to English and `False` if we want to transcribe in the source language.
 
     - To transcribe from microphone:
     ```python
       from whisper_live.client import TranscriptionClient
-      client = TranscriptionClient(host, port, is_multilingual=True, lang="hi", translate=True)
+      client = TranscriptionClient(
+        "localhost",
+        9090,
+        is_multilingual=True,
+        lang="hi",
+        translate=True,
+        model_size="small"
+      )
       client()
     ```
-    This command captures audio from the microphone and sends it to the server for transcription. It uses the same options as the previous command, enabling the multilingual feature and specifying the target language and task.
+    This command captures audio from the microphone and sends it to the server for transcription. It uses the multilingual option with `hi` as the selected language, enabling the multilingual feature and specifying the target language and task. We use whisper `small` by default but can be changed to any other option based on the requirements and the hardware running the server.
 
+    - To transcribe from a HLS stream:
+    ```python
+      client = TranscriptionClient(host, port, is_multilingual=True, lang="en", translate=False) 
+      client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/bbc_1xtra.isml/bbc_1xtra-audio%3d96000.norewind.m3u8") 
+    ```
+    This command streams audio into the server from a HLS stream. It uses the same options as the previous command, enabling the multilingual feature and specifying the target language and task.
 
 ## Transcribe audio from browser
 - Run the server

requirements/server.txt

@@ -1,5 +1,5 @@
 PyAudio
-faster-whisper==0.6.0
+faster-whisper==0.10.0
 --extra-index-url https://download.pytorch.org/whl/cu111
 torch==1.10.1
 torchaudio==0.10.1

setup.py

@@ -41,7 +41,7 @@ setup(name="whisper-live",
       ),
       install_requires=[
         "PyAudio",
-        "faster-whisper==0.6.0",
+        "faster-whisper==0.10.0",
         "torch",
         "torchaudio",
         "websockets",
@@ -51,4 +51,4 @@ setup(name="whisper-live",
         "websocket-client",
       ],
       python_requires=">=3.8"
-)
+)

whisper_live/__version__.py

@@ -1 +1 @@
-__version__="0.0.7"
+__version__="0.0.10"

whisper_live/client.py

@@ -50,7 +50,7 @@ class Client:
     INSTANCES = {}
 
     def __init__(
-        self, host=None, port=None, is_multilingual=False, lang=None, translate=False
+        self, host=None, port=None, is_multilingual=False, lang=None, translate=False, model_size="small"
     ):
         """
         Initializes a Client instance for audio recording and streaming to a server.
@@ -80,7 +80,9 @@ class Client:
         self.last_response_recieved = None
         self.disconnect_if_no_response_for = 15
         self.multilingual = is_multilingual
-        self.language = lang if is_multilingual else "en"
+        self.language = lang
+        self.model_size = model_size
+        self.server_error = False
         if translate:
             self.task = "translate"
 
@@ -140,11 +142,16 @@ class Client:
             print("[ERROR]: invalid client uid")
             return
 
-        if "status" in message.keys() and message["status"] == "WAIT":
-            self.waiting = True
-            print(
-                f"[INFO]:Server is full. Estimated wait time {round(message['message'])} minutes."
-            )
+        if "status" in message.keys():
+            if message["status"] == "WAIT":
+                self.waiting = True
+                print(
+                    f"[INFO]:Server is full. Estimated wait time {round(message['message'])} minutes."
+                )
+            elif message["status"] == "ERROR":
+                print(f"Message from Server: {message['message']}")
+                self.server_error = True
+            return
 
         if "message" in message.keys() and message["message"] == "DISCONNECT":
             print("[INFO]: Server overtime disconnected.")
@@ -213,6 +220,7 @@ class Client:
                     "multilingual": self.multilingual,
                     "language": self.language,
                     "task": self.task,
+                    "model_size": self.model_size,
                 }
             )
         )
@@ -344,6 +352,42 @@ class Client:
             wavfile.setframerate(self.rate)
             wavfile.writeframes(frames)
 
+    def process_hls_stream(self, hls_url):
+        """
+        Connect to an HLS source, process the audio stream, and send it for transcription.
+
+        Args:
+            hls_url (str): The URL of the HLS stream source.
+        """
+        print("[INFO]: Connecting to HLS stream...")
+        process = None  # Initialize process to None
+
+        try:
+            # Connecting to the HLS stream using ffmpeg-python
+            process = (
+                ffmpeg
+                .input(hls_url, threads=0)
+                .output('-', format='s16le', acodec='pcm_s16le', ac=1, ar=self.rate)
+                .run_async(pipe_stdout=True, pipe_stderr=True)
+            )
+
+            # Process the stream
+            while True:
+                in_bytes = process.stdout.read(self.chunk * 2)  # 2 bytes per sample
+                if not in_bytes:
+                    break
+                audio_array = self.bytes_to_float_array(in_bytes)
+                self.send_packet_to_server(audio_array.tobytes())
+
+        except Exception as e:
+            print(f"[ERROR]: Failed to connect to HLS stream: {e}")
+        finally:
+            if process:
+                process.kill()
+
+        print("[INFO]: HLS stream processing finished.")
+
+
     def record(self, out_file="output_recording.wav"):
         """
         Record audio data from the input stream and save it to a WAV file.
@@ -461,10 +505,10 @@ class TranscriptionClient:
         transcription_client()
         ```
     """
-    def __init__(self, host, port, is_multilingual=False, lang=None, translate=False):
-        self.client = Client(host, port, is_multilingual, lang, translate)
+    def __init__(self, host, port, is_multilingual=False, lang=None, translate=False, model_size="small"):
+        self.client = Client(host, port, is_multilingual, lang, translate, model_size)
 
-    def __call__(self, audio=None):
+    def __call__(self, audio=None, hls_url=None):
         """
         Start the transcription process.
 
@@ -478,13 +522,15 @@ class TranscriptionClient:
         """
         print("[INFO]: Waiting for server ready ...")
         while not self.client.recording:
-            if self.client.waiting:
+            if self.client.waiting or self.client.server_error:
                 self.client.close_websocket()
                 return
-            pass
+
         print("[INFO]: Server Ready!")
-        if audio is not None:
+        if hls_url is not None:
+            self.client.process_hls_stream(hls_url)
+        elif audio is not None:
             resampled_file = resample(audio)
             self.client.play_file(resampled_file)
         else:
-            self.client.record()
+            self.client.record()

whisper_live/server.py

@@ -13,7 +13,6 @@ import torch
 import numpy as np
 import time
 from whisper_live.transcriber import WhisperModel
-from whisper_live.vad import VoiceActivityDetection
 
 
 class TranscriptionServer:
@@ -35,8 +34,6 @@ class TranscriptionServer:
 
     def __init__(self):
         # voice activity detection model
-        self.vad_model = VoiceActivityDetection()
-        self.vad_threshold = 0.4
 
         self.clients = {}
         self.websockets = {}
@@ -104,9 +101,10 @@ class TranscriptionServer:
             multilingual=options["multilingual"],
             language=options["language"],
             task=options["task"],
-            client_uid=options["uid"]
+            client_uid=options["uid"],
+            model_size=options["model_size"]
         )
-
+        
         self.clients[websocket] = client
         self.clients_start_time[websocket] = time.time()
 
@@ -115,15 +113,6 @@ class TranscriptionServer:
                 frame_data = websocket.recv()
                 frame_np = np.frombuffer(frame_data, dtype=np.float32)
 
-                try:
-                    speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item()
-                    if speech_prob < self.vad_threshold:
-                        continue
-
-                except Exception as e:
-                    logging.error(e)
-                    return
-
                 self.clients[websocket].add_frames(frame_np)
 
                 elapsed_time = time.time() - self.clients_start_time[websocket]
@@ -139,7 +128,8 @@ class TranscriptionServer:
 
             except Exception as e:
                 logging.error(e)
-                self.clients[websocket].cleanup()
+                if self.clients[websocket].model_size is not None:
+                    self.clients[websocket].cleanup()
                 self.clients.pop(websocket)
                 self.clients_start_time.pop(websocket)
                 logging.info("Connection Closed.")
@@ -192,7 +182,16 @@ class ServeClient:
     SERVER_READY = "SERVER_READY"
     DISCONNECT = "DISCONNECT"
 
-    def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None, client_uid=None):
+    def __init__(
+        self,
+        websocket,
+        task="transcribe",
+        device=None,
+        multilingual=False,
+        language=None,
+        client_uid=None,
+        model_size="small"
+        ):
         """
         Initialize a ServeClient instance.
         The Whisper model is initialized based on the client's language and device availability.
@@ -211,11 +210,22 @@ class ServeClient:
         self.client_uid = client_uid
         self.data = b""
         self.frames = b""
-        self.language = language if multilingual else "en"
+        self.model_sizes = [
+            "tiny", "base", "small", "medium", "large-v2", "large-v3"
+        ]
+        self.multilingual = multilingual
+        self.model_size = self.get_model_size(model_size)
+        self.language = language if self.multilingual else "en"
         self.task = task
+        self.websocket = websocket
+        
         device = "cuda" if torch.cuda.is_available() else "cpu"
+        
+        if self.model_size == None:
+            return
+        
         self.transcriber = WhisperModel(
-            "small" if multilingual else "small.en", 
+            self.model_size, 
             device=device,
             compute_type="int8" if device=="cpu" else "float16", 
             local_files_only=False,
@@ -240,7 +250,6 @@ class ServeClient:
         self.pick_previous_segments = 2
 
         # threading
-        self.websocket = websocket
         self.trans_thread = threading.Thread(target=self.speech_to_text)
         self.trans_thread.start()
         self.websocket.send(
@@ -252,34 +261,30 @@ class ServeClient:
             )
         )
     
-    def fill_output(self, output):
+    def get_model_size(self, model_size):
         """
-        Format the current incomplete transcription output by combining it with previous complete segments.
-        The resulting transcription is wrapped into two lines, each containing a maximum of 50 characters.
-
-        It ensures that the combined transcription fits within two lines, with a maximum of 50 characters per line.
-        Segments are concatenated in the order they exist in the list of previous segments, with the most
-        recent complete segment first and older segments prepended as needed to maintain the character limit.
-        If a 3-second pause is detected in the previous segments, any text preceding it is discarded to ensure
-        the transcription starts with the most recent complete content. The resulting transcription is returned
-        as a single string.
-
-        Args:
-            output(str): The current incomplete transcription segment.
+        Returns the whisper model size based on multilingual.
+        """
+        if model_size not in self.model_sizes:
+            self.websocket.send(
+                json.dumps(
+                    {
+                        "uid": self.client_uid,
+                        "status": "ERROR",
+                        "message": f"Invalid model size {model_size}. Available choices: {self.model_sizes}"
+                    }
+                )
+            )
+            return None
         
-        Returns:
-            str: A formatted transcription wrapped in two lines.
-        """
-        text = ''
-        pick_prev = min(len(self.text), self.pick_previous_segments)
-        for seg in self.text[-pick_prev:]:
-            # discard everything before a 3 second pause
-            if seg == '':
-                text = ''
-            else:
-                text += seg
-        wrapped = "".join(text + output)
-        return wrapped
+        if model_size in ["large-v2", "large-v3"]:
+            self.multilingual = True
+            return model_size
+
+        if not self.multilingual:
+            model_size = model_size + ".en"
+
+        return model_size
     
     def add_frames(self, frame_np):
         """
@@ -322,25 +327,6 @@ class ServeClient:
             Exception: If there is an issue with audio processing or WebSocket communication.
 
         """
-        # detect language
-        if self.language is None:
-            # wait for 30s of audio
-            while self.frames_np is None or self.frames_np.shape[0] < 30*self.RATE:
-                time.sleep(1)
-            input_bytes = self.frames_np[-30*self.RATE:].copy()
-            self.frames_np = None
-            duration = input_bytes.shape[0] / self.RATE
-
-            self.language, lang_prob = self.transcriber.transcribe(
-                    input_bytes, 
-                    initial_prompt=None,
-                    language=self.language,
-                    task=self.task
-                )
-            logging.info(f"Detected language {self.language} with probability {lang_prob}")
-            self.websocket.send(json.dumps(
-                {"uid": self.client_uid, "language": self.language, "language_prob": lang_prob}))
-
         while True:
             if self.exit:
                 logging.info("Exiting speech to text thread")
@@ -358,24 +344,31 @@ class ServeClient:
             samples_take = max(0, (self.timestamp_offset - self.frames_offset)*self.RATE)
             input_bytes = self.frames_np[int(samples_take):].copy()
             duration = input_bytes.shape[0] / self.RATE
-            if duration<1.0: 
+            if duration<1.0:
                 continue
             try:
                 input_sample = input_bytes.copy()
-                # set previous complete segment as initial prompt
-                if len(self.text) and self.text[-1] != '': 
-                    initial_prompt = self.text[-1]
-                else: 
-                    initial_prompt = None
                 
                 # whisper transcribe with prompt
-                result = self.transcriber.transcribe(
+                result, info = self.transcriber.transcribe(
                     input_sample, 
-                    initial_prompt=initial_prompt,
+                    initial_prompt=None,
                     language=self.language,
-                    task=self.task
+                    task=self.task,
+                    vad_filter=True,
+                    vad_parameters={"threshold": 0.5}
                 )
 
+                if self.language is None:
+                    if info.language_probability > 0.5:
+                        self.language = info.language
+                        logging.info(f"Detected language {self.language} with probability {info.language_probability}")
+                        self.websocket.send(json.dumps(
+                            {"uid": self.client_uid, "language": self.language, "language_prob": info.language_probability}))
+                    else:
+                        # detect language again
+                        continue
+
                 if len(result):
                     self.t_start = None
                     last_segment = self.update_segments(result, duration)
@@ -384,17 +377,7 @@ class ServeClient:
                     else:
                         segments = self.transcript[-self.send_last_n_segments:]
                     if last_segment is not None:
-                        segments = segments + [last_segment]
-                    
-                    try:
-                        self.websocket.send(
-                            json.dumps({
-                                "uid": self.client_uid,
-                                "segments": segments
-                            })
-                        )
-                    except Exception as e:
-                        logging.error(f"[ERROR]: {e}")
+                        segments = segments + [last_segment]                    
                 else:
                     # show previous output if there is pause i.e. no output from whisper
                     segments = []
@@ -410,15 +393,16 @@ class ServeClient:
                         if time.time() - self.t_start > self.add_pause_thresh:
                             self.text.append('')
 
-                    try:
-                        self.websocket.send(
-                            json.dumps({
-                                "uid": self.client_uid,
-                                "segments": segments
-                            })
-                        )
-                    except Exception as e:
-                        logging.error(f"[ERROR]: {e}")
+                try:
+                    self.websocket.send(
+                        json.dumps({
+                            "uid": self.client_uid,
+                            "segments": segments
+                        })
+                    )
+                except Exception as e:
+                    logging.error(f"[ERROR]: {e}")
+
             except Exception as e:
                 logging.error(f"[ERROR]: {e}")
                 time.sleep(0.01)

whisper_live/transcriber.py

@@ -4,7 +4,8 @@ import itertools
 import logging
 import os
 import zlib
-import logging
+import json
+from inspect import signature
 
 from typing import BinaryIO, Iterable, List, NamedTuple, Optional, Tuple, Union
 
@@ -14,21 +15,16 @@ import tokenizers
 
 from faster_whisper.audio import decode_audio
 from faster_whisper.feature_extractor import FeatureExtractor
-from faster_whisper.tokenizer import Tokenizer
-from faster_whisper.utils import download_model, format_timestamp
+from faster_whisper.tokenizer import _LANGUAGE_CODES, Tokenizer
+from faster_whisper.utils import download_model, format_timestamp, get_logger
 from faster_whisper.vad import (
     SpeechTimestampsMap,
+    VadOptions,
     collect_chunks,
     get_speech_timestamps,
 )
 
 
-# implement logger not available in faster_whisper==0.4.1
-def get_logger():
-    """Returns the module logger."""
-    return logging.getLogger("faster_whisper")
-
-
 class Word(NamedTuple):
     start: float
     end: float
@@ -37,18 +33,17 @@ class Word(NamedTuple):
 
 
 class Segment(NamedTuple):
+    id: int
+    seek: int
     start: float
     end: float
     text: str
-    words: Optional[List[Word]]
-    avg_log_prob: float
+    tokens: List[int]
+    temperature: float
+    avg_logprob: float
+    compression_ratio: float
     no_speech_prob: float
-
-
-class AudioInfo(NamedTuple):
-    language: str
-    language_probability: float
-    duration: float
+    words: Optional[List[Word]]
 
 
 class TranscriptionOptions(NamedTuple):
@@ -56,12 +51,15 @@ class TranscriptionOptions(NamedTuple):
     best_of: int
     patience: float
     length_penalty: float
+    repetition_penalty: float
+    no_repeat_ngram_size: int
     log_prob_threshold: Optional[float]
     no_speech_threshold: Optional[float]
     compression_ratio_threshold: Optional[float]
     condition_on_previous_text: bool
+    prompt_reset_on_temperature: float
     temperatures: List[float]
-    initial_prompt: Optional[str]
+    initial_prompt: Optional[Union[str, Iterable[int]]]
     prefix: Optional[str]
     suppress_blank: bool
     suppress_tokens: Optional[List[int]]
@@ -72,6 +70,16 @@ class TranscriptionOptions(NamedTuple):
     append_punctuations: str
 
 
+class TranscriptionInfo(NamedTuple):
+    language: str
+    language_probability: float
+    duration: float
+    duration_after_vad: float
+    all_language_probs: Optional[List[Tuple[str, float]]]
+    transcription_options: TranscriptionOptions
+    vad_options: VadOptions
+
+
 class WhisperModel:
     def __init__(
         self,
@@ -82,14 +90,15 @@ class WhisperModel:
         cpu_threads: int = 0,
         num_workers: int = 1,
         download_root: Optional[str] = None,
-        local_files_only: bool = True,
+        local_files_only: bool = False,
     ):
         """Initializes the Whisper model.
 
         Args:
           model_size_or_path: Size of the model to use (tiny, tiny.en, base, base.en,
-            small, small.en, medium, medium.en, large-v1, or large-v2) or a path to a converted
-            model directory. When a size is configured, the converted model is downloaded
+            small, small.en, medium, medium.en, large-v1, large-v2, large-v3, or large), a path to a converted
+            model directory, or a CTranslate2-converted Whisper model ID from the Hugging Face Hub.
+            When a size or a model ID is configured, the converted model is downloaded
             from the Hugging Face Hub.
           device: Device to use for computation ("cpu", "cuda", "auto").
           device_index: Device ID to use.
@@ -104,8 +113,10 @@ class WhisperModel:
             having multiple workers enables true parallelism when running the model
             (concurrent calls to self.model.generate() will run in parallel).
             This can improve the global throughput at the cost of increased memory usage.
-          download_root: Directory where the model should be saved. If not set, the model
-            is saved in the standard Hugging Face cache directory.
+          download_root: Directory where the models should be saved. If not set, the models
+            are saved in the standard Hugging Face cache directory.
+          local_files_only:  If True, avoid downloading the file and return the path to the
+            local cached file if it exists.
         """
         self.logger = get_logger()
 
@@ -135,7 +146,8 @@ class WhisperModel:
                 "openai/whisper-tiny" + ("" if self.model.is_multilingual else ".en")
             )
 
-        self.feature_extractor = FeatureExtractor()
+        self.feat_kwargs = self._get_feature_kwargs(model_path)
+        self.feature_extractor = FeatureExtractor(**self.feat_kwargs)
         self.num_samples_per_token = self.feature_extractor.hop_length * 2
         self.frames_per_second = (
             self.feature_extractor.sampling_rate // self.feature_extractor.hop_length
@@ -147,6 +159,27 @@ class WhisperModel:
         self.time_precision = 0.02
         self.max_length = 448
 
+    @property
+    def supported_languages(self) -> List[str]:
+        """The languages supported by the model."""
+        return list(_LANGUAGE_CODES) if self.model.is_multilingual else ["en"]
+
+    def _get_feature_kwargs(self, model_path) -> dict:
+        preprocessor_config_file = os.path.join(model_path, "preprocessor_config.json")
+        config = {}
+        if os.path.isfile(preprocessor_config_file):
+            try:
+                with open(preprocessor_config_file, "r", encoding="utf-8") as json_file:
+                    config = json.load(json_file)
+                valid_keys = signature(FeatureExtractor.__init__).parameters.keys()
+                config = {k: v for k, v in config.items() if k in valid_keys}
+            except json.JSONDecodeError as e:
+                self.logger.warning(
+                    "Could not load preprocessor_config.json: %s", str(e)
+                )
+
+        return config
+
     def transcribe(
         self,
         audio: Union[str, BinaryIO, np.ndarray],
@@ -156,6 +189,8 @@ class WhisperModel:
         best_of: int = 5,
         patience: float = 1,
         length_penalty: float = 1,
+        repetition_penalty: float = 1,
+        no_repeat_ngram_size: int = 0,
         temperature: Union[float, List[float], Tuple[float, ...]] = [
             0.0,
             0.2,
@@ -168,7 +203,8 @@ class WhisperModel:
         log_prob_threshold: Optional[float] = -1.0,
         no_speech_threshold: Optional[float] = 0.6,
         condition_on_previous_text: bool = True,
-        initial_prompt: Optional[str] = None,
+        prompt_reset_on_temperature: float = 0.5,
+        initial_prompt: Optional[Union[str, Iterable[int]]] = None,
         prefix: Optional[str] = None,
         suppress_blank: bool = True,
         suppress_tokens: Optional[List[int]] = [-1],
@@ -178,8 +214,8 @@ class WhisperModel:
         prepend_punctuations: str = "\"'“¿([{-",
         append_punctuations: str = "\"'.。,，!！?？:：”)]}、",
         vad_filter: bool = False,
-        vad_parameters: Optional[dict] = None,
-    ) -> Tuple[Iterable[Segment], AudioInfo]:
+        vad_parameters: Optional[Union[dict, VadOptions]] = None,
+    ) -> Tuple[Iterable[Segment], TranscriptionInfo]:
         """Transcribes an input file.
 
         Arguments:
@@ -192,6 +228,9 @@ class WhisperModel:
           best_of: Number of candidates when sampling with non-zero temperature.
           patience: Beam search patience factor.
           length_penalty: Exponential length penalty constant.
+          repetition_penalty: Penalty applied to the score of previously generated tokens
+            (set > 1 to penalize).
+          no_repeat_ngram_size: Prevent repetitions of ngrams with this size (set 0 to disable).
           temperature: Temperature for sampling. It can be a tuple of temperatures,
             which will be successively used upon failures according to either
             `compression_ratio_threshold` or `log_prob_threshold`.
@@ -206,7 +245,10 @@ class WhisperModel:
             as a prompt for the next window; disabling may make the text inconsistent across
             windows, but the model becomes less prone to getting stuck in a failure loop,
             such as repetition looping or timestamps going out of sync.
-          initial_prompt: Optional text to provide as a prompt for the first window.
+          prompt_reset_on_temperature: Resets prompt if temperature is above this value.
+            Arg has effect only if condition_on_previous_text is True.
+          initial_prompt: Optional text string or iterable of token ids to provide as a
+            prompt for the first window.
           prefix: Optional text to provide as a prefix for the first window.
           suppress_blank: Suppress blank outputs at the beginning of the sampling.
           suppress_tokens: List of token IDs to suppress. -1 will suppress a default set
@@ -222,14 +264,14 @@ class WhisperModel:
           vad_filter: Enable the voice activity detection (VAD) to filter out parts of the audio
             without speech. This step is using the Silero VAD model
             https://github.com/snakers4/silero-vad.
-          vad_parameters: Dictionary of Silero VAD parameters (see available parameters and
-            default values in the function `get_speech_timestamps`).
+          vad_parameters: Dictionary of Silero VAD parameters or VadOptions class (see available
+            parameters and default values in the class `VadOptions`).
 
         Returns:
           A tuple with:
 
             - a generator over transcribed segments
-            - an instance of AudioInfo
+            - an instance of TranscriptionInfo
         """
         sampling_rate = self.feature_extractor.sampling_rate
 
@@ -237,19 +279,24 @@ class WhisperModel:
             audio = decode_audio(audio, sampling_rate=sampling_rate)
 
         duration = audio.shape[0] / sampling_rate
+        duration_after_vad = duration
 
         self.logger.info(
             "Processing audio with duration %s", format_timestamp(duration)
         )
 
         if vad_filter:
-            vad_parameters = {} if vad_parameters is None else vad_parameters
-            speech_chunks = get_speech_timestamps(audio, **vad_parameters)
+            if vad_parameters is None:
+                vad_parameters = VadOptions()
+            elif isinstance(vad_parameters, dict):
+                vad_parameters = VadOptions(**vad_parameters)
+            speech_chunks = get_speech_timestamps(audio, vad_parameters)
             audio = collect_chunks(audio, speech_chunks)
+            duration_after_vad = audio.shape[0] / sampling_rate
 
             self.logger.info(
                 "VAD filter removed %s of audio",
-                format_timestamp(duration - (audio.shape[0] / sampling_rate)),
+                format_timestamp(duration - duration_after_vad),
             )
 
             if self.logger.isEnabledFor(logging.DEBUG):
@@ -271,6 +318,7 @@ class WhisperModel:
         features = self.feature_extractor(audio)
 
         encoder_output = None
+        all_language_probs = None
 
         if language is None:
             if not self.model.is_multilingual:
@@ -279,17 +327,27 @@ class WhisperModel:
             else:
                 segment = features[:, : self.feature_extractor.nb_max_frames]
                 encoder_output = self.encode(segment)
-                results = self.model.detect_language(encoder_output)
-                language_token, language_probability = results[0][0]
-                language = language_token[2:-2]
+                # results is a list of tuple[str, float] with language names and
+                # probabilities.
+                results = self.model.detect_language(encoder_output)[0]
+                # Parse language names to strip out markers
+                all_language_probs = [(token[2:-2], prob) for (token, prob) in results]
+                # Get top language token and probability
+                language, language_probability = all_language_probs[0]
 
                 self.logger.info(
                     "Detected language '%s' with probability %.2f",
                     language,
                     language_probability,
                 )
-                return language, language_probability
         else:
+            if not self.model.is_multilingual and language != "en":
+                self.logger.warning(
+                    "The current model is English-only but the language parameter is set to '%s'; "
+                    "using 'en' instead." % language
+                )
+                language = "en"
+
             language_probability = 1
 
         tokenizer = Tokenizer(
@@ -304,10 +362,13 @@ class WhisperModel:
             best_of=best_of,
             patience=patience,
             length_penalty=length_penalty,
+            repetition_penalty=repetition_penalty,
+            no_repeat_ngram_size=no_repeat_ngram_size,
             log_prob_threshold=log_prob_threshold,
             no_speech_threshold=no_speech_threshold,
             compression_ratio_threshold=compression_ratio_threshold,
             condition_on_previous_text=condition_on_previous_text,
+            prompt_reset_on_temperature=prompt_reset_on_temperature,
             temperatures=(
                 temperature if isinstance(temperature, (list, tuple)) else [temperature]
             ),
@@ -327,13 +388,17 @@ class WhisperModel:
         if speech_chunks:
             segments = restore_speech_timestamps(segments, speech_chunks, sampling_rate)
 
-        audio_info = AudioInfo(
+        info = TranscriptionInfo(
             language=language,
             language_probability=language_probability,
             duration=duration,
+            duration_after_vad=duration_after_vad,
+            transcription_options=options,
+            vad_options=vad_parameters,
+            all_language_probs=all_language_probs,
         )
 
-        return segments
+        return segments, info
 
     def generate_segments(
         self,
@@ -343,14 +408,20 @@ class WhisperModel:
         encoder_output: Optional[ctranslate2.StorageView] = None,
     ) -> Iterable[Segment]:
         content_frames = features.shape[-1] - self.feature_extractor.nb_max_frames
+        idx = 0
         seek = 0
         all_tokens = []
         prompt_reset_since = 0
 
         if options.initial_prompt is not None:
-            initial_prompt = " " + options.initial_prompt.strip()
-            initial_prompt_tokens = tokenizer.encode(initial_prompt)
-            all_tokens.extend(initial_prompt_tokens)
+            if isinstance(options.initial_prompt, str):
+                initial_prompt = " " + options.initial_prompt.strip()
+                initial_prompt_tokens = tokenizer.encode(initial_prompt)
+                all_tokens.extend(initial_prompt_tokens)
+            else:
+                all_tokens.extend(options.initial_prompt)
+
+        last_speech_timestamp = 0.0
         all_segments = []
         while seek < content_frames:
             time_offset = seek * self.feature_extractor.time_per_frame
@@ -373,12 +444,15 @@ class WhisperModel:
                 prefix=options.prefix if seek == 0 else None,
             )
 
-            if encoder_output is None:
+            if seek > 0 or encoder_output is None:
                 encoder_output = self.encode(segment)
 
-            result, avg_log_prob, temperature = self.generate_with_fallback(
-                encoder_output, prompt, tokenizer, options
-            )
+            (
+                result,
+                avg_logprob,
+                temperature,
+                compression_ratio,
+            ) = self.generate_with_fallback(encoder_output, prompt, tokenizer, options)
 
             if options.no_speech_threshold is not None:
                 # no voice activity check
@@ -386,7 +460,7 @@ class WhisperModel:
 
                 if (
                     options.log_prob_threshold is not None
-                    and avg_log_prob > options.log_prob_threshold
+                    and avg_logprob > options.log_prob_threshold
                 ):
                     # don't skip if the logprob is high enough, despite the no_speech_prob
                     should_skip = False
@@ -482,9 +556,6 @@ class WhisperModel:
 
                 seek += segment_size
 
-            if not options.condition_on_previous_text or temperature > 0.5:
-                prompt_reset_since = len(all_tokens)
-
             if options.word_timestamps:
                 self.add_word_timestamps(
                     current_segments,
@@ -493,12 +564,14 @@ class WhisperModel:
                     segment_size,
                     options.prepend_punctuations,
                     options.append_punctuations,
+                    last_speech_timestamp=last_speech_timestamp,
                 )
 
                 word_end_timestamps = [
                     w["end"] for s in current_segments for w in s["words"]
                 ]
-
+                if len(word_end_timestamps) > 0:
+                    last_speech_timestamp = word_end_timestamps[-1]
                 if not single_timestamp_ending and len(word_end_timestamps) > 0:
                     seek_shift = round(
                         (word_end_timestamps[-1] - time_offset) * self.frames_per_second
@@ -507,8 +580,6 @@ class WhisperModel:
                     if seek_shift > 0:
                         seek = previous_seek + seek_shift
 
-            encoder_output = None
-            
             for segment in current_segments:
                 tokens = segment["tokens"]
                 text = tokenizer.decode(tokens)
@@ -517,19 +588,38 @@ class WhisperModel:
                     continue
 
                 all_tokens.extend(tokens)
+                idx += 1
 
                 all_segments.append(Segment(
+                    id=idx,
+                    seek=seek,
                     start=segment["start"],
                     end=segment["end"],
                     text=text,
+                    tokens=tokens,
+                    temperature=temperature,
+                    avg_logprob=avg_logprob,
+                    compression_ratio=compression_ratio,
+                    no_speech_prob=result.no_speech_prob,
                     words=(
                         [Word(**word) for word in segment["words"]]
                         if options.word_timestamps
                         else None
-                    ),
-                    avg_log_prob=avg_log_prob,
-                    no_speech_prob=result.no_speech_prob,
+                        ),
                 ))
+
+            if (
+                not options.condition_on_previous_text
+                or temperature > options.prompt_reset_on_temperature
+            ):
+                if options.condition_on_previous_text:
+                    self.logger.debug(
+                        "Reset prompt. prompt_reset_on_temperature threshold is met %f > %f",
+                        temperature,
+                        options.prompt_reset_on_temperature,
+                    )
+
+                prompt_reset_since = len(all_tokens)
         return all_segments
 
     def encode(self, features: np.ndarray) -> ctranslate2.StorageView:
@@ -548,10 +638,10 @@ class WhisperModel:
         prompt: List[int],
         tokenizer: Tokenizer,
         options: TranscriptionOptions,
-    ) -> Tuple[ctranslate2.models.WhisperGenerationResult, float, float]:
-        result = None
-        avg_log_prob = None
-        final_temperature = None
+    ) -> Tuple[ctranslate2.models.WhisperGenerationResult, float, float, float]:
+        decode_result = None
+        all_results = []
+        below_cr_threshold_results = []
 
         max_initial_timestamp_index = int(
             round(options.max_initial_timestamp / self.time_precision)
@@ -571,11 +661,12 @@ class WhisperModel:
                     "patience": options.patience,
                 }
 
-            final_temperature = temperature
             result = self.model.generate(
                 encoder_output,
                 [prompt],
                 length_penalty=options.length_penalty,
+                repetition_penalty=options.repetition_penalty,
+                no_repeat_ngram_size=options.no_repeat_ngram_size,
                 max_length=self.max_length,
                 return_scores=True,
                 return_no_speech_prob=True,
@@ -589,44 +680,63 @@ class WhisperModel:
 
             # Recover the average log prob from the returned score.
             seq_len = len(tokens)
-            cum_log_prob = result.scores[0] * (seq_len**options.length_penalty)
-            avg_log_prob = cum_log_prob / (seq_len + 1)
+            cum_logprob = result.scores[0] * (seq_len**options.length_penalty)
+            avg_logprob = cum_logprob / (seq_len + 1)
 
             text = tokenizer.decode(tokens).strip()
             compression_ratio = get_compression_ratio(text)
 
+            decode_result = (
+                result,
+                avg_logprob,
+                temperature,
+                compression_ratio,
+            )
+            all_results.append(decode_result)
+
             needs_fallback = False
 
-            if (
-                options.compression_ratio_threshold is not None
-                and compression_ratio > options.compression_ratio_threshold
-            ):
-                needs_fallback = True  # too repetitive
+            if options.compression_ratio_threshold is not None:
+                if compression_ratio > options.compression_ratio_threshold:
+                    needs_fallback = True  # too repetitive
 
-                self.logger.debug(
-                    "Compression ratio threshold is not met with temperature %.1f (%f > %f)",
-                    temperature,
-                    compression_ratio,
-                    options.compression_ratio_threshold,
-                )
+                    self.logger.debug(
+                        "Compression ratio threshold is not met with temperature %.1f (%f > %f)",
+                        temperature,
+                        compression_ratio,
+                        options.compression_ratio_threshold,
+                    )
+                else:
+                    below_cr_threshold_results.append(decode_result)
 
             if (
                 options.log_prob_threshold is not None
-                and avg_log_prob < options.log_prob_threshold
+                and avg_logprob < options.log_prob_threshold
             ):
                 needs_fallback = True  # average log probability is too low
 
                 self.logger.debug(
                     "Log probability threshold is not met with temperature %.1f (%f < %f)",
                     temperature,
-                    avg_log_prob,
+                    avg_logprob,
                     options.log_prob_threshold,
                 )
 
+            if (
+                options.no_speech_threshold is not None
+                and result.no_speech_prob > options.no_speech_threshold
+            ):
+                needs_fallback = False  # silence
+
             if not needs_fallback:
                 break
+        else:
+            # all failed, select the result with the highest average log probability
+            decode_result = max(
+                below_cr_threshold_results or all_results, key=lambda x: x[1]
+            )
 
-        return result, avg_log_prob, final_temperature
+        return decode_result
 
     def get_prompt(
         self,
@@ -650,6 +760,8 @@ class WhisperModel:
             prefix_tokens = tokenizer.encode(" " + prefix.strip())
             if len(prefix_tokens) >= self.max_length // 2:
                 prefix_tokens = prefix_tokens[: self.max_length // 2 - 1]
+            if not without_timestamps:
+                prompt.append(tokenizer.timestamp_begin)
             prompt.extend(prefix_tokens)
 
         return prompt
@@ -662,7 +774,8 @@ class WhisperModel:
         num_frames: int,
         prepend_punctuations: str,
         append_punctuations: str,
-    ):
+        last_speech_timestamp: float,
+    ) -> None:
         if len(segments) == 0:
             return
 
@@ -675,6 +788,24 @@ class WhisperModel:
         alignment = self.find_alignment(
             tokenizer, text_tokens, encoder_output, num_frames
         )
+        word_durations = np.array([word["end"] - word["start"] for word in alignment])
+        word_durations = word_durations[word_durations.nonzero()]
+        median_duration = np.median(word_durations) if len(word_durations) > 0 else 0.0
+        max_duration = median_duration * 2
+
+        # hack: truncate long words at sentence boundaries.
+        # a better segmentation algorithm based on VAD should be able to replace this.
+        if len(word_durations) > 0:
+            sentence_end_marks = ".。!！?？"
+            # ensure words at sentence boundaries
+            # are not longer than twice the median word duration.
+            for i in range(1, len(alignment)):
+                if alignment[i]["end"] - alignment[i]["start"] > max_duration:
+                    if alignment[i]["word"] in sentence_end_marks:
+                        alignment[i]["end"] = alignment[i]["start"] + max_duration
+                    elif alignment[i - 1]["word"] in sentence_end_marks:
+                        alignment[i]["start"] = alignment[i]["end"] - max_duration
+
         merge_punctuations(alignment, prepend_punctuations, append_punctuations)
 
         time_offset = (
@@ -705,10 +836,51 @@ class WhisperModel:
                 saved_tokens += len(timing["tokens"])
                 word_index += 1
 
+            # hack: truncate long words at segment boundaries.
+            # a better segmentation algorithm based on VAD should be able to replace this.
             if len(words) > 0:
-                # adjust the segment-level timestamps based on the word-level timestamps
-                segment["start"] = words[0]["start"]
-                segment["end"] = words[-1]["end"]
+                # ensure the first and second word after a pause is not longer than
+                # twice the median word duration.
+                if words[0]["end"] - last_speech_timestamp > median_duration * 4 and (
+                    words[0]["end"] - words[0]["start"] > max_duration
+                    or (
+                        len(words) > 1
+                        and words[1]["end"] - words[0]["start"] > max_duration * 2
+                    )
+                ):
+                    if (
+                        len(words) > 1
+                        and words[1]["end"] - words[1]["start"] > max_duration
+                    ):
+                        boundary = max(
+                            words[1]["end"] / 2, words[1]["end"] - max_duration
+                        )
+                        words[0]["end"] = words[1]["start"] = boundary
+                    words[0]["start"] = max(0, words[0]["end"] - max_duration)
+
+                # prefer the segment-level start timestamp if the first word is too long.
+                if (
+                    segment["start"] < words[0]["end"]
+                    and segment["start"] - 0.5 > words[0]["start"]
+                ):
+                    words[0]["start"] = max(
+                        0, min(words[0]["end"] - median_duration, segment["start"])
+                    )
+                else:
+                    segment["start"] = words[0]["start"]
+
+                # prefer the segment-level end timestamp if the last word is too long.
+                if (
+                    segment["end"] > words[-1]["start"]
+                    and segment["end"] + 0.5 < words[-1]["end"]
+                ):
+                    words[-1]["end"] = max(
+                        words[-1]["start"] + median_duration, segment["end"]
+                    )
+                else:
+                    segment["end"] = words[-1]["end"]
+
+                last_speech_timestamp = segment["end"]
 
             segment["words"] = words
 
@@ -741,6 +913,8 @@ class WhisperModel:
             text_tokens + [tokenizer.eot]
         )
         word_boundaries = np.pad(np.cumsum([len(t) for t in word_tokens[:-1]]), (1, 0))
+        if len(word_boundaries) <= 1:
+            return []
 
         jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)
         jump_times = time_indices[jumps] / self.tokens_per_second
@@ -751,22 +925,6 @@ class WhisperModel:
             for i, j in zip(word_boundaries[:-1], word_boundaries[1:])
         ]
 
-        # hack: ensure the first and second word is not longer than twice the median word duration.
-        # a better segmentation algorithm based on VAD should be able to replace this.
-        word_durations = end_times - start_times
-        word_durations = word_durations[word_durations.nonzero()]
-        if len(word_durations) > 0:
-            median_duration = np.median(word_durations)
-            max_duration = median_duration * 2
-            if len(word_durations) >= 2 and word_durations[1] > max_duration:
-                boundary = max(end_times[2] / 2, end_times[2] - max_duration)
-                end_times[0] = start_times[1] = boundary
-            if (
-                len(word_durations) >= 1
-                and end_times[0] - start_times[0] > max_duration
-            ):
-                start_times[0] = max(0, end_times[0] - max_duration)
-
         return [
             dict(
                 word=word, tokens=tokens, start=start, end=end, probability=probability
@@ -775,7 +933,7 @@ class WhisperModel:
                 words, word_tokens, start_times, end_times, word_probabilities
             )
         ]
-    
+
     def destroy(self):
         del self.model
 
@@ -792,7 +950,8 @@ def restore_speech_timestamps(
             words = []
             for word in segment.words:
                 # Ensure the word start and end times are resolved to the same chunk.
-                chunk_index = ts_map.get_chunk_index(word.start)
+                middle = (word.start + word.end) / 2
+                chunk_index = ts_map.get_chunk_index(middle)
                 word = word._replace(
                     start=ts_map.get_original_time(word.start, chunk_index),
                     end=ts_map.get_original_time(word.end, chunk_index),
@@ -811,7 +970,7 @@ def restore_speech_timestamps(
                 end=ts_map.get_original_time(segment.end),
             )
 
-        yield segment
+    return segments
 
 
 def get_ctranslate2_storage(segment: np.ndarray) -> ctranslate2.StorageView:
@@ -825,7 +984,10 @@ def get_compression_ratio(text: str) -> float:
     return len(text_bytes) / len(zlib.compress(text_bytes))
 
 
-def get_suppressed_tokens(tokenizer, suppress_tokens):
+def get_suppressed_tokens(
+    tokenizer: Tokenizer,
+    suppress_tokens: Optional[List[int]],
+) -> Optional[List[int]]:
     if not suppress_tokens or -1 in suppress_tokens:
         return suppress_tokens
 
@@ -846,7 +1008,7 @@ def get_suppressed_tokens(tokenizer, suppress_tokens):
     return sorted(set(suppress_tokens))
 
 
-def merge_punctuations(alignment: List[dict], prepended: str, appended: str):
+def merge_punctuations(alignment: List[dict], prepended: str, appended: str) -> None:
     # merge prepended punctuations
     i = len(alignment) - 2
     j = len(alignment) - 1

whisper_live/vad.py

@@ -1,115 +0,0 @@
-# original: https://github.com/snakers4/silero-vad/blob/master/utils_vad.py
-
-import os
-import subprocess
-import torch
-import numpy as np
-import onnxruntime
-
-
-class VoiceActivityDetection():
-
-    def __init__(self, force_onnx_cpu=True):
-        path = self.download()
-        opts = onnxruntime.SessionOptions()
-        opts.log_severity_level = 3
-
-        opts.inter_op_num_threads = 1
-        opts.intra_op_num_threads = 1
-
-        if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
-            self.session = onnxruntime.InferenceSession(path, providers=['CPUExecutionProvider'], sess_options=opts)
-        else:
-            self.session = onnxruntime.InferenceSession(path, providers=['CUDAExecutionProvider'], sess_options=opts)
-
-
-        self.reset_states()
-        self.sample_rates = [8000, 16000]
-
-    def _validate_input(self, x, sr: int):
-        if x.dim() == 1:
-            x = x.unsqueeze(0)
-        if x.dim() > 2:
-            raise ValueError(f"Too many dimensions for input audio chunk {x.dim()}")
-
-        if sr != 16000 and (sr % 16000 == 0):
-            step = sr // 16000
-            x = x[:,::step]
-            sr = 16000
-
-        if sr not in self.sample_rates:
-            raise ValueError(f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)")
-
-        if sr / x.shape[1] > 31.25:
-            raise ValueError("Input audio chunk is too short")
-
-        return x, sr
-
-    def reset_states(self, batch_size=1):
-        self._h = np.zeros((2, batch_size, 64)).astype('float32')
-        self._c = np.zeros((2, batch_size, 64)).astype('float32')
-        self._last_sr = 0
-        self._last_batch_size = 0
-
-    def __call__(self, x, sr: int):
-
-        x, sr = self._validate_input(x, sr)
-        batch_size = x.shape[0]
-
-        if not self._last_batch_size:
-            self.reset_states(batch_size)
-        if (self._last_sr) and (self._last_sr != sr):
-            self.reset_states(batch_size)
-        if (self._last_batch_size) and (self._last_batch_size != batch_size):
-            self.reset_states(batch_size)
-
-        if sr in [8000, 16000]:
-            ort_inputs = {'input': x.numpy(), 'h': self._h, 'c': self._c, 'sr': np.array(sr, dtype='int64')}
-            ort_outs = self.session.run(None, ort_inputs)
-            out, self._h, self._c = ort_outs
-        else:
-            raise ValueError()
-
-        self._last_sr = sr
-        self._last_batch_size = batch_size
-
-        out = torch.tensor(out)
-        return out
-
-    def audio_forward(self, x, sr: int, num_samples: int = 512):
-        outs = []
-        x, sr = self._validate_input(x, sr)
-
-        if x.shape[1] % num_samples:
-            pad_num = num_samples - (x.shape[1] % num_samples)
-            x = torch.nn.functional.pad(x, (0, pad_num), 'constant', value=0.0)
-
-        self.reset_states(x.shape[0])
-        for i in range(0, x.shape[1], num_samples):
-            wavs_batch = x[:, i:i+num_samples]
-            out_chunk = self.__call__(wavs_batch, sr)
-            outs.append(out_chunk)
-
-        stacked = torch.cat(outs, dim=1)
-        return stacked.cpu()
-
-    @staticmethod
-    def download(model_url="https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx"):
-        target_dir = os.path.expanduser("~/.cache/whisper-live/")
-
-        # Ensure the target directory exists
-        os.makedirs(target_dir, exist_ok=True)
-
-        # Define the target file path
-        model_filename = os.path.join(target_dir, "silero_vad.onnx")
-
-        # Check if the model file already exists
-        if not os.path.exists(model_filename):
-            # If it doesn't exist, download the model using wget
-            print("Downloading VAD ONNX model...")
-            try:
-                subprocess.run(["wget", "-O", model_filename, model_url], check=True)
-            except subprocess.CalledProcessError:
-                print("Failed to download the model using wget.")
-        return model_filename
-