bump version v0.1.0

Merge pull request #128 from lightwastak3n/firefox_remove_multilingual
Firefox remove multilingual
2024-02-05 22:37:11 +05:30 · 2024-02-05 21:42:10 +05:30 · 2024-02-05 08:11:01 -05:00 · 2024-02-05 18:40:03 +05:30 · 2024-02-05 18:26:38 +05:30 · 2024-02-04 18:03:41 +01:00
28 changed files with 2134 additions and 650 deletions
@@ -26,9 +26,9 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to
 ### Options
 When using the Audio Transcription extension, you have the following options:
 - **Use Collabora Server**: We provide a demo server which runs the whisper small model.
- - **Use Multilingual Model**: Enable this option to utilize the multilingual capabilities of OpenAI-whisper.
 - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper.
 - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English.
+ - **Model Size**: Select the whisper model size to run the server with.

 ### Getting Started
 - Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live).
@@ -156,7 +156,8 @@ async function startCapture(options) {
          port: options.port, 
          multilingual: options.useMultilingual,
          language: options.language,
-          task: options.task
+          task: options.task,
+          modelSize: options.modelSize
        },
      });
    } else {
@@ -207,13 +208,3 @@ chrome.runtime.onMessage.addListener(async (message) => {
 });


-/**
- * Listens for if the tab is reloaded.
- * @param {Object} message - The message received from the runtime.
- */
-chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, tab) => {
-  if (changeInfo.status === 'complete') {
-    await executeScriptInTab(tabId, "content.js");
-    await delayExecution(500);
-  }
-});
@@ -59,7 +59,7 @@ function init_element() {

    elem_container = document.createElement('div');
    elem_container.id = "transcription";
-    elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';
+    elem_container.style.cssText = 'padding-top:16px;font-size:18px;position: fixed; top: 85%; left: 50%; transform: translate(-50%, -50%);line-height:18px;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';

    for (var i = 0; i < 4; i++) {
        elem_text = document.createElement('span');
@@ -173,13 +173,13 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
    if (type === "STOP") {
        remove_element();
        sendResponse({data: "STOPPED"});
-        return;
+        return true;
    } else if (type === "showWaitPopup"){
        initPopupElement();

        showPopup(`Estimated wait time ~ ${Math.round(data)} minutes`);
        sendResponse({data: "popup"});
-        return;
+        return true;
    }

    init_element();
@@ -234,4 +234,5 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
    }

    sendResponse({});
+    return true;
 });
@@ -93,16 +93,13 @@ async function startRecord(option) {
    const socket = new WebSocket(`ws://${option.host}:${option.port}/`);
    let isServerReady = false;
    let language = option.language;
-    if (language === null && !option.multilingual) {
-      language = 'en';
-    }
    socket.onopen = function(e) { 
      socket.send(
        JSON.stringify({
          uid: uuid,
-          multilingual: option.multilingual,
          language: option.language,
-          task: option.task
+          task: option.task,
+          model: option.modelSize
        })
      );
    };
@@ -184,16 +181,17 @@ async function startRecord(option) {
 * @param {Object} sender - The sender object containing information about the message sender.
 * @param {Function} sendResponse - The function to send a response back to the message sender.
 */
-chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
+chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
  const { type, data } = request;

  switch (type) {
    case "start_capture":
-      await startRecord(data);
+      startRecord(data);
      break;
    default:
      break;
  }

  sendResponse({});
+  return true;
 });
@@ -15,121 +15,134 @@
    <input type="checkbox" id="useServerCheckbox">
    <label for="useServerCheckbox">Use Collabora Whisper-Live Server</label>
  </div>
-  <div class="checkbox-container">
-    <input type="checkbox" id="useMultilingualCheckbox">
-    <label for="useMultilingualCheckbox">Use Multilingual Model</label>
-  </div>
  <div class="dropdown-container">
    <label for="languageDropdown">Select Language:</label>
-    <select id="languageDropdown" disabled>
-      <option value="">Select Language</option>
-      <option value="zh">Chinese</option>
-      <option value="de">German</option>
-      <option value="es">Spanish</option>
-      <option value="ru">Russian</option>
-      <option value="ko">Korean</option>
-      <option value="fr">French</option>
-      <option value="ja">Japanese</option>
-      <option value="pt">Portuguese</option>
-      <option value="tr">Turkish</option>
-      <option value="pl">Polish</option>
-      <option value="ca">Catalan</option>
-      <option value="nl">Dutch</option>
-      <option value="ar">Arabic</option>
-      <option value="sv">Swedish</option>
-      <option value="it">Italian</option>
-      <option value="id">Indonesian</option>
-      <option value="hi">Hindi</option>
-      <option value="fi">Finnish</option>
-      <option value="vi">Vietnamese</option>
-      <option value="he">Hebrew</option>
-      <option value="uk">Ukrainian</option>
-      <option value="el">Greek</option>
-      <option value="ms">Malay</option>
-      <option value="cs">Czech</option>
-      <option value="ro">Romanian</option>
-      <option value="da">Danish</option>
-      <option value="hu">Hungarian</option>
-      <option value="ta">Tamil</option>
-      <option value="no">Norwegian</option>
-      <option value="th">Thai</option>
-      <option value="ur">Urdu</option>
-      <option value="hr">Croatian</option>
-      <option value="bg">Bulgarian</option>
-      <option value="lt">Lithuanian</option>
-      <option value="la">Latin</option>
-      <option value="mi">Maori</option>
-      <option value="ml">Malayalam</option>
-      <option value="cy">Welsh</option>
-      <option value="sk">Slovak</option>
-      <option value="te">Telugu</option>
-      <option value="fa">Persian</option>
-      <option value="lv">Latvian</option>
-      <option value="bn">Bengali</option>
-      <option value="sr">Serbian</option>
-      <option value="az">Azerbaijani</option>
-      <option value="sl">Slovenian</option>
-      <option value="kn">Kannada</option>
-      <option value="et">Estonian</option>
-      <option value="mk">Macedonian</option>
-      <option value="br">Breton</option>
-      <option value="eu">Basque</option>
-      <option value="is">Icelandic</option>
-      <option value="hy">Armenian</option>
-      <option value="ne">Nepali</option>
-      <option value="mn">Mongolian</option>
-      <option value="bs">Bosnian</option>
-      <option value="kk">Kazakh</option>
-      <option value="sq">Albanian</option>
-      <option value="sw">Swahili</option>
-      <option value="gl">Galician</option>
-      <option value="mr">Marathi</option>
-      <option value="pa">Punjabi</option>
-      <option value="si">Sinhala</option>
-      <option value="km">Khmer</option>
-      <option value="sn">Shona</option>
-      <option value="yo">Yoruba</option>
-      <option value="so">Somali</option>
+    <select id="languageDropdown">
+      <option value="" selected>Automatically detect</option>
      <option value="af">Afrikaans</option>
-      <option value="oc">Occitan</option>
-      <option value="ka">Georgian</option>
-      <option value="be">Belarusian</option>
-      <option value="tg">Tajik</option>
-      <option value="sd">Sindhi</option>
-      <option value="gu">Gujarati</option>
+      <option value="sq">Albanian</option>
      <option value="am">Amharic</option>
-      <option value="yi">Yiddish</option>
-      <option value="lo">Lao</option>
-      <option value="uz">Uzbek</option>
-      <option value="fo">Faroese</option>
-      <option value="ht">Haitian Creole</option>
-      <option value="ps">Pashto</option>
-      <option value="tk">Turkmen</option>
-      <option value="nn">Nynorsk</option>
-      <option value="mt">Maltese</option>
-      <option value="sa">Sanskrit</option>
-      <option value="lb">Luxembourgish</option>
-      <option value="my">Myanmar</option>
-      <option value="bo">Tibetan</option>
-      <option value="tl">Tagalog</option>
-      <option value="mg">Malagasy</option>
+      <option value="ar">Arabic</option>
+      <option value="hy">Armenian</option>
      <option value="as">Assamese</option>
-      <option value="tt">Tatar</option>
-      <option value="haw">Hawaiian</option>
-      <option value="ln">Lingala</option>
-      <option value="ha">Hausa</option>
+      <option value="az">Azerbaijani</option>
      <option value="ba">Bashkir</option>
+      <option value="eu">Basque</option>
+      <option value="be">Belarusian</option>
+      <option value="bn">Bengali</option>
+      <option value="bs">Bosnian</option>
+      <option value="br">Breton</option>
+      <option value="bg">Bulgarian</option>
+      <option value="ca">Catalan</option>
+      <option value="zh">Chinese</option>
+      <option value="hr">Croatian</option>
+      <option value="cs">Czech</option>
+      <option value="da">Danish</option>
+      <option value="nl">Dutch</option>
+      <option value="en">English</option>
+      <option value="et">Estonian</option>
+      <option value="fo">Faroese</option>
+      <option value="fi">Finnish</option>
+      <option value="fr">French</option>
+      <option value="gl">Galician</option>
+      <option value="ka">Georgian</option>
+      <option value="de">German</option>
+      <option value="el">Greek</option>
+      <option value="gu">Gujarati</option>
+      <option value="ht">Haitian Creole</option>
+      <option value="ha">Hausa</option>
+      <option value="haw">Hawaiian</option>
+      <option value="he">Hebrew</option>
+      <option value="hi">Hindi</option>
+      <option value="hu">Hungarian</option>
+      <option value="is">Icelandic</option>
+      <option value="id">Indonesian</option>
+      <option value="it">Italian</option>
+      <option value="ja">Japanese</option>
      <option value="jw">Javanese</option>
+      <option value="kn">Kannada</option>
+      <option value="kk">Kazakh</option>
+      <option value="km">Khmer</option>
+      <option value="ko">Korean</option>
+      <option value="lo">Lao</option>
+      <option value="la">Latin</option>
+      <option value="lv">Latvian</option>
+      <option value="ln">Lingala</option>
+      <option value="lt">Lithuanian</option>
+      <option value="lb">Luxembourgish</option>
+      <option value="mk">Macedonian</option>
+      <option value="mg">Malagasy</option>
+      <option value="ms">Malay</option>
+      <option value="ml">Malayalam</option>
+      <option value="mt">Maltese</option>
+      <option value="mi">Maori</option>
+      <option value="mr">Marathi</option>
+      <option value="mn">Mongolian</option>
+      <option value="my">Myanmar</option>
+      <option value="ne">Nepali</option>
+      <option value="no">Norwegian</option>
+      <option value="nn">Nynorsk</option>
+      <option value="oc">Occitan</option>
+      <option value="ps">Pashto</option>
+      <option value="fa">Persian</option>
+      <option value="pl">Polish</option>
+      <option value="pt">Portuguese</option>
+      <option value="pa">Punjabi</option>
+      <option value="ro">Romanian</option>
+      <option value="ru">Russian</option>
+      <option value="sa">Sanskrit</option>
+      <option value="sr">Serbian</option>
+      <option value="sn">Shona</option>
+      <option value="sd">Sindhi</option>
+      <option value="si">Sinhala</option>
+      <option value="sk">Slovak</option>
+      <option value="sl">Slovenian</option>
+      <option value="so">Somali</option>
+      <option value="es">Spanish</option>
      <option value="su">Sundanese</option>
+      <option value="sw">Swahili</option>
+      <option value="sv">Swedish</option>
+      <option value="tl">Tagalog</option>
+      <option value="tg">Tajik</option>
+      <option value="ta">Tamil</option>
+      <option value="tt">Tatar</option>
+      <option value="te">Telugu</option>
+      <option value="th">Thai</option>
+      <option value="bo">Tibetan</option>
+      <option value="tr">Turkish</option>
+      <option value="tk">Turkmen</option>
+      <option value="uk">Ukrainian</option>
+      <option value="ur">Urdu</option>
+      <option value="uz">Uzbek</option>
+      <option value="vi">Vietnamese</option>
+      <option value="cy">Welsh</option>
+      <option value="yi">Yiddish</option>
+      <option value="yo">Yoruba</option>
    </select>
  </div>
  <div class="dropdown-container">
    <label for="taskDropdown">Select task:</label>
-    <select id="taskDropdown" disabled>
+    <select id="taskDropdown" >
      <option value="">Select Task</option>
      <option value="transcribe" selected>Transcribe</option>
      <option value="translate">Translate</option>
    </select>
  </div>
+  <div class="dropdown-container">
+    <label for="modelSizeDropdown">Select Model Size:</label>
+    <select id="modelSizeDropdown">
+      <option value="">Select model</option>
+      <option value="tiny">Tiny </option>
+      <option value="tiny.en">Tiny (English-only)</option>
+      <option value="base">Base</option>
+      <option value="base.en">Base (English-only)</option>
+      <option value="small" selected>Small</option>
+      <option value="small.en">Small (English-only)</option>
+      <option value="medium">Medium</option>
+      <option value="medium.en">Medium (English-only)</option>
+      <option value="large-v2">Large-v2</option>
+      <option value="large-v3">Large-v3</option>
+    </select>
+  </div>
 </body>
 </html>
@@ -4,11 +4,12 @@ document.addEventListener("DOMContentLoaded", function () {
  const stopButton = document.getElementById("stopCapture");

  const useServerCheckbox = document.getElementById("useServerCheckbox");
-  const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
  const languageDropdown = document.getElementById('languageDropdown');
  const taskDropdown = document.getElementById('taskDropdown');
+  const modelSizeDropdown = document.getElementById('modelSizeDropdown');
  let selectedLanguage = null;
  let selectedTask = taskDropdown.value;
+  let selectedModelSize = modelSizeDropdown.value;

  // Add click event listeners to the buttons
  startButton.addEventListener("click", startCapture);
@@ -30,14 +31,6 @@ document.addEventListener("DOMContentLoaded", function () {
    }
  });

-  chrome.storage.local.get("useMultilingualModelState", ({ useMultilingualModelState }) => {
-    if (useMultilingualModelState !== undefined) {
-      useMultilingualCheckbox.checked = useMultilingualModelState;
-      languageDropdown.disabled = !useMultilingualModelState;
-      taskDropdown.disabled = !useMultilingualModelState;
-    }
-  });
-
  chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
    if (storedLanguage !== undefined) {
      languageDropdown.value = storedLanguage;
@@ -52,6 +45,13 @@ document.addEventListener("DOMContentLoaded", function () {
    }
  });

+  chrome.storage.local.get("selectedModelSize", ({ selectedModelSize: storedModelSize }) => {
+    if (storedModelSize !== undefined) {
+      modelSizeDropdown.value = storedModelSize;
+      selectedModelSize = storedModelSize;
+    }
+  });
+
  // Function to handle the start capture button click event
  async function startCapture() {
    // Ignore click if the button is disabled
@@ -77,9 +77,9 @@ document.addEventListener("DOMContentLoaded", function () {
        tabId: currentTab.id,
        host: host,
        port: port,
-        useMultilingual: useMultilingualCheckbox.checked,
        language: selectedLanguage,
-        task: selectedTask
+        task: selectedTask,
+        modelSize: selectedModelSize
      }, () => {
        // Update capturing state in storage and toggle the buttons
        chrome.storage.local.set({ capturingState: { isCapturing: true } }, () => {
@@ -119,8 +119,9 @@ document.addEventListener("DOMContentLoaded", function () {
    startButton.disabled = isCapturing;
    stopButton.disabled = !isCapturing;
    useServerCheckbox.disabled = isCapturing; 
-    useMultilingualCheckbox.disabled = isCapturing;
-    
+    modelSizeDropdown.disabled = isCapturing;
+    languageDropdown.disabled = isCapturing;
+    taskDropdown.disabled = isCapturing; 
    startButton.classList.toggle("disabled", isCapturing);
    stopButton.classList.toggle("disabled", !isCapturing);
  }
@@ -131,18 +132,6 @@ document.addEventListener("DOMContentLoaded", function () {
    chrome.storage.local.set({ useServerState });
  });

-  useMultilingualCheckbox.addEventListener('change', function() {
-    const useMultilingualModelState = useMultilingualCheckbox.checked;
-    if (useMultilingualModelState) {
-      languageDropdown.disabled = false;
-      taskDropdown.disabled = false;
-    } else {
-      languageDropdown.disabled = true;
-      taskDropdown.disabled = true;
-    }
-    chrome.storage.local.set({ useMultilingualModelState });
-  });
-
  languageDropdown.addEventListener('change', function() {
    if (languageDropdown.value === "") {
      selectedLanguage = null;
@@ -157,6 +146,11 @@ document.addEventListener("DOMContentLoaded", function () {
    chrome.storage.local.set({ selectedTask });
  });

+  modelSizeDropdown.addEventListener('change', function() {
+    selectedModelSize = modelSizeDropdown.value;
+    chrome.storage.local.set({ selectedModelSize });
+  });
+
  chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
    if (request.action === "updateSelectedLanguage") {
      const detectedLanguage = request.detectedLanguage;
@@ -24,9 +24,9 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to
 ### Options
 When using the Audio Transcription extension, you have the following options:
 - **Use Collabora Server**: We provide a demo server which runs the whisper small model.
- - **Use Multilingual Model**: Enable this option to utilize the multilingual capabilities of OpenAI-whisper.
 - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper.
 - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English.
+  - **Model Size**: Select the whisper model size to run the server with.

 ### Getting Started
 - Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live).
@@ -66,18 +66,15 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) {
 function startRecording(data) {
    socket = new WebSocket(`ws://${data.host}:${data.port}/`);
    language = data.language;
-    if (language === null && !data.useMultilingual) {
-      language = 'en';
-    }

    const uuid = generateUUID();
    socket.onopen = function(e) { 
      socket.send(
        JSON.stringify({
            uid: uuid,
-            multilingual: data.useMultilingual,
            language: data.language,
-            task: data.task
+            task: data.task,
+            model: data.modelSize
        })
      );
    };
@@ -200,7 +197,7 @@ function init_element() {

    elem_container = document.createElement('div');
    elem_container.id = "transcription";
-    elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';
+    elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;position:fixed;top:85%;left:50%;transform:translate(-50%,-50%);width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';

    for (var i = 0; i < 4; i++) {
        elem_text = document.createElement('span');
@@ -16,113 +16,109 @@
    <label for="useServerCheckbox">Use Collabora Whisper-Live Server</label>
  </div>
  <textarea id="waitTextBox" style="display: none;"></textarea>
-
-  <div class="checkbox-container">
-    <input type="checkbox" id="useMultilingualCheckbox">
-    <label for="useMultilingualCheckbox">Use Multilingual Model</label>
-  </div>
  <div class="dropdown-container">
    <label for="languageDropdown">Select Language:</label>
-    <select id="languageDropdown" disabled>
-      <option value="">Select Language</option>
-      <option value="zh">Chinese</option>
-      <option value="de">German</option>
-      <option value="es">Spanish</option>
-      <option value="ru">Russian</option>
-      <option value="ko">Korean</option>
-      <option value="fr">French</option>
-      <option value="ja">Japanese</option>
-      <option value="pt">Portuguese</option>
-      <option value="tr">Turkish</option>
-      <option value="pl">Polish</option>
-      <option value="ca">Catalan</option>
-      <option value="nl">Dutch</option>
-      <option value="ar">Arabic</option>
-      <option value="sv">Swedish</option>
-      <option value="it">Italian</option>
-      <option value="id">Indonesian</option>
-      <option value="hi">Hindi</option>
-      <option value="fi">Finnish</option>
-      <option value="vi">Vietnamese</option>
-      <option value="he">Hebrew</option>
-      <option value="uk">Ukrainian</option>
-      <option value="el">Greek</option>
-      <option value="ms">Malay</option>
-      <option value="cs">Czech</option>
-      <option value="ro">Romanian</option>
-      <option value="da">Danish</option>
-      <option value="hu">Hungarian</option>
-      <option value="ta">Tamil</option>
-      <option value="no">Norwegian</option>
-      <option value="th">Thai</option>
-      <option value="ur">Urdu</option>
-      <option value="hr">Croatian</option>
-      <option value="bg">Bulgarian</option>
-      <option value="lt">Lithuanian</option>
-      <option value="la">Latin</option>
-      <option value="mi">Maori</option>
-      <option value="ml">Malayalam</option>
-      <option value="cy">Welsh</option>
-      <option value="sk">Slovak</option>
-      <option value="te">Telugu</option>
-      <option value="fa">Persian</option>
-      <option value="lv">Latvian</option>
-      <option value="bn">Bengali</option>
-      <option value="sr">Serbian</option>
-      <option value="az">Azerbaijani</option>
-      <option value="sl">Slovenian</option>
-      <option value="kn">Kannada</option>
-      <option value="et">Estonian</option>
-      <option value="mk">Macedonian</option>
-      <option value="br">Breton</option>
-      <option value="eu">Basque</option>
-      <option value="is">Icelandic</option>
-      <option value="hy">Armenian</option>
-      <option value="ne">Nepali</option>
-      <option value="mn">Mongolian</option>
-      <option value="bs">Bosnian</option>
-      <option value="kk">Kazakh</option>
-      <option value="sq">Albanian</option>
-      <option value="sw">Swahili</option>
-      <option value="gl">Galician</option>
-      <option value="mr">Marathi</option>
-      <option value="pa">Punjabi</option>
-      <option value="si">Sinhala</option>
-      <option value="km">Khmer</option>
-      <option value="sn">Shona</option>
-      <option value="yo">Yoruba</option>
-      <option value="so">Somali</option>
+    <select id="languageDropdown">
+      <option value="" selected>Automatically detect</option>
      <option value="af">Afrikaans</option>
-      <option value="oc">Occitan</option>
-      <option value="ka">Georgian</option>
-      <option value="be">Belarusian</option>
-      <option value="tg">Tajik</option>
-      <option value="sd">Sindhi</option>
-      <option value="gu">Gujarati</option>
+      <option value="sq">Albanian</option>
      <option value="am">Amharic</option>
-      <option value="yi">Yiddish</option>
-      <option value="lo">Lao</option>
-      <option value="uz">Uzbek</option>
-      <option value="fo">Faroese</option>
-      <option value="ht">Haitian Creole</option>
-      <option value="ps">Pashto</option>
-      <option value="tk">Turkmen</option>
-      <option value="nn">Nynorsk</option>
-      <option value="mt">Maltese</option>
-      <option value="sa">Sanskrit</option>
-      <option value="lb">Luxembourgish</option>
-      <option value="my">Myanmar</option>
-      <option value="bo">Tibetan</option>
-      <option value="tl">Tagalog</option>
-      <option value="mg">Malagasy</option>
+      <option value="ar">Arabic</option>
+      <option value="hy">Armenian</option>
      <option value="as">Assamese</option>
-      <option value="tt">Tatar</option>
-      <option value="haw">Hawaiian</option>
-      <option value="ln">Lingala</option>
-      <option value="ha">Hausa</option>
+      <option value="az">Azerbaijani</option>
      <option value="ba">Bashkir</option>
+      <option value="eu">Basque</option>
+      <option value="be">Belarusian</option>
+      <option value="bn">Bengali</option>
+      <option value="bs">Bosnian</option>
+      <option value="br">Breton</option>
+      <option value="bg">Bulgarian</option>
+      <option value="ca">Catalan</option>
+      <option value="zh">Chinese</option>
+      <option value="hr">Croatian</option>
+      <option value="cs">Czech</option>
+      <option value="da">Danish</option>
+      <option value="nl">Dutch</option>
+      <option value="en">English</option>
+      <option value="et">Estonian</option>
+      <option value="fo">Faroese</option>
+      <option value="fi">Finnish</option>
+      <option value="fr">French</option>
+      <option value="gl">Galician</option>
+      <option value="ka">Georgian</option>
+      <option value="de">German</option>
+      <option value="el">Greek</option>
+      <option value="gu">Gujarati</option>
+      <option value="ht">Haitian Creole</option>
+      <option value="ha">Hausa</option>
+      <option value="haw">Hawaiian</option>
+      <option value="he">Hebrew</option>
+      <option value="hi">Hindi</option>
+      <option value="hu">Hungarian</option>
+      <option value="is">Icelandic</option>
+      <option value="id">Indonesian</option>
+      <option value="it">Italian</option>
+      <option value="ja">Japanese</option>
      <option value="jw">Javanese</option>
+      <option value="kn">Kannada</option>
+      <option value="kk">Kazakh</option>
+      <option value="km">Khmer</option>
+      <option value="ko">Korean</option>
+      <option value="lo">Lao</option>
+      <option value="la">Latin</option>
+      <option value="lv">Latvian</option>
+      <option value="ln">Lingala</option>
+      <option value="lt">Lithuanian</option>
+      <option value="lb">Luxembourgish</option>
+      <option value="mk">Macedonian</option>
+      <option value="mg">Malagasy</option>
+      <option value="ms">Malay</option>
+      <option value="ml">Malayalam</option>
+      <option value="mt">Maltese</option>
+      <option value="mi">Maori</option>
+      <option value="mr">Marathi</option>
+      <option value="mn">Mongolian</option>
+      <option value="my">Myanmar</option>
+      <option value="ne">Nepali</option>
+      <option value="no">Norwegian</option>
+      <option value="nn">Nynorsk</option>
+      <option value="oc">Occitan</option>
+      <option value="ps">Pashto</option>
+      <option value="fa">Persian</option>
+      <option value="pl">Polish</option>
+      <option value="pt">Portuguese</option>
+      <option value="pa">Punjabi</option>
+      <option value="ro">Romanian</option>
+      <option value="ru">Russian</option>
+      <option value="sa">Sanskrit</option>
+      <option value="sr">Serbian</option>
+      <option value="sn">Shona</option>
+      <option value="sd">Sindhi</option>
+      <option value="si">Sinhala</option>
+      <option value="sk">Slovak</option>
+      <option value="sl">Slovenian</option>
+      <option value="so">Somali</option>
+      <option value="es">Spanish</option>
      <option value="su">Sundanese</option>
+      <option value="sw">Swahili</option>
+      <option value="sv">Swedish</option>
+      <option value="tl">Tagalog</option>
+      <option value="tg">Tajik</option>
+      <option value="ta">Tamil</option>
+      <option value="tt">Tatar</option>
+      <option value="te">Telugu</option>
+      <option value="th">Thai</option>
+      <option value="bo">Tibetan</option>
+      <option value="tr">Turkish</option>
+      <option value="tk">Turkmen</option>
+      <option value="uk">Ukrainian</option>
+      <option value="ur">Urdu</option>
+      <option value="uz">Uzbek</option>
+      <option value="vi">Vietnamese</option>
+      <option value="cy">Welsh</option>
+      <option value="yi">Yiddish</option>
+      <option value="yo">Yoruba</option>
    </select>
  </div>
  <div class="dropdown-container">
@@ -133,5 +129,21 @@
      <option value="translate">Translate</option>
    </select>
  </div>
+  <div class="dropdown-container">
+    <label for="modelSizeDropdown">Select Model Size:</label>
+    <select id="modelSizeDropdown">
+      <option value="">Select model</option>
+      <option value="tiny">Tiny </option>
+      <option value="tiny.en">Tiny (English-only)</option>
+      <option value="base">Base</option>
+      <option value="base.en">Base (English-only)</option>
+      <option value="small" selected>Small</option>
+      <option value="small.en">Small (English-only)</option>
+      <option value="medium">Medium</option>
+      <option value="medium.en">Medium (English-only)</option>
+      <option value="large-v2">Large-v2</option>
+      <option value="large-v3">Large-v3</option>
+    </select>
+  </div>
 </body>
-</html>
+</html>
@@ -3,11 +3,13 @@ document.addEventListener("DOMContentLoaded", function() {
  const stopButton = document.getElementById("stopCapture");

  const useServerCheckbox = document.getElementById("useServerCheckbox");
-  const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
  const languageDropdown = document.getElementById('languageDropdown');
  const taskDropdown = document.getElementById('taskDropdown');
+  const modelSizeDropdown = document.getElementById('modelSizeDropdown');
  let selectedLanguage = null;
  let selectedTask = taskDropdown.value;
+  let selectedModelSize = modelSizeDropdown.value;
+  

  browser.storage.local.get("capturingState")
    .then(function(result) {
@@ -32,14 +34,6 @@ document.addEventListener("DOMContentLoaded", function() {
    }
  });

-  browser.storage.local.get("useMultilingualModelState", ({ useMultilingualModelState }) => {
-    if (useMultilingualModelState !== undefined) {
-      useMultilingualCheckbox.checked = useMultilingualModelState;
-      languageDropdown.disabled = !useMultilingualModelState;
-      taskDropdown.disabled = !useMultilingualModelState;
-    }
-  });
-
  browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
    if (storedLanguage !== undefined) {
      languageDropdown.value = storedLanguage;
@@ -54,6 +48,13 @@ document.addEventListener("DOMContentLoaded", function() {
    }
  });

+  browser.storage.local.get("selectedModelSize", ({ selectedModelSize: storedModelSize }) => {
+    if (storedModelSize !== undefined) {
+      modelSizeDropdown.value = storedModelSize;
+      selectedModelSize = storedModelSize;
+    }
+  });
+
  startButton.addEventListener("click", function() {
    let host = "localhost";
    let port = "9090";
@@ -73,9 +74,9 @@ document.addEventListener("DOMContentLoaded", function() {
            data: {
              host: host,
              port: port,
-              useMultilingual: useMultilingualCheckbox.checked,
              language: selectedLanguage,
-              task: selectedTask
+              task: selectedTask,
+              modelSize: selectedModelSize
            } 
          });
        toggleCaptureButtons(true);
@@ -114,8 +115,9 @@ document.addEventListener("DOMContentLoaded", function() {
    startButton.disabled = isCapturing;
    stopButton.disabled = !isCapturing;
    useServerCheckbox.disabled = isCapturing;
-    useMultilingualCheckbox.disabled = isCapturing;
-
+    modelSizeDropdown.disabled = isCapturing;
+    languageDropdown.disabled = isCapturing;
+    taskDropdown.disabled = isCapturing; 
    startButton.classList.toggle("disabled", isCapturing);
    stopButton.classList.toggle("disabled", !isCapturing);
  }
@@ -126,18 +128,6 @@ document.addEventListener("DOMContentLoaded", function() {
    browser.storage.local.set({ useServerState });
  });

-  useMultilingualCheckbox.addEventListener('change', function() {
-    const useMultilingualModelState = useMultilingualCheckbox.checked;
-    if (useMultilingualModelState) {
-      languageDropdown.disabled = false;
-      taskDropdown.disabled = false;
-    } else {
-      languageDropdown.disabled = true;
-      taskDropdown.disabled = true;
-    }
-    browser.storage.local.set({ useMultilingualModelState });
-  });
-
  languageDropdown.addEventListener('change', function() {
    if (languageDropdown.value === "") {
      selectedLanguage = null;
@@ -152,6 +142,11 @@ document.addEventListener("DOMContentLoaded", function() {
    browser.storage.local.set({ selectedTask });
  });

+  modelSizeDropdown.addEventListener('change', function() {
+    selectedModelSize = modelSizeDropdown.value;
+    browser.storage.local.set({ selectedModelSize });
+  });
+
  browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
    if (request.action === "updateSelectedLanguage") {
      const detectedLanguage = request.data;
@@ -108,4 +108,4 @@ label {

 .dropdown-container {
  padding: 10px; 
-}
+}
@@ -8,7 +8,7 @@ Unlike traditional speech recognition systems that rely on continuous audio stre
 ## Installation
 - Install PyAudio and ffmpeg
 ```bash
- bash setup.sh
+ bash scripts/setup.sh
 ```

 - Install whisper-live from pip
@@ -16,40 +16,79 @@ Unlike traditional speech recognition systems that rely on continuous audio stre
 pip install whisper-live
 ```

+### Setting up NVIDIA/TensorRT-LLM for TensorRT backend
+- Please follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) for setup of [NVIDIA/TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) and for building Whisper-TensorRT engine.
+
 ## Getting Started
- Run the server
-```python
- from whisper_live.server import TranscriptionServer
- server = TranscriptionServer()
- server.run("0.0.0.0", 9090)
+The server supports two backends `faster_whisper` and `tensorrt`. If running `tensorrt` backend follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md)
+
+### Running the Server
+- [Faster Whisper](https://github.com/SYSTRAN/faster-whisper) backend
+```bash
+python3 run_server.py --port 9090 \
+                      --backend faster_whisper
+  
+# running with custom model
+python3 run_server.py --port 9090 \
+                      --backend faster_whisper
+                      -fw "/path/to/custom/faster/whisper/model"
 ```

- On the client side
-    - To transcribe an audio file:
-    ```python
-      from whisper_live.client import TranscriptionClient
-      client = TranscriptionClient("localhost", 9090, is_multilingual=True, lang="hi", translate=True)
-      client(audio_file_path)
-    ```
-    This command transcribes the specified audio file (audio.wav) using the Whisper model. It connects to the server running on localhost at port 9090. It also enables the multilingual feature, allowing transcription in multiple languages. The language option specifies the target language for transcription, in this case, Hindi ("hi"). The translate option should be set to `True` if we want to translate from the source language to English and `False` if we want to transcribe in the source language.
+- TensorRT backend. Currently, we recommend to only use the docker setup for TensorRT. Follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) which works as expected. Make sure to build your TensorRT Engines before running the server with TensorRT backend.
+```bash
+# Run English only model
+python3 run_server.py -p 9090 \
+                      -b tensorrt \
+                      -trt /home/TensorRT-LLM/examples/whisper/whisper_small_en

-    - To transcribe from microphone:
-    ```python
-      from whisper_live.client import TranscriptionClient
-      client = TranscriptionClient(host, port, is_multilingual=True, lang="hi", translate=True)
-      client()
-    ```
-    This command captures audio from the microphone and sends it to the server for transcription. It uses the same options as the previous command, enabling the multilingual feature and specifying the target language and task.
+# Run Multilingual model
+python3 run_server.py -p 9090 \
+                      -b tensorrt \
+                      -trt /home/TensorRT-LLM/examples/whisper/whisper_small \
+                      -m
+```


+### Running the Client
+- To transcribe an audio file:
+```python
+from whisper_live.client import TranscriptionClient
+client = TranscriptionClient(
+  "localhost",
+  9090,
+  lang="en",
+  translate=False,
+  model="small"
+)
+
+client("tests/jfk.wav")
+```
+This command transcribes the specified audio file (audio.wav) using the Whisper model. It connects to the server running on localhost at port 9090. Using a multilingual model, language for the transcription will be automatically detected. You can also use the language option to specify the target language for the transcription, in this case, English ("en"). The translate option should be set to `True` if we want to translate from the source language to English and `False` if we want to transcribe in the source language.
+
+- To transcribe from microphone:
+```python
+from whisper_live.client import TranscriptionClient
+client = TranscriptionClient(
+  "localhost",
+  9090,
+  lang="hi",
+  translate=True,
+  model="small"
+)
+client()
+```
+This command captures audio from the microphone and sends it to the server for transcription. It uses the multilingual model with `hi` as the selected language. We use whisper `small` by default but can be changed to any other option based on the requirements and the hardware running the server.
+
+- To transcribe from a HLS stream:
+```python
+from whisper_live.client import TranscriptionClient
+client = TranscriptionClient(host, port, lang="en", translate=False) 
+client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/bbc_1xtra.isml/bbc_1xtra-audio%3d96000.norewind.m3u8") 
+```
+This command streams audio into the server from a HLS stream. It uses the same options as the previous command, using the multilingual model and specifying the target language and task.
+
 ## Transcribe audio from browser
- Run the server
-```python
- from whisper_live.server import TranscriptionServer
- server = TranscriptionServer()
- server.run("0.0.0.0", 9090)
-```
-This would start the websocket server on port ```9090```.
+- Run the server with your desired backend as shown [here](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server)

 ### Chrome Extension
 - Refer to [Audio-Transcription-Chrome](https://github.com/collabora/whisper-live/tree/main/Audio-Transcription-Chrome#readme) to use Chrome extension.
@@ -59,21 +98,24 @@ This would start the websocket server on port ```9090```.

 ## Whisper Live Server in Docker
 - GPU
-```bash
- docker build . -t whisper-live -f docker/Dockerfile.gpu
- docker run -it --gpus all -p 9090:9090 whisper-live:latest
-```
+  - Faster-Whisper
+  ```bash
+  docker build . -t whisper-live -f docker/Dockerfile.gpu
+  docker run -it --gpus all -p 9090:9090 whisper-live:latest
+  ```
+
+  - TensorRT. Follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) in order to setup docker and use TensorRT backend. We provide a pre-built docker image which has TensorRT-LLM built and ready to use.

 - CPU
 ```bash
- docker build . -t whisper-live -f docker/Dockerfile.cpu
- docker run -it -p 9090:9090 whisper-live:latest
+docker build . -t whisper-live -f docker/Dockerfile.cpu
+docker run -it -p 9090:9090 whisper-live:latest
 ```
 **Note**: By default we use "small" model size. To build docker image for a different model size, change the size in server.py and then build the docker image.

 ## Future Work
 - [ ] Add translation to other languages on top of transcription.
- [ ] TensorRT backend for Whisper.
+- [x] TensorRT backend for Whisper.

 ## Contact

@@ -0,0 +1,67 @@
+# Whisper-TensorRT
+We have only tested the TensorRT backend in docker so, we recommend docker for a smooth TensorRT backend setup.
+**Note**: We use [our fork to setup TensorRT](https://github.com/makaveli10/TensorRT-LLM)
+
+## Installation
+- Install [docker](https://docs.docker.com/engine/install/)
+- Install [nvidia-container-toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
+
+- Clone this repo.
+```bash
+git clone https://github.com/collabora/WhisperLive.git
+cd WhisperLive
+```
+
+- Pull the TensorRT-LLM docker image which we prebuilt for WhisperLive TensorRT backend.
+```bash
+docker pull ghcr.io/collabora/whisperbot-base:latest
+```
+
+- Next, we run the docker image and mount WhisperLive repo to the containers `/home` directory.
+```bash
+docker run -it --gpus all --shm-size=8g \
+       --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \
+       -v /path/to/WhisperLive:/home/WhisperLive \
+       ghcr.io/collabora/whisperbot-base:latest
+```
+
+- Make sure to test the installation. 
+```bash
+# export ENV=${ENV:-/etc/shinit_v2} 
+# source $ENV
+python -c "import torch; import tensorrt; import tensorrt_llm"
+```
+**NOTE**: Uncomment and update library paths if imports fail.
+
+## Whisper TensorRT Engine
+- We build `small.en` and `small` multilingual TensorRT engine. The script logs the path of the directory with Whisper TensorRT engine. We need the model_path to run the server.
+```bash
+# convert small.en
+bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small.en
+
+# convert small multilingual model
+bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small
+```
+
+## Run WhisperLive Server with TensorRT Backend
+```bash
+cd /home/WhisperLive
+
+# Install requirements
+bash scripts/setup.sh
+pip install -r requirements/server.txt
+
+# Required to create mel spectogram
+wget --directory-prefix=assets assets/mel_filters.npz https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz
+
+# Run English only model
+python3 run_server.py --port 9090 \
+                      --backend tensorrt \
+                      --trt_model_path "path/to/whisper_trt/from/build/step"
+
+# Run Multilingual model
+python3 run_server.py --port 9090 \
+                      --backend tensorrt \
+                      --trt_model_path "path/to/whisper_trt/from/build/step" \
+                      --trt_multilingual
+```
@@ -33,7 +33,7 @@ RUN apt install python3-pip -y
 RUN mkdir /app
 WORKDIR /app

-COPY setup.sh /app
+COPY scripts/setup.sh /app
 COPY requirements/ /app

 RUN bash setup.sh
@@ -33,7 +33,7 @@ RUN apt install python3-pip -y
 RUN mkdir /app
 WORKDIR /app

-COPY setup.sh /app
+COPY scripts/setup.sh /app
 COPY requirements/ /app

 RUN apt update --fix-missing
@@ -1,7 +1,10 @@
-PyAudio
-faster-whisper==0.6.0
--extra-index-url https://download.pytorch.org/whl/cu111
-torch==1.10.1
-torchaudio==0.10.1
+faster-whisper==0.10.0
+torch
 websockets
-onnxruntime==1.16.0
+onnxruntime==1.16.0
+numba
+openai-whisper
+kaldialign
+soundfile
+ffmpeg-python
+scipy
@@ -1,5 +1,38 @@
+import argparse
 from whisper_live.server import TranscriptionServer

 if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--port', '-p',
+                        type=int, 
+                        default=9090,
+                        help="Websocket port to run the server on.")
+    parser.add_argument('--backend', '-b',
+                        type=str, 
+                        default='faster_whisper', 
+                        help='Backends from ["tensorrt", "faster_whisper"]')
+    parser.add_argument('--faster_whisper_custom_model_path', '-fw',
+                        type=str, default=None, 
+                        help="Custom Faster Whisper Model")
+    parser.add_argument('--trt_model_path', '-trt',
+                        type=str,
+                        default=None,
+                        help='Whisper TensorRT model path')
+    parser.add_argument('--trt_multilingual', '-m',
+                        action="store_true",
+                        help='Boolean only for TensorRT model. True if multilingual.')
+    args = parser.parse_args()
+
+    if args.backend == "tensorrt":
+        if args.trt_model_path is None:
+            raise ValueError("Please Provide a valid tensorrt model path")
+
    server = TranscriptionServer()
-    server.run("0.0.0.0")
+    server.run(
+        "0.0.0.0",
+        port=args.port, 
+        backend=args.backend,
+        faster_whisper_custom_model_path=args.faster_whisper_custom_model_path,
+        whisper_tensorrt_path=args.trt_model_path,
+        trt_multilingual=args.trt_multilingual
+    )
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+download_and_build_model() {
+    local model_name="$1"
+    local model_url=""
+
+    case "$model_name" in
+        "tiny.en")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt"
+            ;;
+        "tiny")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt"
+            ;;
+        "base.en")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt"
+            ;;
+        "base")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt"
+            ;;
+        "small.en")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt"
+            ;;
+        "small")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt"
+            ;;
+        "medium.en")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt"
+            ;;
+        "medium")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt"
+            ;;
+        "large-v1")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large-v1.pt"
+            ;;
+        "large-v2")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt"
+            ;;
+        "large-v3" | "large")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt"
+            ;;
+        *)
+            echo "Invalid model name: $model_name"
+            exit 1
+            ;;
+    esac
+
+    echo "Downloading $model_name..."
+    # wget --directory-prefix=assets "$model_url"
+    # echo "Download completed: ${model_name}.pt"
+    if [ ! -f "assets/${model_name}.pt" ]; then
+        wget --directory-prefix=assets "$model_url"
+        echo "Download completed: ${model_name}.pt"
+    else
+        echo "${model_name}.pt already exists in assets directory."
+    fi
+
+    local output_dir="whisper_${model_name//./_}"
+    echo "$output_dir"
+    echo "Running build script for $model_name with output directory $output_dir"
+    python3 build.py --output_dir "$output_dir" --use_gpt_attention_plugin --use_gemm_plugin --use_bert_attention_plugin --model_name "$model_name"
+    echo "Whisper $model_name TensorRT engine built."
+    echo "========================================="
+    echo "Model is located at: $(pwd)/$output_dir"
+}
+
+if [ "$#" -lt 1 ]; then
+    echo "Usage: $0 <path-to-tensorrt-examples-dir> [model-name]"
+    exit 1
+fi
+
+tensorrt_examples_dir="$1"
+model_name="${2:-small.en}"
+
+cd $1/whisper
+pip install --no-deps -r requirements.txt
+
+download_and_build_model "$model_name"
@@ -41,14 +41,18 @@ setup(name="whisper-live",
      ),
      install_requires=[
        "PyAudio",
-        "faster-whisper==0.6.0",
+        "faster-whisper==0.10.0",
        "torch",
        "torchaudio",
        "websockets",
-        "onnxruntime",
+        "onnxruntime==1.16.0",
        "ffmpeg-python",
        "scipy",
        "websocket-client",
+        "numba",
+        "openai-whisper",
+        "kaldialign",
+        "soundfile",
      ],
      python_requires=">=3.8"
-)
+)
@@ -1 +1 @@
-__version__="0.0.7"
+__version__="0.1.0"
@@ -13,6 +13,29 @@ import uuid
 import time


+def format_time(s):
+    """Convert seconds (float) to SRT time format."""
+    hours = int(s // 3600)
+    minutes = int((s % 3600) // 60)
+    seconds = int(s % 60)
+    milliseconds = int((s - int(s)) * 1000)
+    return f"{hours:02}:{minutes:02}:{seconds:02},{milliseconds:03}"
+
+def create_srt_file(segments, output_file):
+    with open(output_file, 'w', encoding='utf-8') as srt_file:
+        segment_number = 1
+        for segment in segments:
+            start_time = format_time(float(segment['start']))
+            end_time = format_time(float(segment['end']))
+            text = segment['text']
+            
+            srt_file.write(f"{segment_number}\n")
+            srt_file.write(f"{start_time} --> {end_time}\n")
+            srt_file.write(f"{text}\n\n")
+            
+            segment_number += 1
+
+
 def resample(file: str, sr: int = 16000):
    """
    # https://github.com/openai/whisper/blob/7858aa9c08d98f75575035ecd6481f462d66ca27/whisper/audio.py#L22
@@ -50,7 +73,13 @@ class Client:
    INSTANCES = {}

    def __init__(
-        self, host=None, port=None, is_multilingual=False, lang=None, translate=False
+        self,
+        host=None,
+        port=None,
+        lang=None,
+        translate=False,
+        model="small",
+        srt_file_path="output.srt"
    ):
        """
        Initializes a Client instance for audio recording and streaming to a server.
@@ -62,25 +91,25 @@ class Client:
        Args:
            host (str): The hostname or IP address of the server.
            port (int): The port number for the WebSocket server.
-            is_multilingual (bool, optional): Specifies if multilingual transcription is enabled. Default is False.
-            lang (str, optional): The selected language for transcription when multilingual is disabled. Default is None.
+            lang (str, optional): The selected language for transcription. Default is None.
            translate (bool, optional): Specifies if the task is translation. Default is False.
        """
-        self.chunk = 1024
+        self.chunk = 4096
        self.format = pyaudio.paInt16
        self.channels = 1
        self.rate = 16000
        self.record_seconds = 60000
        self.recording = False
-        self.multilingual = False
-        self.language = None
        self.task = "transcribe"
        self.uid = str(uuid.uuid4())
        self.waiting = False
        self.last_response_recieved = None
        self.disconnect_if_no_response_for = 15
-        self.multilingual = is_multilingual
-        self.language = lang if is_multilingual else "en"
+        self.language = lang
+        self.model = model
+        self.server_error = False
+        self.srt_file_path = srt_file_path
+
        if translate:
            self.task = "translate"

@@ -118,6 +147,7 @@ class Client:
        self.ws_thread.start()

        self.frames = b""
+        self.transcript = []
        print("[INFO]: * recording")

    def on_message(self, ws, message):
@@ -140,11 +170,16 @@ class Client:
            print("[ERROR]: invalid client uid")
            return

-        if "status" in message.keys() and message["status"] == "WAIT":
-            self.waiting = True
-            print(
-                f"[INFO]:Server is full. Estimated wait time {round(message['message'])} minutes."
-            )
+        if "status" in message.keys():
+            if message["status"] == "WAIT":
+                self.waiting = True
+                print(
+                    f"[INFO]:Server is full. Estimated wait time {round(message['message'])} minutes."
+                )
+            elif message["status"] == "ERROR":
+                print(f"Message from Server: {message['message']}")
+                self.server_error = True
+            return

        if "message" in message.keys() and message["message"] == "DISCONNECT":
            print("[INFO]: Server overtime disconnected.")
@@ -152,6 +187,8 @@ class Client:

        if "message" in message.keys() and message["message"] == "SERVER_READY":
            self.recording = True
+            self.server_backend = message["backend"]
+            print(f"[INFO]: Server Running with backend {self.server_backend}")
            return

        if "language" in message.keys():
@@ -167,12 +204,21 @@ class Client:

        message = message["segments"]
        text = []
-        if len(message):
-            for seg in message:
+        n_segments = len(message)
+
+        if n_segments:
+            for i, seg in enumerate(message):
                if text and text[-1] == seg["text"]:
                    # already got it
                    continue
                text.append(seg["text"])
+
+                if i == n_segments-1: 
+                    self.last_segment = seg
+                elif self.server_backend == "faster_whisper":
+                    if not len(self.transcript) or float(seg['start']) >= float(self.transcript[-1]['end']):
+                        self.transcript.append(seg)
+
        # keep only last 3
        if len(text) > 3:
            text = text[-3:]
@@ -196,23 +242,21 @@ class Client:
        """
        Callback function called when the WebSocket connection is successfully opened.
        
-        Sends an initial configuration message to the server, including client UID, multilingual mode,
+        Sends an initial configuration message to the server, including client UID,
        language selection, and task type.

        Args:
            ws (websocket.WebSocketApp): The WebSocket client instance.

        """
-        print(self.multilingual, self.language, self.task)
-
        print("[INFO]: Opened connection")
        ws.send(
            json.dumps(
                {
                    "uid": self.uid,
-                    "multilingual": self.multilingual,
                    "language": self.language,
                    "task": self.task,
+                    "model": self.model,
                }
            )
        )
@@ -287,6 +331,9 @@ class Client:
                assert self.last_response_recieved
                while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for:
                    continue
+
+                if self.server_backend == "faster_whisper":
+                    self.write_srt_file(self.srt_file_path)
                self.stream.close()
                self.close_websocket()

@@ -296,6 +343,8 @@ class Client:
                self.stream.close()
                self.p.terminate()
                self.close_websocket()
+                if self.server_backend == "faster_whisper":
+                    self.write_srt_file(self.srt_file_path)
                print("[INFO]: Keyboard interrupt.")

    def close_websocket(self):
@@ -344,6 +393,42 @@ class Client:
            wavfile.setframerate(self.rate)
            wavfile.writeframes(frames)

+    def process_hls_stream(self, hls_url):
+        """
+        Connect to an HLS source, process the audio stream, and send it for transcription.
+
+        Args:
+            hls_url (str): The URL of the HLS stream source.
+        """
+        print("[INFO]: Connecting to HLS stream...")
+        process = None  # Initialize process to None
+
+        try:
+            # Connecting to the HLS stream using ffmpeg-python
+            process = (
+                ffmpeg
+                .input(hls_url, threads=0)
+                .output('-', format='s16le', acodec='pcm_s16le', ac=1, ar=self.rate)
+                .run_async(pipe_stdout=True, pipe_stderr=True)
+            )
+
+            # Process the stream
+            while True:
+                in_bytes = process.stdout.read(self.chunk * 2)  # 2 bytes per sample
+                if not in_bytes:
+                    break
+                audio_array = self.bytes_to_float_array(in_bytes)
+                self.send_packet_to_server(audio_array.tobytes())
+
+        except Exception as e:
+            print(f"[ERROR]: Failed to connect to HLS stream: {e}")
+        finally:
+            if process:
+                process.kill()
+
+        print("[INFO]: HLS stream processing finished.")
+
+
    def record(self, out_file="output_recording.wav"):
        """
        Record audio data from the input stream and save it to a WAV file.
@@ -368,7 +453,7 @@ class Client:
            for _ in range(0, int(self.rate / self.chunk * self.record_seconds)):
                if not self.recording:
                    break
-                data = self.stream.read(self.chunk)
+                data = self.stream.read(self.chunk, exception_on_overflow = False)
                self.frames += data

                audio_array = Client.bytes_to_float_array(data)
@@ -387,6 +472,8 @@ class Client:
                    t.start()
                    n_audio_file += 1
                    self.frames = b""
+            if self.server_backend == "faster_whisper":
+                self.write_srt_file(self.srt_file_path)

        except KeyboardInterrupt:
            if len(self.frames):
@@ -400,6 +487,8 @@ class Client:
            self.close_websocket()

            self.write_output_recording(n_audio_file, out_file)
+            if self.server_backend == "faster_whisper":
+                self.write_srt_file(self.srt_file_path)

    def write_output_recording(self, n_audio_file, out_file):
        """
@@ -436,6 +525,10 @@ class Client:
                os.remove(in_file)
        wavfile.close()

+    def write_srt_file(self, output_path="output.srt"):
+        self.transcript.append(self.last_segment)
+        create_srt_file(self.transcript, output_path)
+

 class TranscriptionClient:
    """
@@ -447,8 +540,7 @@ class TranscriptionClient:
    Args:
        host (str): The hostname or IP address of the server.
        port (int): The port number to connect to on the server.
-        is_multilingual (bool, optional): Indicates whether the transcription should support multiple languages (default is False).
-        lang (str, optional): The primary language for transcription (used if `is_multilingual` is False). Default is None, which defaults to English ('en').
+        lang (str, optional): The primary language for transcription. Default is None, which defaults to English ('en').
        translate (bool, optional): Indicates whether translation tasks are required (default is False).

    Attributes:
@@ -457,14 +549,20 @@ class TranscriptionClient:
    Example:
        To create a TranscriptionClient and start transcription on microphone audio:
        ```python
-        transcription_client = TranscriptionClient(host="localhost", port=9090, is_multilingual=True)
+        transcription_client = TranscriptionClient(host="localhost", port=9090)
        transcription_client()
        ```
    """
-    def __init__(self, host, port, is_multilingual=False, lang=None, translate=False):
-        self.client = Client(host, port, is_multilingual, lang, translate)
+    def __init__(self,
+        host,
+        port,
+        lang=None,
+        translate=False,
+        model="small",
+    ):
+        self.client = Client(host, port, lang, translate, model)

-    def __call__(self, audio=None):
+    def __call__(self, audio=None, hls_url=None):
        """
        Start the transcription process.

@@ -478,13 +576,15 @@ class TranscriptionClient:
        """
        print("[INFO]: Waiting for server ready ...")
        while not self.client.recording:
-            if self.client.waiting:
+            if self.client.waiting or self.client.server_error:
                self.client.close_websocket()
                return
-            pass
+
        print("[INFO]: Server Ready!")
-        if audio is not None:
+        if hls_url is not None:
+            self.client.process_hls_stream(hls_url)
+        elif audio is not None:
            resampled_file = resample(audio)
            self.client.play_file(resampled_file)
        else:
-            self.client.record()
+            self.client.record()
@@ -1,3 +1,4 @@
+import os
 import websockets
 import time
 import threading
@@ -5,15 +6,24 @@ import json
 import textwrap

 import logging
-# logging.basicConfig(level = logging.INFO)
+logging.basicConfig(level = logging.INFO)

 from websockets.sync.server import serve

 import torch
 import numpy as np
-import time
-from whisper_live.transcriber import WhisperModel
+import queue
+
 from whisper_live.vad import VoiceActivityDetection
+from scipy.io.wavfile import write
+import functools
+
+from whisper_live.vad import VoiceActivityDetection
+from whisper_live.transcriber import WhisperModel
+try:
+    from whisper_live.transcriber_tensorrt import WhisperTRTLLM
+except Exception as e:
+    logging.warn("cannot import WhisperTRTLLM")


 class TranscriptionServer:
@@ -35,9 +45,7 @@ class TranscriptionServer:

    def __init__(self):
        # voice activity detection model
-        self.vad_model = VoiceActivityDetection()
-        self.vad_threshold = 0.4
-
+        
        self.clients = {}
        self.websockets = {}
        self.clients_start_time = {}
@@ -61,7 +69,12 @@ class TranscriptionServer:

        return wait_time / 60

-    def recv_audio(self, websocket):
+    def recv_audio(self,
+                   websocket,
+                   backend="faster_whisper",
+                   faster_whisper_custom_model_path=None,
+                   whisper_tensorrt_path=None,
+                   trt_multilingual=False):
        """
        Receive audio chunks from a client in an infinite loop.
        
@@ -78,10 +91,19 @@ class TranscriptionServer:

        Args:
            websocket (WebSocket): The WebSocket connection for the client.
-        
+            backend (str): The backend to run the server with.
+            faster_whisper_custom_model_path (str): path to custom faster whisper model.
+            whisper_tensorrt_path (str): Required for tensorrt backend.
+            trt_multilingual(bool): Only used for tensorrt, True if multilingual model.
+
        Raises:
            Exception: If there is an error during the audio frame processing.
        """
+        self.backend = backend
+        if self.backend == "tensorrt":
+            self.vad_model = VoiceActivityDetection()
+            self.vad_threshold = 0.5
+
        logging.info("New client connected")
        options = websocket.recv()
        options = json.loads(options)
@@ -99,37 +121,82 @@ class TranscriptionServer:
            del websocket
            return

-        client = ServeClient(
-            websocket,
-            multilingual=options["multilingual"],
-            language=options["language"],
-            task=options["task"],
-            client_uid=options["uid"]
-        )
+        if self.backend == "tensorrt":
+            try:
+                import tensorrt as trt
+                import tensorrt_llm
+                self.backend = "tensorrt"
+                client = ServeClientTensorRT(
+                    websocket,
+                    multilingual=trt_multilingual,
+                    language=options["language"],
+                    task=options["task"],
+                    client_uid=options["uid"],
+                    model=whisper_tensorrt_path
+                )
+                logging.info(f"Running TensorRT backend.")
+            except Exception as e:
+                self.client_uid = options["uid"]
+                websocket.send(
+                    json.dumps(
+                        {
+                            "uid": self.client_uid,
+                            "status": "ERROR",
+                            "message": f"TensorRT-LLM not supported on Server yet. Reverting to available backend: 'faster_whisper'"
+                        }
+                    )
+                )
+                self.backend = "faster_whisper"

+        if self.backend == "faster_whisper":
+            # validate custom model
+            if faster_whisper_custom_model_path is not None and os.path.exists(faster_whisper_custom_model_path):
+                logging.info(f"Using custom model {faster_whisper_custom_model_path}")
+                options["model"] = faster_whisper_custom_model_path
+            client = ServeClientFasterWhisper(
+                websocket,
+                language=options["language"],
+                task=options["task"],
+                client_uid=options["uid"],
+                model=options["model"],
+                initial_prompt=options.get("initial_prompt"),
+                vad_parameters=options.get("vad_parameters")
+            )
+            logging.info(f"Running faster_whisper backend.")
+        
        self.clients[websocket] = client
        self.clients_start_time[websocket] = time.time()
+        no_voice_activity_chunks = 0

        while True:
            try:
                frame_data = websocket.recv()
                frame_np = np.frombuffer(frame_data, dtype=np.float32)

-                try:
-                    speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item()
-                    if speech_prob < self.vad_threshold:
-                        continue
-
-                except Exception as e:
-                    logging.error(e)
-                    return
+                # VAD, for faster_whisper VAD model is already integrated
+                if self.backend == "tensorrt":
+                    try:
+                        speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item()
+                        if speech_prob < self.vad_threshold:
+                            no_voice_activity_chunks += 1
+                            if no_voice_activity_chunks > 3:
+                                if not self.clients[websocket].eos:
+                                    self.clients[websocket].set_eos(True)
+                                time.sleep(0.1)    # Sleep 100m; wait some voice activity.
+                            continue
+                        no_voice_activity_chunks = 0
+                        self.clients[websocket].set_eos(False)

+                    except Exception as e:
+                        logging.error(e)
+                        return
+                
                self.clients[websocket].add_frames(frame_np)

                elapsed_time = time.time() - self.clients_start_time[websocket]
                if elapsed_time >= self.max_connection_time:
                    self.clients[websocket].disconnect()
-                    logging.warning(f"{self.clients[websocket]} Client disconnected due to overtime.")
+                    logging.warning(f"Client with uid '{self.clients[websocket].client_uid}' disconnected due to overtime.")
                    self.clients[websocket].cleanup()
                    self.clients.pop(websocket)
                    self.clients_start_time.pop(websocket)
@@ -142,12 +209,17 @@ class TranscriptionServer:
                self.clients[websocket].cleanup()
                self.clients.pop(websocket)
                self.clients_start_time.pop(websocket)
-                logging.info("Connection Closed.")
-                logging.info(self.clients)
                del websocket
                break

-    def run(self, host, port=9090):
+    def run(self, 
+            host, 
+            port=9090, 
+            backend="tensorrt", 
+            faster_whisper_custom_model_path=None,
+            whisper_tensorrt_path=None, 
+            trt_multilingual=False
+        ):
        """
        Run the transcription server.

@@ -155,11 +227,111 @@ class TranscriptionServer:
            host (str): The host address to bind the server.
            port (int): The port number to bind the server.
        """
-        with serve(self.recv_audio, host, port) as server:
+        with serve(
+            functools.partial(
+                self.recv_audio,
+                backend=backend,
+                faster_whisper_custom_model_path=faster_whisper_custom_model_path,
+                whisper_tensorrt_path=whisper_tensorrt_path,
+                trt_multilingual=trt_multilingual
+            ),
+            host,
+            port
+        ) as server:
            server.serve_forever()


-class ServeClient:
+class ServeClientBase(object):
+    RATE = 16000
+    SERVER_READY = "SERVER_READY"
+    DISCONNECT = "DISCONNECT"
+
+    def __init__(self, client_uid, websocket):
+        self.client_uid = client_uid
+        self.websocket = websocket
+        self.data = b""
+        self.frames = b""
+        self.timestamp_offset = 0.0
+        self.frames_np = None
+        self.frames_offset = 0.0
+        self.text = []
+        self.current_out = ''
+        self.prev_out = ''
+        self.t_start=None
+        self.exit = False
+        self.same_output_threshold = 0
+        self.show_prev_out_thresh = 5   # if pause(no output from whisper) show previous output for 5 seconds
+        self.add_pause_thresh = 3       # add a blank to segment list as a pause(no speech) for 3 seconds
+        self.transcript = []
+        self.send_last_n_segments = 10
+
+        # text formatting
+        self.wrapper = textwrap.TextWrapper(width=50)
+        self.pick_previous_segments = 2
+
+        # threading
+        self.lock = threading.Lock()
+    
+    def add_frames(self, frame_np):
+        """
+        Add audio frames to the ongoing audio stream buffer.
+
+        This method is responsible for maintaining the audio stream buffer, allowing the continuous addition
+        of audio frames as they are received. It also ensures that the buffer does not exceed a specified size
+        to prevent excessive memory usage.
+
+        If the buffer size exceeds a threshold (45 seconds of audio data), it discards the oldest 30 seconds
+        of audio data to maintain a reasonable buffer size. If the buffer is empty, it initializes it with the provided
+        audio frame. The audio stream buffer is used for real-time processing of audio data for transcription.
+
+        Args:
+            frame_np (numpy.ndarray): The audio frame data as a NumPy array.
+
+        """
+        self.lock.acquire()
+        if self.frames_np is not None and self.frames_np.shape[0] > 45*self.RATE:
+            self.frames_offset += 30.0
+            self.frames_np = self.frames_np[int(30*self.RATE):]
+        if self.frames_np is None:
+            self.frames_np = frame_np.copy()
+        else:
+            self.frames_np = np.concatenate((self.frames_np, frame_np), axis=0)
+        self.lock.release()
+
+    def speech_to_text(self):
+        raise NotImplementedError("Please implement in child Class.")
+    
+    def disconnect(self):
+        """
+        Notify the client of disconnection and send a disconnect message.
+
+        This method sends a disconnect message to the client via the WebSocket connection to notify them
+        that the transcription service is disconnecting gracefully.
+
+        """
+        self.websocket.send(
+            json.dumps(
+                {
+                    "uid": self.client_uid,
+                    "message": self.DISCONNECT
+                }
+            )
+        )
+    
+    def cleanup(self):
+        """
+        Perform cleanup tasks before exiting the transcription service.
+
+        This method performs necessary cleanup tasks, including stopping the transcription thread, marking
+        the exit flag to indicate the transcription thread should exit gracefully, and destroying resources
+        associated with the transcription process.
+
+        """
+        logging.info("Cleaning up.")
+        self.exit = True
+
+
+class ServeClientTensorRT(ServeClientBase):
    """
    Attributes:
        RATE (int): The audio sampling rate (constant) set to 16000.
@@ -188,11 +360,16 @@ class ServeClient:
        pick_previous_segments (int): Number of previous segments to include in the output.
        websocket: The WebSocket connection for the client.
    """
-    RATE = 16000
-    SERVER_READY = "SERVER_READY"
-    DISCONNECT = "DISCONNECT"
-
-    def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None, client_uid=None):
+    def __init__(
+        self,
+        websocket,
+        task="transcribe",
+        device=None,
+        multilingual=False,
+        language=None, 
+        client_uid=None,
+        model=None
+        ):
        """
        Initialize a ServeClient instance.
        The Whisper model is initialized based on the client's language and device availability.
@@ -208,78 +385,44 @@ class ServeClient:
            client_uid (str, optional): A unique identifier for the client. Defaults to None.

        """
-        self.client_uid = client_uid
-        self.data = b""
-        self.frames = b""
+        super().__init__(client_uid, websocket)
        self.language = language if multilingual else "en"
        self.task = task
-        device = "cuda" if torch.cuda.is_available() else "cpu"
-        self.transcriber = WhisperModel(
-            "small" if multilingual else "small.en", 
-            device=device,
-            compute_type="int8" if device=="cpu" else "float16", 
-            local_files_only=False,
+        self.eos = False
+        self.transcriber = WhisperTRTLLM(
+            model, 
+            assets_dir="assets", 
+            device="cuda",
+            is_multilingual=multilingual,
+            language=self.language,
+            task=self.task
        )
-        
-        self.timestamp_offset = 0.0
-        self.frames_np = None
-        self.frames_offset = 0.0
-        self.text = []
-        self.current_out = ''
-        self.prev_out = ''
-        self.t_start=None
-        self.exit = False
-        self.same_output_threshold = 0
-        self.show_prev_out_thresh = 5   # if pause(no output from whisper) show previous output for 5 seconds
-        self.add_pause_thresh = 3       # add a blank to segment list as a pause(no speech) for 3 seconds
-        self.transcript = []
-        self.send_last_n_segments = 10
-
-        # text formatting
-        self.wrapper = textwrap.TextWrapper(width=50)
-        self.pick_previous_segments = 2
+        self.warmup()

        # threading
-        self.websocket = websocket
        self.trans_thread = threading.Thread(target=self.speech_to_text)
        self.trans_thread.start()
+
        self.websocket.send(
            json.dumps(
                {
                    "uid": self.client_uid,
-                    "message": self.SERVER_READY
+                    "message": self.SERVER_READY,
+                    "backend": "tensorrt"
                }
            )
        )
+
+    def warmup(self, warmup_steps=10):
+        logging.info("[INFO:] Warming up TensorRT engine..")
+        mel, _ = self.transcriber.log_mel_spectrogram("tests/jfk.flac")
+        for i in range(warmup_steps):
+            self.transcriber.transcribe(mel)
    
-    def fill_output(self, output):
-        """
-        Format the current incomplete transcription output by combining it with previous complete segments.
-        The resulting transcription is wrapped into two lines, each containing a maximum of 50 characters.
-
-        It ensures that the combined transcription fits within two lines, with a maximum of 50 characters per line.
-        Segments are concatenated in the order they exist in the list of previous segments, with the most
-        recent complete segment first and older segments prepended as needed to maintain the character limit.
-        If a 3-second pause is detected in the previous segments, any text preceding it is discarded to ensure
-        the transcription starts with the most recent complete content. The resulting transcription is returned
-        as a single string.
-
-        Args:
-            output(str): The current incomplete transcription segment.
-        
-        Returns:
-            str: A formatted transcription wrapped in two lines.
-        """
-        text = ''
-        pick_prev = min(len(self.text), self.pick_previous_segments)
-        for seg in self.text[-pick_prev:]:
-            # discard everything before a 3 second pause
-            if seg == '':
-                text = ''
-            else:
-                text += seg
-        wrapped = "".join(text + output)
-        return wrapped
+    def set_eos(self, eos):
+        self.lock.acquire()
+        self.eos = eos
+        self.lock.release()
    
    def add_frames(self, frame_np):
        """
@@ -297,6 +440,7 @@ class ServeClient:
            frame_np (numpy.ndarray): The audio frame data as a NumPy array.

        """
+        self.lock.acquire()
        if self.frames_np is not None and self.frames_np.shape[0] > 45*self.RATE:
            self.frames_offset += 30.0
            self.frames_np = self.frames_np[int(30*self.RATE):]
@@ -304,6 +448,7 @@ class ServeClient:
            self.frames_np = frame_np.copy()
        else:
            self.frames_np = np.concatenate((self.frames_np, frame_np), axis=0)
+        self.lock.release()

    def speech_to_text(self):
        """
@@ -322,25 +467,197 @@ class ServeClient:
            Exception: If there is an issue with audio processing or WebSocket communication.

        """
-        # detect language
-        if self.language is None:
-            # wait for 30s of audio
-            while self.frames_np is None or self.frames_np.shape[0] < 30*self.RATE:
-                time.sleep(1)
-            input_bytes = self.frames_np[-30*self.RATE:].copy()
-            self.frames_np = None
+        while True:
+            if self.exit:
+                logging.info("Exiting speech to text thread")
+                break
+            
+            if self.frames_np is None:
+                time.sleep(0.02)    # wait for any audio to arrive
+                continue
+
+            # clip audio if the current chunk exceeds 30 seconds, this basically implies that
+            # no valid segment for the last 30 seconds from whisper
+            if self.frames_np[int((self.timestamp_offset - self.frames_offset)*self.RATE):].shape[0] > 25 * self.RATE:
+                duration = self.frames_np.shape[0] / self.RATE
+                self.timestamp_offset = self.frames_offset + duration - 5
+    
+            samples_take = max(0, (self.timestamp_offset - self.frames_offset)*self.RATE)
+            input_bytes = self.frames_np[int(samples_take):].copy()
            duration = input_bytes.shape[0] / self.RATE
+            if duration<0.4:
+                continue

-            self.language, lang_prob = self.transcriber.transcribe(
-                    input_bytes, 
-                    initial_prompt=None,
-                    language=self.language,
-                    task=self.task
+            try:
+                input_sample = input_bytes.copy()
+                logging.info(f"[WhisperTensorRT:] Processing audio with duration: {duration}")
+                mel, duration = self.transcriber.log_mel_spectrogram(input_sample)
+                last_segment = self.transcriber.transcribe(mel)
+                segments = []
+                if len(last_segment):
+                    if len(self.transcript) < self.send_last_n_segments:
+                        segments = self.transcript[:].copy()
+                    else:
+                        segments = self.transcript[-self.send_last_n_segments:].copy()
+                    if last_segment is not None:
+                        segments.append({"text": last_segment})
+                    try:
+                        self.websocket.send(
+                            json.dumps({
+                                "uid": self.client_uid,
+                                "segments": segments,
+                            })
+                        )
+
+                        if self.eos:
+                            if not len(self.transcript):
+                                self.transcript.append({"text": last_segment + " "})
+                            elif self.transcript[-1]["text"].strip() != last_segment:
+                                self.transcript.append({"text": last_segment + " "})
+                            self.timestamp_offset += duration
+
+                            
+                    except Exception as e:
+                        logging.error(f"[ERROR]: {e}")
+
+            except Exception as e:
+                logging.error(f"[ERROR]: {e}")
+
+
+class ServeClientFasterWhisper(ServeClientBase):
+    """
+    Attributes:
+        RATE (int): The audio sampling rate (constant) set to 16000.
+        SERVER_READY (str): A constant message indicating that the server is ready.
+        DISCONNECT (str): A constant message indicating that the client should disconnect.
+        client_uid (str): A unique identifier for the client.
+        data (bytes): Accumulated audio data.
+        frames (bytes): Accumulated audio frames.
+        language (str): The language for transcription.
+        task (str): The task type, e.g., "transcribe."
+        transcriber (WhisperModel): The Whisper model for speech-to-text.
+        timestamp_offset (float): The offset in audio timestamps.
+        frames_np (numpy.ndarray): NumPy array to store audio frames.
+        frames_offset (float): The offset in audio frames.
+        text (list): List of transcribed text segments.
+        current_out (str): The current incomplete transcription.
+        prev_out (str): The previous incomplete transcription.
+        t_start (float): Timestamp for the start of transcription.
+        exit (bool): A flag to exit the transcription thread.
+        same_output_threshold (int): Threshold for consecutive same output segments.
+        show_prev_out_thresh (int): Threshold for showing previous output segments.
+        add_pause_thresh (int): Threshold for adding a pause (blank) segment.
+        transcript (list): List of transcribed segments.
+        send_last_n_segments (int): Number of last segments to send to the client.
+        wrapper (textwrap.TextWrapper): Text wrapper for formatting text.
+        pick_previous_segments (int): Number of previous segments to include in the output.
+        websocket: The WebSocket connection for the client.
+    """
+    def __init__(
+        self,
+        websocket,
+        task="transcribe",
+        device=None,
+        language=None,
+        client_uid=None,
+        model="small.en",
+        initial_prompt=None,
+        vad_parameters=None,
+        ):
+        """
+        Initialize a ServeClient instance.
+        The Whisper model is initialized based on the client's language and device availability.
+        The transcription thread is started upon initialization. A "SERVER_READY" message is sent
+        to the client to indicate that the server is ready.
+
+        Args:
+            websocket (WebSocket): The WebSocket connection for the client.
+            task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe".
+            device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None.
+            language (str, optional): The language for transcription. Defaults to None.
+            client_uid (str, optional): A unique identifier for the client. Defaults to None.
+
+        """
+        super().__init__(client_uid, websocket)
+        self.model_sizes = [
+            "tiny", "tiny.en", "base", "base.en", "small", "small.en",
+            "medium", "medium.en", "large-v2", "large-v3",
+        ]
+        if not os.path.exists(model):
+            self.model_size_or_path = self.check_valid_model(model)
+        else:
+            self.model_size_or_path = model
+        self.language = "en" if self.model_size_or_path.endswith("en") else language
+        self.task = task
+        self.initial_prompt = initial_prompt
+        self.vad_parameters = vad_parameters or {"threshold": 0.5}
+        self.no_speech_thresh = 0.45
+        
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        
+        if self.model_size_or_path == None:
+            return
+
+        self.transcriber = WhisperModel(
+            self.model_size_or_path, 
+            device=device,
+            compute_type="int8" if device=="cpu" else "float16", 
+            local_files_only=False,
+        )
+
+        # threading
+        self.trans_thread = threading.Thread(target=self.speech_to_text)
+        self.trans_thread.start()
+        self.websocket.send(
+            json.dumps(
+                {
+                    "uid": self.client_uid,
+                    "message": self.SERVER_READY,
+                    "backend": "faster_whisper"
+                }
+            )
+        )
+    
+    def check_valid_model(self, model_size):
+        """
+        Check if it's a valid whisper model size.
+
+        Args:
+            model_size (str): The name of the model size to check.
+
+        Returns:
+            str: The model size if valid, None otherwise.
+        """
+        if model_size not in self.model_sizes:
+            self.websocket.send(
+                json.dumps(
+                    {
+                        "uid": self.client_uid,
+                        "status": "ERROR",
+                        "message": f"Invalid model size {model_size}. Available choices: {self.model_sizes}"
+                    }
                )
-            logging.info(f"Detected language {self.language} with probability {lang_prob}")
-            self.websocket.send(json.dumps(
-                {"uid": self.client_uid, "language": self.language, "language_prob": lang_prob}))
+            )
+            return None
+        return model_size
+    
+    def speech_to_text(self):
+        """
+        Process an audio stream in an infinite loop, continuously transcribing the speech.

+        This method continuously receives audio frames, performs real-time transcription, and sends
+        transcribed segments to the client via a WebSocket connection.
+
+        If the client's language is not detected, it waits for 30 seconds of audio input to make a language prediction.
+        It utilizes the Whisper ASR model to transcribe the audio, continuously processing and streaming results. Segments
+        are sent to the client in real-time, and a history of segments is maintained to provide context.Pauses in speech 
+        (no output from Whisper) are handled by showing the previous output for a set duration. A blank segment is added if 
+        there is no speech for a specified duration to indicate a pause.
+
+        Raises:
+            Exception: If there is an issue with audio processing or WebSocket communication.
+
+        """
        while True:
            if self.exit:
                logging.info("Exiting speech to text thread")
@@ -358,24 +675,31 @@ class ServeClient:
            samples_take = max(0, (self.timestamp_offset - self.frames_offset)*self.RATE)
            input_bytes = self.frames_np[int(samples_take):].copy()
            duration = input_bytes.shape[0] / self.RATE
-            if duration<1.0: 
+            if duration<1.0:
                continue
            try:
                input_sample = input_bytes.copy()
-                # set previous complete segment as initial prompt
-                if len(self.text) and self.text[-1] != '': 
-                    initial_prompt = self.text[-1]
-                else: 
-                    initial_prompt = None
                
                # whisper transcribe with prompt
-                result = self.transcriber.transcribe(
+                result, info = self.transcriber.transcribe(
                    input_sample, 
-                    initial_prompt=initial_prompt,
+                    initial_prompt=self.initial_prompt,
                    language=self.language,
-                    task=self.task
+                    task=self.task,
+                    vad_filter=True,
+                    vad_parameters=self.vad_parameters
                )

+                if self.language is None:
+                    if info.language_probability > 0.5:
+                        self.language = info.language
+                        logging.info(f"Detected language {self.language} with probability {info.language_probability}")
+                        self.websocket.send(json.dumps(
+                            {"uid": self.client_uid, "language": self.language, "language_prob": info.language_probability}))
+                    else:
+                        # detect language again
+                        continue
+
                if len(result):
                    self.t_start = None
                    last_segment = self.update_segments(result, duration)
@@ -384,17 +708,7 @@ class ServeClient:
                    else:
                        segments = self.transcript[-self.send_last_n_segments:]
                    if last_segment is not None:
-                        segments = segments + [last_segment]
-                    
-                    try:
-                        self.websocket.send(
-                            json.dumps({
-                                "uid": self.client_uid,
-                                "segments": segments
-                            })
-                        )
-                    except Exception as e:
-                        logging.error(f"[ERROR]: {e}")
+                        segments = segments + [last_segment]                    
                else:
                    # show previous output if there is pause i.e. no output from whisper
                    segments = []
@@ -410,19 +724,29 @@ class ServeClient:
                        if time.time() - self.t_start > self.add_pause_thresh:
                            self.text.append('')

-                    try:
-                        self.websocket.send(
-                            json.dumps({
-                                "uid": self.client_uid,
-                                "segments": segments
-                            })
-                        )
-                    except Exception as e:
-                        logging.error(f"[ERROR]: {e}")
+                if not len(segments): continue
+                try:
+                    self.websocket.send(
+                        json.dumps({
+                            "uid": self.client_uid,
+                            "segments": segments
+                        })
+                    )
+                except Exception as e:
+                    logging.error(f"[ERROR]: Failed to send message to client: {e}")
+
            except Exception as e:
-                logging.error(f"[ERROR]: {e}")
+                logging.error(f"[ERROR]: Failed to transcribe audio chunk: {e}")
                time.sleep(0.01)
    
+    def format_segment(self, start, end, text):
+        """Helper function to format a segment with string timestamps."""
+        return {
+            'start': "{:.3f}".format(start),
+            'end': "{:.3f}".format(end),
+            'text': text
+        }
+
    def update_segments(self, segments, duration):
        """
        Processes the segments from whisper. Appends all the segments to the list
@@ -453,22 +777,20 @@ class ServeClient:
                text_ = s.text
                self.text.append(text_)
                start, end = self.timestamp_offset + s.start, self.timestamp_offset + min(duration, s.end)
-                self.transcript.append(
-                    {
-                        'start': start,
-                        'end': end,
-                        'text': text_
-                    }
-                )
+
+                if start >= end: continue
+                if s.no_speech_prob > self.no_speech_thresh: continue
+
+                self.transcript.append(self.format_segment(start, end, text_))
                
                offset = min(duration, s.end)

        self.current_out += segments[-1].text
-        last_segment = {
-            'start': self.timestamp_offset + segments[-1].start,
-            'end': self.timestamp_offset + min(duration, segments[-1].end),
-            'text': self.current_out
-        }
+        last_segment = self.format_segment(
+            self.timestamp_offset + segments[-1].start,
+            self.timestamp_offset + min(duration, segments[-1].end),
+            self.current_out
+        )
        
        # if same incomplete segment is seen multiple times then update the offset
        # and append the segment to the list
@@ -480,13 +802,11 @@ class ServeClient:
        if self.same_output_threshold > 5:
            if not len(self.text) or self.text[-1].strip().lower()!=self.current_out.strip().lower():          
                self.text.append(self.current_out)
-                self.transcript.append(
-                    {
-                        'start': self.timestamp_offset,
-                        'end': self.timestamp_offset + duration,
-                        'text': self.current_out
-                    }
-                )
+                self.transcript.append(self.format_segment(
+                    self.timestamp_offset,
+                    self.timestamp_offset + duration,
+                    self.current_out
+                ))
            self.current_out = ''
            offset = duration
            self.same_output_threshold = 0
@@ -499,33 +819,3 @@ class ServeClient:
            self.timestamp_offset += offset

        return last_segment
-    
-    def disconnect(self):
-        """
-        Notify the client of disconnection and send a disconnect message.
-
-        This method sends a disconnect message to the client via the WebSocket connection to notify them
-        that the transcription service is disconnecting gracefully.
-
-        """
-        self.websocket.send(
-            json.dumps(
-                {
-                    "uid": self.client_uid,
-                    "message": self.DISCONNECT
-                }
-            )
-        )
-    
-    def cleanup(self):
-        """
-        Perform cleanup tasks before exiting the transcription service.
-
-        This method performs necessary cleanup tasks, including stopping the transcription thread, marking
-        the exit flag to indicate the transcription thread should exit gracefully, and destroying resources
-        associated with the transcription process.
-
-        """
-        logging.info("Cleaning up.")
-        self.exit = True
-        self.transcriber.destroy()
@@ -0,0 +1,365 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+from collections import defaultdict
+from functools import lru_cache
+from pathlib import Path
+from subprocess import CalledProcessError, run
+from typing import Dict, Iterable, List, Optional, TextIO, Tuple, Union
+
+import kaldialign
+import numpy as np
+import soundfile
+import torch
+import torch.nn.functional as F
+
+Pathlike = Union[str, Path]
+
+SAMPLE_RATE = 16000
+N_FFT = 400
+HOP_LENGTH = 160
+CHUNK_LENGTH = 30
+N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE  # 480000 samples in a 30-second chunk
+
+
+def load_audio(file: str, sr: int = SAMPLE_RATE):
+    """
+    Open an audio file and read as mono waveform, resampling as necessary
+
+    Parameters
+    ----------
+    file: str
+        The audio file to open
+
+    sr: int
+        The sample rate to resample the audio if necessary
+
+    Returns
+    -------
+    A NumPy array containing the audio waveform, in float32 dtype.
+    """
+
+    # This launches a subprocess to decode audio while down-mixing
+    # and resampling as necessary.  Requires the ffmpeg CLI in PATH.
+    # fmt: off
+    cmd = [
+        "ffmpeg", "-nostdin", "-threads", "0", "-i", file, "-f", "s16le", "-ac",
+        "1", "-acodec", "pcm_s16le", "-ar",
+        str(sr), "-"
+    ]
+    # fmt: on
+    try:
+        out = run(cmd, capture_output=True, check=True).stdout
+    except CalledProcessError as e:
+        raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e
+
+    return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0
+
+
+def load_audio_wav_format(wav_path):
+    # make sure audio in .wav format
+    assert wav_path.endswith(
+        '.wav'), f"Only support .wav format, but got {wav_path}"
+    waveform, sample_rate = soundfile.read(wav_path)
+    assert sample_rate == 16000, f"Only support 16k sample rate, but got {sample_rate}"
+    return waveform, sample_rate
+
+
+def pad_or_trim(array, length: int = N_SAMPLES, *, axis: int = -1):
+    """
+    Pad or trim the audio array to N_SAMPLES, as expected by the encoder.
+    """
+    if torch.is_tensor(array):
+        if array.shape[axis] > length:
+            array = array.index_select(dim=axis,
+                                       index=torch.arange(length,
+                                                          device=array.device))
+
+        if array.shape[axis] < length:
+            pad_widths = [(0, 0)] * array.ndim
+            pad_widths[axis] = (0, length - array.shape[axis])
+            array = F.pad(array,
+                          [pad for sizes in pad_widths[::-1] for pad in sizes])
+    else:
+        if array.shape[axis] > length:
+            array = array.take(indices=range(length), axis=axis)
+
+        if array.shape[axis] < length:
+            pad_widths = [(0, 0)] * array.ndim
+            pad_widths[axis] = (0, length - array.shape[axis])
+            array = np.pad(array, pad_widths)
+
+    return array
+
+
+@lru_cache(maxsize=None)
+def mel_filters(device,
+                n_mels: int,
+                mel_filters_dir: str = None) -> torch.Tensor:
+    """
+    load the mel filterbank matrix for projecting STFT into a Mel spectrogram.
+    Allows decoupling librosa dependency; saved using:
+
+        np.savez_compressed(
+            "mel_filters.npz",
+            mel_80=librosa.filters.mel(sr=16000, n_fft=400, n_mels=80),
+        )
+    """
+    assert n_mels in {80, 128}, f"Unsupported n_mels: {n_mels}"
+    if mel_filters_dir is None:
+        mel_filters_path = os.path.join(os.path.dirname(__file__), "assets",
+                                        "mel_filters.npz")
+    else:
+        mel_filters_path = os.path.join(mel_filters_dir, "mel_filters.npz")
+    with np.load(mel_filters_path) as f:
+        return torch.from_numpy(f[f"mel_{n_mels}"]).to(device)
+
+
+def log_mel_spectrogram(
+    audio: Union[str, np.ndarray, torch.Tensor],
+    n_mels: int,
+    padding: int = 0,
+    device: Optional[Union[str, torch.device]] = None,
+    return_duration: bool = False,
+    mel_filters_dir: str = None,
+):
+    """
+    Compute the log-Mel spectrogram of
+
+    Parameters
+    ----------
+    audio: Union[str, np.ndarray, torch.Tensor], shape = (*)
+        The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz
+
+    n_mels: int
+        The number of Mel-frequency filters, only 80 and 128 are supported
+
+    padding: int
+        Number of zero samples to pad to the right
+
+    device: Optional[Union[str, torch.device]]
+        If given, the audio tensor is moved to this device before STFT
+
+    Returns
+    -------
+    torch.Tensor, shape = (80 or 128, n_frames)
+        A Tensor that contains the Mel spectrogram
+    """
+    if not torch.is_tensor(audio):
+        if isinstance(audio, str):
+            if audio.endswith('.wav'):
+                audio, _ = load_audio_wav_format(audio)
+            else:
+                audio = load_audio(audio)
+        assert isinstance(audio,
+                          np.ndarray), f"Unsupported audio type: {type(audio)}"
+        duration = audio.shape[-1] / SAMPLE_RATE
+        audio = pad_or_trim(audio, N_SAMPLES)
+        audio = audio.astype(np.float32)
+        audio = torch.from_numpy(audio)
+
+    if device is not None:
+        audio = audio.to(device)
+    if padding > 0:
+        audio = F.pad(audio, (0, padding))
+    window = torch.hann_window(N_FFT).to(audio.device)
+    stft = torch.stft(audio,
+                      N_FFT,
+                      HOP_LENGTH,
+                      window=window,
+                      return_complex=True)
+    magnitudes = stft[..., :-1].abs()**2
+
+    filters = mel_filters(audio.device, n_mels, mel_filters_dir)
+    mel_spec = filters @ magnitudes
+
+    log_spec = torch.clamp(mel_spec, min=1e-10).log10()
+    log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
+    log_spec = (log_spec + 4.0) / 4.0
+    if return_duration:
+        return log_spec, duration
+    else:
+        return log_spec
+
+
+def store_transcripts(filename: Pathlike, texts: Iterable[Tuple[str, str,
+                                                                str]]) -> None:
+    """Save predicted results and reference transcripts to a file.
+    https://github.com/k2-fsa/icefall/blob/master/icefall/utils.py
+    Args:
+      filename:
+        File to save the results to.
+      texts:
+        An iterable of tuples. The first element is the cur_id, the second is
+        the reference transcript and the third element is the predicted result.
+    Returns:
+      Return None.
+    """
+    with open(filename, "w") as f:
+        for cut_id, ref, hyp in texts:
+            print(f"{cut_id}:\tref={ref}", file=f)
+            print(f"{cut_id}:\thyp={hyp}", file=f)
+
+
+def write_error_stats(
+    f: TextIO,
+    test_set_name: str,
+    results: List[Tuple[str, str]],
+    enable_log: bool = True,
+) -> float:
+    """Write statistics based on predicted results and reference transcripts.
+    https://github.com/k2-fsa/icefall/blob/master/icefall/utils.py
+    It will write the following to the given file:
+
+        - WER
+        - number of insertions, deletions, substitutions, corrects and total
+          reference words. For example::
+
+              Errors: 23 insertions, 57 deletions, 212 substitutions, over 2606
+              reference words (2337 correct)
+
+        - The difference between the reference transcript and predicted result.
+          An instance is given below::
+
+            THE ASSOCIATION OF (EDISON->ADDISON) ILLUMINATING COMPANIES
+
+          The above example shows that the reference word is `EDISON`,
+          but it is predicted to `ADDISON` (a substitution error).
+
+          Another example is::
+
+            FOR THE FIRST DAY (SIR->*) I THINK
+
+          The reference word `SIR` is missing in the predicted
+          results (a deletion error).
+      results:
+        An iterable of tuples. The first element is the cur_id, the second is
+        the reference transcript and the third element is the predicted result.
+      enable_log:
+        If True, also print detailed WER to the console.
+        Otherwise, it is written only to the given file.
+    Returns:
+      Return None.
+    """
+    subs: Dict[Tuple[str, str], int] = defaultdict(int)
+    ins: Dict[str, int] = defaultdict(int)
+    dels: Dict[str, int] = defaultdict(int)
+
+    # `words` stores counts per word, as follows:
+    #   corr, ref_sub, hyp_sub, ins, dels
+    words: Dict[str, List[int]] = defaultdict(lambda: [0, 0, 0, 0, 0])
+    num_corr = 0
+    ERR = "*"
+    for cut_id, ref, hyp in results:
+        ali = kaldialign.align(ref, hyp, ERR)
+        for ref_word, hyp_word in ali:
+            if ref_word == ERR:
+                ins[hyp_word] += 1
+                words[hyp_word][3] += 1
+            elif hyp_word == ERR:
+                dels[ref_word] += 1
+                words[ref_word][4] += 1
+            elif hyp_word != ref_word:
+                subs[(ref_word, hyp_word)] += 1
+                words[ref_word][1] += 1
+                words[hyp_word][2] += 1
+            else:
+                words[ref_word][0] += 1
+                num_corr += 1
+    ref_len = sum([len(r) for _, r, _ in results])
+    sub_errs = sum(subs.values())
+    ins_errs = sum(ins.values())
+    del_errs = sum(dels.values())
+    tot_errs = sub_errs + ins_errs + del_errs
+    tot_err_rate = "%.2f" % (100.0 * tot_errs / ref_len)
+
+    if enable_log:
+        logging.info(f"[{test_set_name}] %WER {tot_errs / ref_len:.2%} "
+                     f"[{tot_errs} / {ref_len}, {ins_errs} ins, "
+                     f"{del_errs} del, {sub_errs} sub ]")
+
+    print(f"%WER = {tot_err_rate}", file=f)
+    print(
+        f"Errors: {ins_errs} insertions, {del_errs} deletions, "
+        f"{sub_errs} substitutions, over {ref_len} reference "
+        f"words ({num_corr} correct)",
+        file=f,
+    )
+    print(
+        "Search below for sections starting with PER-UTT DETAILS:, "
+        "SUBSTITUTIONS:, DELETIONS:, INSERTIONS:, PER-WORD STATS:",
+        file=f,
+    )
+
+    print("", file=f)
+    print("PER-UTT DETAILS: corr or (ref->hyp)  ", file=f)
+    for cut_id, ref, hyp in results:
+        ali = kaldialign.align(ref, hyp, ERR)
+        combine_successive_errors = True
+        if combine_successive_errors:
+            ali = [[[x], [y]] for x, y in ali]
+            for i in range(len(ali) - 1):
+                if ali[i][0] != ali[i][1] and ali[i + 1][0] != ali[i + 1][1]:
+                    ali[i + 1][0] = ali[i][0] + ali[i + 1][0]
+                    ali[i + 1][1] = ali[i][1] + ali[i + 1][1]
+                    ali[i] = [[], []]
+            ali = [[
+                list(filter(lambda a: a != ERR, x)),
+                list(filter(lambda a: a != ERR, y)),
+            ] for x, y in ali]
+            ali = list(filter(lambda x: x != [[], []], ali))
+            ali = [[
+                ERR if x == [] else " ".join(x),
+                ERR if y == [] else " ".join(y),
+            ] for x, y in ali]
+
+        print(
+            f"{cut_id}:\t" + " ".join((ref_word if ref_word == hyp_word else
+                                       f"({ref_word}->{hyp_word})"
+                                       for ref_word, hyp_word in ali)),
+            file=f,
+        )
+
+    print("", file=f)
+    print("SUBSTITUTIONS: count ref -> hyp", file=f)
+
+    for count, (ref, hyp) in sorted([(v, k) for k, v in subs.items()],
+                                    reverse=True):
+        print(f"{count}   {ref} -> {hyp}", file=f)
+
+    print("", file=f)
+    print("DELETIONS: count ref", file=f)
+    for count, ref in sorted([(v, k) for k, v in dels.items()], reverse=True):
+        print(f"{count}   {ref}", file=f)
+
+    print("", file=f)
+    print("INSERTIONS: count hyp", file=f)
+    for count, hyp in sorted([(v, k) for k, v in ins.items()], reverse=True):
+        print(f"{count}   {hyp}", file=f)
+
+    print("", file=f)
+    print("PER-WORD STATS: word  corr tot_errs count_in_ref count_in_hyp",
+          file=f)
+    for _, word, counts in sorted([(sum(v[1:]), k, v)
+                                   for k, v in words.items()],
+                                  reverse=True):
+        (corr, ref_sub, hyp_sub, ins, dels) = counts
+        tot_errs = ref_sub + hyp_sub + ins + dels
+        ref_count = corr + ref_sub + dels
+        hyp_count = corr + hyp_sub + ins
+
+        print(f"{word}   {corr} {tot_errs} {ref_count} {hyp_count}", file=f)
+    return float(tot_err_rate)
@@ -4,7 +4,8 @@ import itertools
 import logging
 import os
 import zlib
-import logging
+import json
+from inspect import signature

 from typing import BinaryIO, Iterable, List, NamedTuple, Optional, Tuple, Union

@@ -14,21 +15,16 @@ import tokenizers

 from faster_whisper.audio import decode_audio
 from faster_whisper.feature_extractor import FeatureExtractor
-from faster_whisper.tokenizer import Tokenizer
-from faster_whisper.utils import download_model, format_timestamp
+from faster_whisper.tokenizer import _LANGUAGE_CODES, Tokenizer
+from faster_whisper.utils import download_model, format_timestamp, get_logger
 from faster_whisper.vad import (
    SpeechTimestampsMap,
+    VadOptions,
    collect_chunks,
    get_speech_timestamps,
 )


-# implement logger not available in faster_whisper==0.4.1
-def get_logger():
-    """Returns the module logger."""
-    return logging.getLogger("faster_whisper")
-
-
 class Word(NamedTuple):
    start: float
    end: float
@@ -37,18 +33,17 @@ class Word(NamedTuple):


 class Segment(NamedTuple):
+    id: int
+    seek: int
    start: float
    end: float
    text: str
-    words: Optional[List[Word]]
-    avg_log_prob: float
+    tokens: List[int]
+    temperature: float
+    avg_logprob: float
+    compression_ratio: float
    no_speech_prob: float
-
-
-class AudioInfo(NamedTuple):
-    language: str
-    language_probability: float
-    duration: float
+    words: Optional[List[Word]]


 class TranscriptionOptions(NamedTuple):
@@ -56,12 +51,15 @@ class TranscriptionOptions(NamedTuple):
    best_of: int
    patience: float
    length_penalty: float
+    repetition_penalty: float
+    no_repeat_ngram_size: int
    log_prob_threshold: Optional[float]
    no_speech_threshold: Optional[float]
    compression_ratio_threshold: Optional[float]
    condition_on_previous_text: bool
+    prompt_reset_on_temperature: float
    temperatures: List[float]
-    initial_prompt: Optional[str]
+    initial_prompt: Optional[Union[str, Iterable[int]]]
    prefix: Optional[str]
    suppress_blank: bool
    suppress_tokens: Optional[List[int]]
@@ -72,6 +70,16 @@ class TranscriptionOptions(NamedTuple):
    append_punctuations: str


+class TranscriptionInfo(NamedTuple):
+    language: str
+    language_probability: float
+    duration: float
+    duration_after_vad: float
+    all_language_probs: Optional[List[Tuple[str, float]]]
+    transcription_options: TranscriptionOptions
+    vad_options: VadOptions
+
+
 class WhisperModel:
    def __init__(
        self,
@@ -82,14 +90,15 @@ class WhisperModel:
        cpu_threads: int = 0,
        num_workers: int = 1,
        download_root: Optional[str] = None,
-        local_files_only: bool = True,
+        local_files_only: bool = False,
    ):
        """Initializes the Whisper model.

        Args:
          model_size_or_path: Size of the model to use (tiny, tiny.en, base, base.en,
-            small, small.en, medium, medium.en, large-v1, or large-v2) or a path to a converted
-            model directory. When a size is configured, the converted model is downloaded
+            small, small.en, medium, medium.en, large-v1, large-v2, large-v3, or large), a path to a converted
+            model directory, or a CTranslate2-converted Whisper model ID from the Hugging Face Hub.
+            When a size or a model ID is configured, the converted model is downloaded
            from the Hugging Face Hub.
          device: Device to use for computation ("cpu", "cuda", "auto").
          device_index: Device ID to use.
@@ -104,8 +113,10 @@ class WhisperModel:
            having multiple workers enables true parallelism when running the model
            (concurrent calls to self.model.generate() will run in parallel).
            This can improve the global throughput at the cost of increased memory usage.
-          download_root: Directory where the model should be saved. If not set, the model
-            is saved in the standard Hugging Face cache directory.
+          download_root: Directory where the models should be saved. If not set, the models
+            are saved in the standard Hugging Face cache directory.
+          local_files_only:  If True, avoid downloading the file and return the path to the
+            local cached file if it exists.
        """
        self.logger = get_logger()

@@ -135,7 +146,8 @@ class WhisperModel:
                "openai/whisper-tiny" + ("" if self.model.is_multilingual else ".en")
            )

-        self.feature_extractor = FeatureExtractor()
+        self.feat_kwargs = self._get_feature_kwargs(model_path)
+        self.feature_extractor = FeatureExtractor(**self.feat_kwargs)
        self.num_samples_per_token = self.feature_extractor.hop_length * 2
        self.frames_per_second = (
            self.feature_extractor.sampling_rate // self.feature_extractor.hop_length
@@ -147,6 +159,27 @@ class WhisperModel:
        self.time_precision = 0.02
        self.max_length = 448

+    @property
+    def supported_languages(self) -> List[str]:
+        """The languages supported by the model."""
+        return list(_LANGUAGE_CODES) if self.model.is_multilingual else ["en"]
+
+    def _get_feature_kwargs(self, model_path) -> dict:
+        preprocessor_config_file = os.path.join(model_path, "preprocessor_config.json")
+        config = {}
+        if os.path.isfile(preprocessor_config_file):
+            try:
+                with open(preprocessor_config_file, "r", encoding="utf-8") as json_file:
+                    config = json.load(json_file)
+                valid_keys = signature(FeatureExtractor.__init__).parameters.keys()
+                config = {k: v for k, v in config.items() if k in valid_keys}
+            except json.JSONDecodeError as e:
+                self.logger.warning(
+                    "Could not load preprocessor_config.json: %s", str(e)
+                )
+
+        return config
+
    def transcribe(
        self,
        audio: Union[str, BinaryIO, np.ndarray],
@@ -156,6 +189,8 @@ class WhisperModel:
        best_of: int = 5,
        patience: float = 1,
        length_penalty: float = 1,
+        repetition_penalty: float = 1,
+        no_repeat_ngram_size: int = 0,
        temperature: Union[float, List[float], Tuple[float, ...]] = [
            0.0,
            0.2,
@@ -168,7 +203,8 @@ class WhisperModel:
        log_prob_threshold: Optional[float] = -1.0,
        no_speech_threshold: Optional[float] = 0.6,
        condition_on_previous_text: bool = True,
-        initial_prompt: Optional[str] = None,
+        prompt_reset_on_temperature: float = 0.5,
+        initial_prompt: Optional[Union[str, Iterable[int]]] = None,
        prefix: Optional[str] = None,
        suppress_blank: bool = True,
        suppress_tokens: Optional[List[int]] = [-1],
@@ -178,8 +214,8 @@ class WhisperModel:
        prepend_punctuations: str = "\"'“¿([{-",
        append_punctuations: str = "\"'.。,，!！?？:：”)]}、",
        vad_filter: bool = False,
-        vad_parameters: Optional[dict] = None,
-    ) -> Tuple[Iterable[Segment], AudioInfo]:
+        vad_parameters: Optional[Union[dict, VadOptions]] = None,
+    ) -> Tuple[Iterable[Segment], TranscriptionInfo]:
        """Transcribes an input file.

        Arguments:
@@ -192,6 +228,9 @@ class WhisperModel:
          best_of: Number of candidates when sampling with non-zero temperature.
          patience: Beam search patience factor.
          length_penalty: Exponential length penalty constant.
+          repetition_penalty: Penalty applied to the score of previously generated tokens
+            (set > 1 to penalize).
+          no_repeat_ngram_size: Prevent repetitions of ngrams with this size (set 0 to disable).
          temperature: Temperature for sampling. It can be a tuple of temperatures,
            which will be successively used upon failures according to either
            `compression_ratio_threshold` or `log_prob_threshold`.
@@ -206,7 +245,10 @@ class WhisperModel:
            as a prompt for the next window; disabling may make the text inconsistent across
            windows, but the model becomes less prone to getting stuck in a failure loop,
            such as repetition looping or timestamps going out of sync.
-          initial_prompt: Optional text to provide as a prompt for the first window.
+          prompt_reset_on_temperature: Resets prompt if temperature is above this value.
+            Arg has effect only if condition_on_previous_text is True.
+          initial_prompt: Optional text string or iterable of token ids to provide as a
+            prompt for the first window.
          prefix: Optional text to provide as a prefix for the first window.
          suppress_blank: Suppress blank outputs at the beginning of the sampling.
          suppress_tokens: List of token IDs to suppress. -1 will suppress a default set
@@ -222,14 +264,14 @@ class WhisperModel:
          vad_filter: Enable the voice activity detection (VAD) to filter out parts of the audio
            without speech. This step is using the Silero VAD model
            https://github.com/snakers4/silero-vad.
-          vad_parameters: Dictionary of Silero VAD parameters (see available parameters and
-            default values in the function `get_speech_timestamps`).
+          vad_parameters: Dictionary of Silero VAD parameters or VadOptions class (see available
+            parameters and default values in the class `VadOptions`).

        Returns:
          A tuple with:

            - a generator over transcribed segments
-            - an instance of AudioInfo
+            - an instance of TranscriptionInfo
        """
        sampling_rate = self.feature_extractor.sampling_rate

@@ -237,19 +279,24 @@ class WhisperModel:
            audio = decode_audio(audio, sampling_rate=sampling_rate)

        duration = audio.shape[0] / sampling_rate
+        duration_after_vad = duration

        self.logger.info(
            "Processing audio with duration %s", format_timestamp(duration)
        )

        if vad_filter:
-            vad_parameters = {} if vad_parameters is None else vad_parameters
-            speech_chunks = get_speech_timestamps(audio, **vad_parameters)
+            if vad_parameters is None:
+                vad_parameters = VadOptions()
+            elif isinstance(vad_parameters, dict):
+                vad_parameters = VadOptions(**vad_parameters)
+            speech_chunks = get_speech_timestamps(audio, vad_parameters)
            audio = collect_chunks(audio, speech_chunks)
+            duration_after_vad = audio.shape[0] / sampling_rate

            self.logger.info(
                "VAD filter removed %s of audio",
-                format_timestamp(duration - (audio.shape[0] / sampling_rate)),
+                format_timestamp(duration - duration_after_vad),
            )

            if self.logger.isEnabledFor(logging.DEBUG):
@@ -271,6 +318,7 @@ class WhisperModel:
        features = self.feature_extractor(audio)

        encoder_output = None
+        all_language_probs = None

        if language is None:
            if not self.model.is_multilingual:
@@ -279,17 +327,27 @@ class WhisperModel:
            else:
                segment = features[:, : self.feature_extractor.nb_max_frames]
                encoder_output = self.encode(segment)
-                results = self.model.detect_language(encoder_output)
-                language_token, language_probability = results[0][0]
-                language = language_token[2:-2]
+                # results is a list of tuple[str, float] with language names and
+                # probabilities.
+                results = self.model.detect_language(encoder_output)[0]
+                # Parse language names to strip out markers
+                all_language_probs = [(token[2:-2], prob) for (token, prob) in results]
+                # Get top language token and probability
+                language, language_probability = all_language_probs[0]

                self.logger.info(
                    "Detected language '%s' with probability %.2f",
                    language,
                    language_probability,
                )
-                return language, language_probability
        else:
+            if not self.model.is_multilingual and language != "en":
+                self.logger.warning(
+                    "The current model is English-only but the language parameter is set to '%s'; "
+                    "using 'en' instead." % language
+                )
+                language = "en"
+
            language_probability = 1

        tokenizer = Tokenizer(
@@ -304,10 +362,13 @@ class WhisperModel:
            best_of=best_of,
            patience=patience,
            length_penalty=length_penalty,
+            repetition_penalty=repetition_penalty,
+            no_repeat_ngram_size=no_repeat_ngram_size,
            log_prob_threshold=log_prob_threshold,
            no_speech_threshold=no_speech_threshold,
            compression_ratio_threshold=compression_ratio_threshold,
            condition_on_previous_text=condition_on_previous_text,
+            prompt_reset_on_temperature=prompt_reset_on_temperature,
            temperatures=(
                temperature if isinstance(temperature, (list, tuple)) else [temperature]
            ),
@@ -327,13 +388,17 @@ class WhisperModel:
        if speech_chunks:
            segments = restore_speech_timestamps(segments, speech_chunks, sampling_rate)

-        audio_info = AudioInfo(
+        info = TranscriptionInfo(
            language=language,
            language_probability=language_probability,
            duration=duration,
+            duration_after_vad=duration_after_vad,
+            transcription_options=options,
+            vad_options=vad_parameters,
+            all_language_probs=all_language_probs,
        )

-        return segments
+        return segments, info

    def generate_segments(
        self,
@@ -343,14 +408,20 @@ class WhisperModel:
        encoder_output: Optional[ctranslate2.StorageView] = None,
    ) -> Iterable[Segment]:
        content_frames = features.shape[-1] - self.feature_extractor.nb_max_frames
+        idx = 0
        seek = 0
        all_tokens = []
        prompt_reset_since = 0

        if options.initial_prompt is not None:
-            initial_prompt = " " + options.initial_prompt.strip()
-            initial_prompt_tokens = tokenizer.encode(initial_prompt)
-            all_tokens.extend(initial_prompt_tokens)
+            if isinstance(options.initial_prompt, str):
+                initial_prompt = " " + options.initial_prompt.strip()
+                initial_prompt_tokens = tokenizer.encode(initial_prompt)
+                all_tokens.extend(initial_prompt_tokens)
+            else:
+                all_tokens.extend(options.initial_prompt)
+
+        last_speech_timestamp = 0.0
        all_segments = []
        while seek < content_frames:
            time_offset = seek * self.feature_extractor.time_per_frame
@@ -373,12 +444,15 @@ class WhisperModel:
                prefix=options.prefix if seek == 0 else None,
            )

-            if encoder_output is None:
+            if seek > 0 or encoder_output is None:
                encoder_output = self.encode(segment)

-            result, avg_log_prob, temperature = self.generate_with_fallback(
-                encoder_output, prompt, tokenizer, options
-            )
+            (
+                result,
+                avg_logprob,
+                temperature,
+                compression_ratio,
+            ) = self.generate_with_fallback(encoder_output, prompt, tokenizer, options)

            if options.no_speech_threshold is not None:
                # no voice activity check
@@ -386,7 +460,7 @@ class WhisperModel:

                if (
                    options.log_prob_threshold is not None
-                    and avg_log_prob > options.log_prob_threshold
+                    and avg_logprob > options.log_prob_threshold
                ):
                    # don't skip if the logprob is high enough, despite the no_speech_prob
                    should_skip = False
@@ -482,9 +556,6 @@ class WhisperModel:

                seek += segment_size

-            if not options.condition_on_previous_text or temperature > 0.5:
-                prompt_reset_since = len(all_tokens)
-
            if options.word_timestamps:
                self.add_word_timestamps(
                    current_segments,
@@ -493,12 +564,14 @@ class WhisperModel:
                    segment_size,
                    options.prepend_punctuations,
                    options.append_punctuations,
+                    last_speech_timestamp=last_speech_timestamp,
                )

                word_end_timestamps = [
                    w["end"] for s in current_segments for w in s["words"]
                ]
-
+                if len(word_end_timestamps) > 0:
+                    last_speech_timestamp = word_end_timestamps[-1]
                if not single_timestamp_ending and len(word_end_timestamps) > 0:
                    seek_shift = round(
                        (word_end_timestamps[-1] - time_offset) * self.frames_per_second
@@ -507,8 +580,6 @@ class WhisperModel:
                    if seek_shift > 0:
                        seek = previous_seek + seek_shift

-            encoder_output = None
-            
            for segment in current_segments:
                tokens = segment["tokens"]
                text = tokenizer.decode(tokens)
@@ -517,19 +588,38 @@ class WhisperModel:
                    continue

                all_tokens.extend(tokens)
+                idx += 1

                all_segments.append(Segment(
+                    id=idx,
+                    seek=seek,
                    start=segment["start"],
                    end=segment["end"],
                    text=text,
+                    tokens=tokens,
+                    temperature=temperature,
+                    avg_logprob=avg_logprob,
+                    compression_ratio=compression_ratio,
+                    no_speech_prob=result.no_speech_prob,
                    words=(
                        [Word(**word) for word in segment["words"]]
                        if options.word_timestamps
                        else None
-                    ),
-                    avg_log_prob=avg_log_prob,
-                    no_speech_prob=result.no_speech_prob,
+                        ),
                ))
+
+            if (
+                not options.condition_on_previous_text
+                or temperature > options.prompt_reset_on_temperature
+            ):
+                if options.condition_on_previous_text:
+                    self.logger.debug(
+                        "Reset prompt. prompt_reset_on_temperature threshold is met %f > %f",
+                        temperature,
+                        options.prompt_reset_on_temperature,
+                    )
+
+                prompt_reset_since = len(all_tokens)
        return all_segments

    def encode(self, features: np.ndarray) -> ctranslate2.StorageView:
@@ -548,10 +638,10 @@ class WhisperModel:
        prompt: List[int],
        tokenizer: Tokenizer,
        options: TranscriptionOptions,
-    ) -> Tuple[ctranslate2.models.WhisperGenerationResult, float, float]:
-        result = None
-        avg_log_prob = None
-        final_temperature = None
+    ) -> Tuple[ctranslate2.models.WhisperGenerationResult, float, float, float]:
+        decode_result = None
+        all_results = []
+        below_cr_threshold_results = []

        max_initial_timestamp_index = int(
            round(options.max_initial_timestamp / self.time_precision)
@@ -571,11 +661,12 @@ class WhisperModel:
                    "patience": options.patience,
                }

-            final_temperature = temperature
            result = self.model.generate(
                encoder_output,
                [prompt],
                length_penalty=options.length_penalty,
+                repetition_penalty=options.repetition_penalty,
+                no_repeat_ngram_size=options.no_repeat_ngram_size,
                max_length=self.max_length,
                return_scores=True,
                return_no_speech_prob=True,
@@ -589,44 +680,63 @@ class WhisperModel:

            # Recover the average log prob from the returned score.
            seq_len = len(tokens)
-            cum_log_prob = result.scores[0] * (seq_len**options.length_penalty)
-            avg_log_prob = cum_log_prob / (seq_len + 1)
+            cum_logprob = result.scores[0] * (seq_len**options.length_penalty)
+            avg_logprob = cum_logprob / (seq_len + 1)

            text = tokenizer.decode(tokens).strip()
            compression_ratio = get_compression_ratio(text)

+            decode_result = (
+                result,
+                avg_logprob,
+                temperature,
+                compression_ratio,
+            )
+            all_results.append(decode_result)
+
            needs_fallback = False

-            if (
-                options.compression_ratio_threshold is not None
-                and compression_ratio > options.compression_ratio_threshold
-            ):
-                needs_fallback = True  # too repetitive
+            if options.compression_ratio_threshold is not None:
+                if compression_ratio > options.compression_ratio_threshold:
+                    needs_fallback = True  # too repetitive

-                self.logger.debug(
-                    "Compression ratio threshold is not met with temperature %.1f (%f > %f)",
-                    temperature,
-                    compression_ratio,
-                    options.compression_ratio_threshold,
-                )
+                    self.logger.debug(
+                        "Compression ratio threshold is not met with temperature %.1f (%f > %f)",
+                        temperature,
+                        compression_ratio,
+                        options.compression_ratio_threshold,
+                    )
+                else:
+                    below_cr_threshold_results.append(decode_result)

            if (
                options.log_prob_threshold is not None
-                and avg_log_prob < options.log_prob_threshold
+                and avg_logprob < options.log_prob_threshold
            ):
                needs_fallback = True  # average log probability is too low

                self.logger.debug(
                    "Log probability threshold is not met with temperature %.1f (%f < %f)",
                    temperature,
-                    avg_log_prob,
+                    avg_logprob,
                    options.log_prob_threshold,
                )

+            if (
+                options.no_speech_threshold is not None
+                and result.no_speech_prob > options.no_speech_threshold
+            ):
+                needs_fallback = False  # silence
+
            if not needs_fallback:
                break
+        else:
+            # all failed, select the result with the highest average log probability
+            decode_result = max(
+                below_cr_threshold_results or all_results, key=lambda x: x[1]
+            )

-        return result, avg_log_prob, final_temperature
+        return decode_result

    def get_prompt(
        self,
@@ -650,6 +760,8 @@ class WhisperModel:
            prefix_tokens = tokenizer.encode(" " + prefix.strip())
            if len(prefix_tokens) >= self.max_length // 2:
                prefix_tokens = prefix_tokens[: self.max_length // 2 - 1]
+            if not without_timestamps:
+                prompt.append(tokenizer.timestamp_begin)
            prompt.extend(prefix_tokens)

        return prompt
@@ -662,7 +774,8 @@ class WhisperModel:
        num_frames: int,
        prepend_punctuations: str,
        append_punctuations: str,
-    ):
+        last_speech_timestamp: float,
+    ) -> None:
        if len(segments) == 0:
            return

@@ -675,6 +788,24 @@ class WhisperModel:
        alignment = self.find_alignment(
            tokenizer, text_tokens, encoder_output, num_frames
        )
+        word_durations = np.array([word["end"] - word["start"] for word in alignment])
+        word_durations = word_durations[word_durations.nonzero()]
+        median_duration = np.median(word_durations) if len(word_durations) > 0 else 0.0
+        max_duration = median_duration * 2
+
+        # hack: truncate long words at sentence boundaries.
+        # a better segmentation algorithm based on VAD should be able to replace this.
+        if len(word_durations) > 0:
+            sentence_end_marks = ".。!！?？"
+            # ensure words at sentence boundaries
+            # are not longer than twice the median word duration.
+            for i in range(1, len(alignment)):
+                if alignment[i]["end"] - alignment[i]["start"] > max_duration:
+                    if alignment[i]["word"] in sentence_end_marks:
+                        alignment[i]["end"] = alignment[i]["start"] + max_duration
+                    elif alignment[i - 1]["word"] in sentence_end_marks:
+                        alignment[i]["start"] = alignment[i]["end"] - max_duration
+
        merge_punctuations(alignment, prepend_punctuations, append_punctuations)

        time_offset = (
@@ -705,10 +836,51 @@ class WhisperModel:
                saved_tokens += len(timing["tokens"])
                word_index += 1

+            # hack: truncate long words at segment boundaries.
+            # a better segmentation algorithm based on VAD should be able to replace this.
            if len(words) > 0:
-                # adjust the segment-level timestamps based on the word-level timestamps
-                segment["start"] = words[0]["start"]
-                segment["end"] = words[-1]["end"]
+                # ensure the first and second word after a pause is not longer than
+                # twice the median word duration.
+                if words[0]["end"] - last_speech_timestamp > median_duration * 4 and (
+                    words[0]["end"] - words[0]["start"] > max_duration
+                    or (
+                        len(words) > 1
+                        and words[1]["end"] - words[0]["start"] > max_duration * 2
+                    )
+                ):
+                    if (
+                        len(words) > 1
+                        and words[1]["end"] - words[1]["start"] > max_duration
+                    ):
+                        boundary = max(
+                            words[1]["end"] / 2, words[1]["end"] - max_duration
+                        )
+                        words[0]["end"] = words[1]["start"] = boundary
+                    words[0]["start"] = max(0, words[0]["end"] - max_duration)
+
+                # prefer the segment-level start timestamp if the first word is too long.
+                if (
+                    segment["start"] < words[0]["end"]
+                    and segment["start"] - 0.5 > words[0]["start"]
+                ):
+                    words[0]["start"] = max(
+                        0, min(words[0]["end"] - median_duration, segment["start"])
+                    )
+                else:
+                    segment["start"] = words[0]["start"]
+
+                # prefer the segment-level end timestamp if the last word is too long.
+                if (
+                    segment["end"] > words[-1]["start"]
+                    and segment["end"] + 0.5 < words[-1]["end"]
+                ):
+                    words[-1]["end"] = max(
+                        words[-1]["start"] + median_duration, segment["end"]
+                    )
+                else:
+                    segment["end"] = words[-1]["end"]
+
+                last_speech_timestamp = segment["end"]

            segment["words"] = words

@@ -741,6 +913,8 @@ class WhisperModel:
            text_tokens + [tokenizer.eot]
        )
        word_boundaries = np.pad(np.cumsum([len(t) for t in word_tokens[:-1]]), (1, 0))
+        if len(word_boundaries) <= 1:
+            return []

        jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)
        jump_times = time_indices[jumps] / self.tokens_per_second
@@ -751,22 +925,6 @@ class WhisperModel:
            for i, j in zip(word_boundaries[:-1], word_boundaries[1:])
        ]

-        # hack: ensure the first and second word is not longer than twice the median word duration.
-        # a better segmentation algorithm based on VAD should be able to replace this.
-        word_durations = end_times - start_times
-        word_durations = word_durations[word_durations.nonzero()]
-        if len(word_durations) > 0:
-            median_duration = np.median(word_durations)
-            max_duration = median_duration * 2
-            if len(word_durations) >= 2 and word_durations[1] > max_duration:
-                boundary = max(end_times[2] / 2, end_times[2] - max_duration)
-                end_times[0] = start_times[1] = boundary
-            if (
-                len(word_durations) >= 1
-                and end_times[0] - start_times[0] > max_duration
-            ):
-                start_times[0] = max(0, end_times[0] - max_duration)
-
        return [
            dict(
                word=word, tokens=tokens, start=start, end=end, probability=probability
@@ -775,9 +933,6 @@ class WhisperModel:
                words, word_tokens, start_times, end_times, word_probabilities
            )
        ]
-    
-    def destroy(self):
-        del self.model


 def restore_speech_timestamps(
@@ -792,7 +947,8 @@ def restore_speech_timestamps(
            words = []
            for word in segment.words:
                # Ensure the word start and end times are resolved to the same chunk.
-                chunk_index = ts_map.get_chunk_index(word.start)
+                middle = (word.start + word.end) / 2
+                chunk_index = ts_map.get_chunk_index(middle)
                word = word._replace(
                    start=ts_map.get_original_time(word.start, chunk_index),
                    end=ts_map.get_original_time(word.end, chunk_index),
@@ -811,7 +967,7 @@ def restore_speech_timestamps(
                end=ts_map.get_original_time(segment.end),
            )

-        yield segment
+    return segments


 def get_ctranslate2_storage(segment: np.ndarray) -> ctranslate2.StorageView:
@@ -825,7 +981,10 @@ def get_compression_ratio(text: str) -> float:
    return len(text_bytes) / len(zlib.compress(text_bytes))


-def get_suppressed_tokens(tokenizer, suppress_tokens):
+def get_suppressed_tokens(
+    tokenizer: Tokenizer,
+    suppress_tokens: Optional[List[int]],
+) -> Optional[List[int]]:
    if not suppress_tokens or -1 in suppress_tokens:
        return suppress_tokens

@@ -846,7 +1005,7 @@ def get_suppressed_tokens(tokenizer, suppress_tokens):
    return sorted(set(suppress_tokens))


-def merge_punctuations(alignment: List[dict], prepended: str, appended: str):
+def merge_punctuations(alignment: List[dict], prepended: str, appended: str) -> None:
    # merge prepended punctuations
    i = len(alignment) - 2
    j = len(alignment) - 1
@@ -0,0 +1,340 @@
+import argparse
+import json
+import re
+import time
+from collections import OrderedDict
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, TextIO, Tuple, Union
+
+import torch
+import numpy as np
+from whisper.tokenizer import get_tokenizer
+from whisper_live.tensorrt_utils import (mel_filters, store_transcripts,
+                           write_error_stats, load_audio_wav_format,
+                           pad_or_trim, load_audio)
+
+import tensorrt_llm
+import tensorrt_llm.logger as logger
+from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt,
+                                 trt_dtype_to_torch)
+from tensorrt_llm.runtime import ModelConfig, SamplingConfig
+from tensorrt_llm.runtime.session import Session, TensorInfo
+
+
+SAMPLE_RATE = 16000
+N_FFT = 400
+HOP_LENGTH = 160
+CHUNK_LENGTH = 30
+N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE  # 480000 samples in a 30-second chunk
+
+
+class WhisperEncoding:
+
+    def __init__(self, engine_dir):
+        self.session = self.get_session(engine_dir)
+
+    def get_session(self, engine_dir):
+        config_path = engine_dir / 'encoder_config.json'
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+
+        use_gpt_attention_plugin = config['plugin_config'][
+            'gpt_attention_plugin']
+        dtype = config['builder_config']['precision']
+        n_mels = config['builder_config']['n_mels']
+        num_languages = config['builder_config']['num_languages']
+
+        self.dtype = dtype
+        self.n_mels = n_mels
+        self.num_languages = num_languages
+
+        serialize_path = engine_dir / f'whisper_encoder_{self.dtype}_tp1_rank0.engine'
+
+        with open(serialize_path, 'rb') as f:
+            session = Session.from_serialized_engine(f.read())
+
+        return session
+
+    def get_audio_features(self, mel):
+        inputs = OrderedDict()
+        output_list = []
+
+        inputs.update({'x': mel})
+        output_list.append(
+            TensorInfo('x', str_dtype_to_trt(self.dtype), mel.shape))
+
+        output_info = (self.session).infer_shapes(output_list)
+
+        logger.debug(f'output info {output_info}')
+        outputs = {
+            t.name: torch.empty(tuple(t.shape),
+                                dtype=trt_dtype_to_torch(t.dtype),
+                                device='cuda')
+            for t in output_info
+        }
+        stream = torch.cuda.current_stream()
+        ok = self.session.run(inputs=inputs,
+                              outputs=outputs,
+                              stream=stream.cuda_stream)
+        assert ok, 'Engine execution failed'
+        stream.synchronize()
+        audio_features = outputs['output']
+        return audio_features
+
+
+class WhisperDecoding:
+
+    def __init__(self, engine_dir, runtime_mapping, debug_mode=False):
+
+        self.decoder_config = self.get_config(engine_dir)
+        self.decoder_generation_session = self.get_session(
+            engine_dir, runtime_mapping, debug_mode)
+
+    def get_config(self, engine_dir):
+        config_path = engine_dir / 'decoder_config.json'
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+        decoder_config = OrderedDict()
+        decoder_config.update(config['plugin_config'])
+        decoder_config.update(config['builder_config'])
+        return decoder_config
+
+    def get_session(self, engine_dir, runtime_mapping, debug_mode=False):
+        dtype = self.decoder_config['precision']
+        serialize_path = engine_dir / f'whisper_decoder_{dtype}_tp1_rank0.engine'
+        with open(serialize_path, "rb") as f:
+            decoder_engine_buffer = f.read()
+
+        decoder_model_config = ModelConfig(
+            num_heads=self.decoder_config['num_heads'],
+            num_kv_heads=self.decoder_config['num_heads'],
+            hidden_size=self.decoder_config['hidden_size'],
+            vocab_size=self.decoder_config['vocab_size'],
+            num_layers=self.decoder_config['num_layers'],
+            gpt_attention_plugin=self.decoder_config['gpt_attention_plugin'],
+            remove_input_padding=self.decoder_config['remove_input_padding'],
+            cross_attention=self.decoder_config['cross_attention'],
+            has_position_embedding=self.
+            decoder_config['has_position_embedding'],
+            has_token_type_embedding=self.
+            decoder_config['has_token_type_embedding'],
+        )
+        decoder_generation_session = tensorrt_llm.runtime.GenerationSession(
+            decoder_model_config,
+            decoder_engine_buffer,
+            runtime_mapping,
+            debug_mode=debug_mode)
+
+        return decoder_generation_session
+
+    def generate(self,
+                 decoder_input_ids,
+                 encoder_outputs,
+                 eot_id,
+                 max_new_tokens=40,
+                 num_beams=1):
+        encoder_input_lengths = torch.tensor(
+            [encoder_outputs.shape[1] for x in range(encoder_outputs.shape[0])],
+            dtype=torch.int32,
+            device='cuda')
+
+        decoder_input_lengths = torch.tensor([
+            decoder_input_ids.shape[-1]
+            for _ in range(decoder_input_ids.shape[0])
+        ],
+                                             dtype=torch.int32,
+                                             device='cuda')
+        decoder_max_input_length = torch.max(decoder_input_lengths).item()
+
+        # generation config
+        sampling_config = SamplingConfig(end_id=eot_id,
+                                         pad_id=eot_id,
+                                         num_beams=num_beams)
+        self.decoder_generation_session.setup(
+            decoder_input_lengths.size(0),
+            decoder_max_input_length,
+            max_new_tokens,
+            beam_width=num_beams,
+            encoder_max_input_length=encoder_outputs.shape[1])
+
+        torch.cuda.synchronize()
+
+        decoder_input_ids = decoder_input_ids.type(torch.int32).cuda()
+        output_ids = self.decoder_generation_session.decode(
+            decoder_input_ids,
+            decoder_input_lengths,
+            sampling_config,
+            encoder_output=encoder_outputs,
+            encoder_input_lengths=encoder_input_lengths,
+        )
+        torch.cuda.synchronize()
+
+        # get the list of int from output_ids tensor
+        output_ids = output_ids.cpu().numpy().tolist()
+        return output_ids
+
+
+class WhisperTRTLLM(object):
+
+    def __init__(
+        self,
+        engine_dir,
+        debug_mode=False,
+        assets_dir=None,
+        device=None,
+        is_multilingual=False,
+        language="en",
+        task="transcribe"
+        ):
+        world_size = 1
+        runtime_rank = tensorrt_llm.mpi_rank()
+        runtime_mapping = tensorrt_llm.Mapping(world_size, runtime_rank)
+        torch.cuda.set_device(runtime_rank % runtime_mapping.gpus_per_node)
+        engine_dir = Path(engine_dir)
+
+        self.encoder = WhisperEncoding(engine_dir)
+        self.decoder = WhisperDecoding(engine_dir,
+                                       runtime_mapping,
+                                       debug_mode=False)
+        self.n_mels = self.encoder.n_mels
+        # self.tokenizer = get_tokenizer(num_languages=self.encoder.num_languages,
+        #                                tokenizer_dir=assets_dir)
+        self.device = device
+        self.tokenizer = get_tokenizer(
+            is_multilingual,
+            num_languages=self.encoder.num_languages,
+            language=language,
+            task=task,
+        )
+        self.filters = mel_filters(self.device, self.encoder.n_mels, assets_dir)
+
+    def log_mel_spectrogram(
+        self,
+        audio: Union[str, np.ndarray, torch.Tensor],
+        padding: int = 0,
+        return_duration = True
+    ):
+        """
+        Compute the log-Mel spectrogram of
+
+        Parameters
+        ----------
+        audio: Union[str, np.ndarray, torch.Tensor], shape = (*)
+            The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz
+
+        n_mels: int
+            The number of Mel-frequency filters, only 80 and 128 are supported
+
+        padding: int
+            Number of zero samples to pad to the right
+
+        device: Optional[Union[str, torch.device]]
+            If given, the audio tensor is moved to this device before STFT
+
+        Returns
+        -------
+        torch.Tensor, shape = (80 or 128, n_frames)
+            A Tensor that contains the Mel spectrogram
+        """
+        if not torch.is_tensor(audio):
+            if isinstance(audio, str):
+                if audio.endswith('.wav'):
+                    audio, _ = load_audio_wav_format(audio)
+                else:
+                    audio = load_audio(audio)
+            assert isinstance(audio,
+                            np.ndarray), f"Unsupported audio type: {type(audio)}"
+            duration = audio.shape[-1] / SAMPLE_RATE
+            audio = pad_or_trim(audio, N_SAMPLES)
+            audio = audio.astype(np.float32)
+            audio = torch.from_numpy(audio)
+
+        if self.device is not None:
+            audio = audio.to(self.device)
+        if padding > 0:
+            audio = F.pad(audio, (0, padding))
+        window = torch.hann_window(N_FFT).to(audio.device)
+        stft = torch.stft(audio,
+                        N_FFT,
+                        HOP_LENGTH,
+                        window=window,
+                        return_complex=True)
+        magnitudes = stft[..., :-1].abs()**2
+
+        
+        mel_spec = self.filters @ magnitudes
+
+        log_spec = torch.clamp(mel_spec, min=1e-10).log10()
+        log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
+        log_spec = (log_spec + 4.0) / 4.0
+        if return_duration:
+            return log_spec, duration
+        else:
+            return log_spec
+
+
+    def process_batch(
+            self,
+            mel,
+            text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
+            num_beams=1):
+        prompt_id = self.tokenizer.encode(
+            text_prefix, allowed_special=set(self.tokenizer.special_tokens.keys()))
+
+        prompt_id = torch.tensor(prompt_id)
+        batch_size = mel.shape[0]
+        decoder_input_ids = prompt_id.repeat(batch_size, 1)
+
+        encoder_output = self.encoder.get_audio_features(mel)
+        output_ids = self.decoder.generate(decoder_input_ids,
+                                           encoder_output,
+                                           self.tokenizer.eot,
+                                           max_new_tokens=96,
+                                           num_beams=num_beams)
+        texts = []
+        for i in range(len(output_ids)):
+            text = self.tokenizer.decode(output_ids[i][0]).strip()
+            texts.append(text)
+        return texts
+    
+    def transcribe(
+            self,
+            mel,
+            text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
+            dtype='float16',
+            batch_size=1,
+            num_beams=1,
+            ):
+        mel = mel.type(str_dtype_to_torch(dtype))
+        mel = mel.unsqueeze(0)
+        predictions = self.process_batch(mel, text_prefix, num_beams)
+        prediction = predictions[0]
+
+        # remove all special tokens in the prediction
+        prediction = re.sub(r'<\|.*?\|>', '', prediction)
+        return prediction.strip()
+
+
+def decode_wav_file(
+        model,
+        mel,
+        text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
+        dtype='float16',
+        batch_size=1,
+        num_beams=1,
+        normalizer=None,
+        mel_filters_dir=None):
+
+    mel = mel.type(str_dtype_to_torch(dtype))
+    mel = mel.unsqueeze(0)
+    # repeat the mel spectrogram to match the batch size
+    mel = mel.repeat(batch_size, 1, 1)
+    predictions = model.process_batch(mel, text_prefix, num_beams)
+    prediction = predictions[0]
+
+    # remove all special tokens in the prediction
+    prediction = re.sub(r'<\|.*?\|>', '', prediction)
+    if normalizer:
+        prediction = normalizer(prediction)
+    
+    return prediction.strip()
@@ -10,19 +10,23 @@ import onnxruntime
 class VoiceActivityDetection():

    def __init__(self, force_onnx_cpu=True):
+        print("downloading ONNX model...")
        path = self.download()
+        print("loading session")
+
        opts = onnxruntime.SessionOptions()
        opts.log_severity_level = 3

        opts.inter_op_num_threads = 1
        opts.intra_op_num_threads = 1

+        print("loading onnx model")
        if force_onnx_cpu and 'CPUExecutionProvider' in onnxruntime.get_available_providers():
            self.session = onnxruntime.InferenceSession(path, providers=['CPUExecutionProvider'], sess_options=opts)
        else:
            self.session = onnxruntime.InferenceSession(path, providers=['CUDAExecutionProvider'], sess_options=opts)

-
+        print("reset states")
        self.reset_states()
        self.sample_rates = [8000, 16000]

@@ -111,5 +115,4 @@ class VoiceActivityDetection():
                subprocess.run(["wget", "-O", model_filename, model_url], check=True)
            except subprocess.CalledProcessError:
                print("Failed to download the model using wget.")
-        return model_filename
-
+        return model_filename
Author	SHA1	Message	Date
makaveli10	4ecc59783e	bump version v0.1.0	2024-02-05 22:37:11 +05:30
makaveli	ec9074d712	Merge pull request #128 from lightwastak3n/firefox_remove_multilingual Firefox remove multilingual	2024-02-05 21:42:10 +05:30
Marcus Edel	3a25db4cb9	Merge pull request #127 from makaveli10/update_setup_requirements Update required packages for setup.	2024-02-05 08:11:01 -05:00
makaveli	4924ec0adb	remove empty lines	2024-02-05 18:40:03 +05:30
makaveli	f35abc7f81	Merge branch 'collabora:main' into update_setup_requirements	2024-02-05 18:26:38 +05:30
Sasa Trivic	f383121ec3	Remove multilingual option description from the extension readme	2024-02-04 18:03:41 +01:00
Sasa Trivic	17d62272cf	Update README.md Remove multilingual from README	2024-02-04 17:52:21 +01:00
Sasa Trivic	91e1b75bfc	Merge branch 'collabora:main' into firefox_remove_multilingual	2024-02-04 17:36:57 +01:00
Sasa Trivic	7aad2ae721	Remove multilingual from Firefox. Sort languages, disable all inputs when capturing. Move both transcripts to the bottom center.	2024-02-04 17:35:20 +01:00
makaveli10	6a1b82f953	update required packages for setup	2024-02-04 16:02:02 +05:30
makaveli	dc84839873	Merge pull request #126 from makaveli10/fix_typo_multilingual fix: typo; remove multilingual debug stat	2024-02-04 14:38:54 +05:30
makaveli10	56d19f5469	fix: typo; remove multilingual debug stat	2024-02-04 14:34:16 +05:30
makaveli	08fa183ba4	Merge pull request #123 from lightwastak3n/main Chrome extension update; remove multililingual option faster-whisper	2024-02-03 19:24:45 +05:30
Marcus Edel	d89b27b8aa	Merge pull request #124 from Stinosko/patch-1 Add scipy to server.txt.	2024-02-02 15:31:35 -05:00
Stinosko	ce68cc6c87	Add scipy to server.txt The server script uses scipy but is not installed with the current server requirements file.	2024-02-02 21:07:57 +01:00
Sasa Trivic	e697574870	Remove multilingual from client. Remove multilingual from faster whisper backend. Disable task dropdown when capturing in chrome extension.	2024-02-02 13:56:26 +01:00
Sasa Trivic	b098b52a4d	Merge branch 'main' of github.com:lightwastak3n/WhisperLive	2024-02-01 17:37:18 +01:00
Sasa Trivic	ad5543b03e	Merge remote-tracking branch 'upstream/main'	2024-02-01 17:31:35 +01:00
Marcus Edel	60455b1583	Merge pull request #121 from makaveli10/save_transcript Save transcript.	2024-02-01 11:11:05 -05:00
Sasa Trivic	cb458fc207	Merge pull request #1 from lightwastak3n/extension_rewrite Extension rewrite	2024-02-01 15:16:47 +01:00
Sasa Trivic	32ed089a76	Change faster whisper to work with new extension	2024-02-01 14:54:29 +01:00
Sasa Trivic	5b28ddefbd	Chrome extension - QOL. Remove multilingual part.	2024-02-01 14:44:07 +01:00
Sasa Trivic	e1f531eccf	Remove duplicate assignment	2024-02-01 13:56:09 +01:00
Sasa Trivic	8200207530	Center transcription div	2024-02-01 13:16:23 +01:00
makaveli10	08575a03c2	write srt file only for faster_whisper backend	2024-02-01 14:22:45 +05:30
makaveli10	f590446865	Merge remote-tracking branch 'upstream/main' into save_transcript	2024-02-01 12:14:09 +05:30
Sasa Trivic	36d137888e	Merge branch 'collabora:main' into main	2024-01-31 18:03:29 +01:00
Marcus Edel	e64bc9f3d6	Merge pull request #116 from makaveli10/tensorrt_model_warmup Tensorrt model warmup.	2024-01-31 11:49:40 -05:00
Sasa Trivic	7cc945aded	self.client_uid accessed without being defined	2024-01-31 16:45:31 +01:00
makaveli	2c8a25d355	Merge pull request #119 from gchust/main fix: keyError: 'model' in server, when using browser extension	2024-01-31 18:40:27 +05:30
makaveli10	f4027de343	add: save_transcript to srt file	2024-01-31 17:37:30 +05:30
makaveli10	d1754d2c46	fix: model_size, no_speech, segment timings	2024-01-31 17:37:03 +05:30
gchust	0e6b1c0632	fix: keyError: 'model' in server, when using browser extension	2024-01-31 11:33:48 +00:00
makaveli	d6b51ccd7d	Update TensorRT_whisper.md typo: setup.sh file path	2024-01-29 12:33:12 +05:30
makaveli	2f3c1cd172	Update TensorRT_whisper.md fix: typo in bash script name to build tensorrt engine	2024-01-29 12:27:51 +05:30
makaveli	703263b375	wamrup tensorrt engine	2024-01-29 12:17:27 +05:30
makaveli	30d2cffb93	load audio for warmup	2024-01-29 12:15:39 +05:30
makaveli	4d94c6b38b	Update server.txt install ffmpeg-python to load test file for model warmup	2024-01-29 12:10:18 +05:30
makaveli	d5a0f5859e	Update TensorRT_whisper.md ffmpeg is needed for model warmup	2024-01-29 12:06:56 +05:30
makaveli	025873d2ca	Update TensorRT server requirements	2024-01-29 11:57:03 +05:30
makaveli	8c36768f7f	Merge pull request #112 from lightwastak3n/main Readme: Fix transcribe examples	2024-01-26 00:27:43 +05:30
Sasa Trivic	ce13e7b622	Fix transcribe examples	2024-01-25 18:44:23 +01:00
makaveli	3498787ccd	Merge pull request #104 from makaveli10/tensorrt_backend Tensorrt backend	2024-01-24 16:42:18 +05:30
makaveli	5cd59b1e4c	Update README.md Co-authored-by: Marcus Edel <marcus.edel@fu-berlin.de>	2024-01-24 10:09:22 +05:30
makaveli10	bd543295f3	update readme	2024-01-22 11:54:21 +00:00
makaveli10	8e2642283a	update tensorrt readme	2024-01-22 11:48:23 +00:00
makaveli10	3bf5b47947	fix: server; remove debug stats	2024-01-22 11:47:01 +00:00
makaveli10	634dae835b	add numba to req(trt-llm)	2024-01-22 11:45:11 +00:00
makaveli10	969a5aa9e5	remove trt_llm install script	2024-01-22 11:44:48 +00:00
makaveli10	44a2e20c68	remove trt-llm dockerfile	2024-01-22 11:44:21 +00:00
makaveli10	986823dbef	Merge remote-tracking branch 'upstream/main' into tensorrt_backend	2024-01-19 10:45:01 -05:00
makaveli10	1e2faa3f2b	fix: tensorrt llm idocker setup & docs	2024-01-19 10:40:59 -05:00
makaveli10	e3084b34cb	update tensorrt docker & readme	2024-01-19 07:30:50 -05:00
makaveli10	b955e63dc1	update READM	2024-01-19 12:02:18 +00:00
makaveli10	f25ff1785a	increase chunk size from 64ms to 256ms	2024-01-19 12:02:05 +00:00
makaveli10	867ff522ae	add tensorrt installation & whisper conversion script	2024-01-19 11:59:16 +00:00
makaveli10	75001ae6b7	updatetensorrt-llm dockerfile	2024-01-19 11:58:40 +00:00
makaveli10	6f1d13f25b	update requirements	2024-01-19 11:57:46 +00:00
makaveli10	7a9dc6db40	add tensorrt readme	2024-01-19 11:53:47 +00:00
makaveli10	735d6c7763	merge with main	2024-01-19 11:43:21 +00:00
Marcus Edel	0942dc2cfd	Merge pull request #102 from makaveli10/change_model_size_param_name Server to control custom model usage.	2024-01-18 11:07:18 -05:00
makaveli10	881fd55776	run server with custom model from args	2024-01-18 15:02:51 +08:00
makaveli	0c01d7b1e5	Merge pull request #98 from makaveli10/change_model_size_param_name Change model size param name	2024-01-15 21:08:26 +05:30
makaveli10	c810369324	revert the default port of chrom/firefox extension to 9090	2024-01-15 23:30:05 +08:00
makaveli10	71d0fe69c6	add option to use custom model	2024-01-15 23:28:02 +08:00
makaveli10	67232fffd5	install whl	2024-01-12 11:39:05 +00:00
makaveli10	076aebf3b6	bump version v0.0.11	2024-01-12 15:21:27 +05:30
makaveli10	4cf9d95f73	merge main	2024-01-12 08:17:46 +00:00
makaveli10	389bb5ae37	add docker setup for tensorrt-llm; update readme	2024-01-12 08:15:53 +00:00
Marcus Edel	a7eedc5d84	Merge pull request #94 from makaveli10/fix_error_messages Fix: error messages.	2024-01-11 09:52:07 -05:00
makaveli	d91330d790	Merge branch 'collabora:main' into fix_error_messages	2024-01-11 18:18:58 +05:30
makaveli	783d147316	Merge pull request #96 from makaveli10/fix_key_error fix: key error	2024-01-11 16:29:07 +05:30
makaveli10	058c93e55e	fix: key error	2024-01-11 18:55:43 +08:00
makaveli10	f06b9bc827	remove torch req	2024-01-11 08:18:51 +00:00
makaveli10	3c202bf836	update README; add TensorRT doc	2024-01-11 08:18:25 +00:00
makaveli10	647c576e6a	update with multilingual option	2024-01-11 08:17:56 +00:00
makaveli10	71a062b726	update dockerfiles	2024-01-10 14:30:57 +00:00
makaveli10	a26f990586	update readme to new setup.sh path	2024-01-10 14:22:51 +00:00
makaveli10	ddb1e0947f	move setup.sh to scripts	2024-01-10 14:22:20 +00:00
makaveli10	6dff4fbdd3	add tensorrt_llm installation script	2024-01-10 14:21:39 +00:00
makaveli10	244ca9e6ba	remove duplicate code	2024-01-10 14:14:48 +00:00
makaveli10	0f9e93d203	add: tensorrt backend to server	2024-01-09 18:10:17 +00:00
makaveli10	fd86340f30	add: tensorrt backend	2024-01-09 18:09:50 +00:00
makaveli10	2300eedc8b	fix: error messages	2024-01-09 13:35:38 +08:00
makaveli	cafcb04fbc	Merge pull request #92 from hcljsq/main feat: set initial_prompt and vad_parameters in the first message	2024-01-09 10:00:48 +05:30
Chen Hua	72ead71eeb	feat: set initial_prompt and vad_parameters in the first message	2024-01-08 15:33:55 +08:00
makaveli	7b2f5cff72	Merge pull request #89 from hcljsq/main format segment timestamps	2024-01-03 21:27:06 +05:30
Chen Hua	32c6a565d7	refactor(server): add format_segment helper to standardize timestamp output	2024-01-03 12:25:25 +08:00
makaveli	e30286c046	Merge pull request #83 from Chronoz/fix_exception_on_overflow fix exception on overflow	2024-01-02 19:01:36 +05:30
makaveli10	7c0b32b85e	bump version 0.0.10	2024-01-01 18:12:51 +05:30
makaveli	01665a54c1	Merge pull request #82 from hcljsq/main Add the `large-v3` model	2024-01-01 18:09:41 +05:30
华晨	02793a93f8	feat: Update transcriber to support large-v3 model with 128 mel filters	2024-01-01 20:22:37 +08:00
makaveli	db2e0bbcdd	Merge pull request #81 from k0hacuu/main README Spelling correction	2024-01-01 13:53:38 +05:30
Chronoz	e92ddd291a	fix exception on overflow	2023-12-31 20:59:33 +07:00
华晨	5918b5ed42	fix: update faster-whisper	2023-12-31 15:45:41 +08:00
华晨	71d207a607	feat: add large-v3 model	2023-12-30 23:19:30 +08:00
Jonny Yang	6ee4cd09f2	Update README.md	2023-12-25 13:12:24 +00:00
Marcus Edel	5de4de4b84	Merge pull request #76 from makaveli10/model_size_option Model size option.	2023-12-20 09:10:57 -05:00
makaveli10	e006722da7	add model size option to client	2023-12-20 18:06:02 +05:30
makaveli10	a52dc0cbf8	update chrome/firefox plugin readme	2023-12-15 00:09:55 +05:30
makaveli10	048ab0a8f4	remove debugging script	2023-12-15 02:26:01 +08:00
makaveli10	261bb9e961	Merge branch 'model_size_option' of github.com:makaveli10/whisper-live into model_size_option	2023-12-15 02:23:20 +08:00
makaveli10	091f6179d4	add sample audio for testing	2023-12-15 02:23:13 +08:00
makaveli10	1e1349cd80	update firefox plugin with model size dropdown	2023-12-14 23:52:51 +05:30
makaveli10	14beb4f942	update chrome plugin with model size dropdown	2023-12-14 23:52:25 +05:30
makaveli10	7ffcad64ba	update readme	2023-12-15 02:21:35 +08:00
makaveli10	402fceb9f3	remove emptyline	2023-12-15 02:21:09 +08:00
makaveli10	09b18e8ab8	add model size option from server	2023-12-14 22:08:12 +08:00
makaveli	da72d03073	Merge pull request #73 from jhormigo/hls_support Support for HLS transcription	2023-12-13 00:05:11 +05:30
Jesús Hormigo	a1a8d5f92a	Added a HLS stream sample URL	2023-12-12 13:46:52 +01:00
Jesús Hormigo	b6dee4e46e	Using ffmpeg-python package instead of requiring having ffmpeg installed in system	2023-12-10 19:34:27 +01:00
Jesús Hormigo	f3cd20fbf3	Support for HLS transcription (resolves #62 )	2023-12-09 21:24:55 +01:00
makaveli10	da86c18205	bump version: 0.0.9	2023-12-06 15:09:50 +05:30
makaveli	8097e9b44a	Merge pull request #70 from ethanzrd/main Update `faster_whisper` version setup.py	2023-12-06 15:04:06 +05:30
Ethan Zerad	222852ff33	Update setup.py Update faster-whisper version to match requirements/server.txt	2023-12-02 11:07:41 +02:00
Marcus Edel	2de67ee02f	Merge pull request #64 from makaveli10/update_faster_whisper Update faster whisper.	2023-11-20 10:26:42 -05:00
makaveli10	073cfc20f3	update version faster whisper server requirements	2023-11-20 19:05:02 +08:00
makaveli10	ee80bd21bd	update faster_whisper backend	2023-11-20 19:04:31 +08:00
makaveli	410b91d133	Merge pull request #58 from makaveli10/fix_chrome_ext Fix chrome extension errors in the options tab	2023-10-19 00:01:39 +05:30
makaveli10	a2b5220738	send empty responses and remove async; its not helping	2023-10-18 16:52:47 +02:00
makaveli10	1938dfb490	remove irrelevant reload	2023-10-18 16:52:07 +02:00